package org.sonar.search;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.hppc.cursors.ObjectCursor;
-import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.internal.InternalNode;
import org.slf4j.LoggerFactory;
-import org.sonar.process.MessageException;
import org.sonar.process.MinimumViableSystem;
import org.sonar.process.Monitored;
import org.sonar.process.ProcessEntryPoint;
node = new InternalNode(settings.build(), true);
node.start();
-
- // When joining a cluster, make sur the master(s) have a
- // replication factor on all indices > 0
- if (settings.inCluster() && !settings.isMaster()) {
- for (ObjectCursor<Settings> settingCursor : node.client().admin().indices()
- .prepareGetSettings().get().getIndexToSettings().values()) {
- Settings settings = settingCursor.value;
- String clusterReplicationFactor = settings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "-1");
- if (Integer.parseInt(clusterReplicationFactor) <= 0) {
- node.stop();
- throw new MessageException("Invalid number of Elasticsearch replicas: " + clusterReplicationFactor);
- }
- }
- }
}
@Override
import java.util.Properties;
import static org.fest.assertions.Assertions.assertThat;
-import static org.fest.assertions.Fail.fail;
public class SearchServerTest {
slaveServer.awaitStop();
}
- @Test
- public void slave_failed_replication() throws Exception {
- Props props = new Props(new Properties());
- props.set(ProcessConstants.SEARCH_PORT, String.valueOf(port));
- props.set(ProcessConstants.CLUSTER_ACTIVATE, "false");
- props.set(ProcessConstants.CLUSTER_NAME, CLUSTER_NAME);
- props.set(ProcessConstants.CLUSTER_NODE_NAME, "NOT_MASTER");
- props.set(ProcessConstants.PATH_HOME, temp.newFolder().getAbsolutePath());
- searchServer = new SearchServer(props);
- assertThat(searchServer).isNotNull();
-
- searchServer.start();
- assertThat(searchServer.isReady()).isTrue();
-
- client = getSearchClient();
- client.admin().indices().prepareCreate("test").get();
-
- // start a slave
- props = new Props(new Properties());
- props.set(ProcessConstants.CLUSTER_ACTIVATE, "true");
- props.set(ProcessConstants.CLUSTER_MASTER, "false");
- props.set(ProcessConstants.CLUSTER_MASTER_HOST, "localhost:" + port);
- props.set(ProcessConstants.CLUSTER_NAME, CLUSTER_NAME);
- props.set(ProcessConstants.CLUSTER_NODE_NAME, "SLAVE");
- props.set(ProcessConstants.SEARCH_PORT, String.valueOf(NetworkUtils.freePort()));
- props.set(ProcessConstants.PATH_HOME, temp.newFolder().getAbsolutePath());
- SearchServer slaveServer = new SearchServer(props);
- assertThat(slaveServer).isNotNull();
-
- try {
- slaveServer.start();
- fail();
- } catch (Exception e) {
- assertThat(e).hasMessage("Invalid number of Elasticsearch replicas: 0");
- }
-
- assertThat(client.admin().cluster().prepareClusterStats().get()
- .getNodesStats().getCounts().getTotal()).isEqualTo(1);
-
- slaveServer.stop();
- slaveServer.awaitStop();
- }
-
private Client getSearchClient() {
Settings settings = ImmutableSettings.settingsBuilder()
.put("cluster.name", CLUSTER_NAME).build();
import org.sonar.server.rule.db.RuleDao;
import org.sonar.server.user.db.GroupDao;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.IdentityHashMap;
import java.util.Map;
private <K> K getDao(Map<Class, DaoComponent> map, Class<K> clazz) {
return (K) map.get(clazz);
}
+
+ /**
+ * Create a PreparedStatement for SELECT requests with scrolling of results
+ */
+ public final PreparedStatement newScrollingSelectStatement(Connection connection, String sql) {
+ try {
+ PreparedStatement stmt = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ stmt.setFetchSize(database().getDialect().getScrollDefaultFetchSize());
+ return stmt;
+ } catch (SQLException e) {
+ throw new IllegalStateException("Fail to create SQL statement: " + sql, e);
+ }
+ }
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.db;
+
+import org.apache.commons.dbutils.DbUtils;
+
+import javax.annotation.CheckForNull;
+
+import java.io.Closeable;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Iterator;
+
+/**
+ * {@link java.util.Iterator} applied to {@link java.sql.ResultSet}
+ */
+public abstract class ResultSetIterator<E> implements Iterator<E>, Closeable {
+
+ private final ResultSet rs;
+ private final PreparedStatement stmt;
+
+ // TODO can be simpler by using rs.isLast(). See ResultSetIterator from commons-dbutils
+ private boolean didNext = false;
+ private boolean hasNext = false;
+
+ public ResultSetIterator(PreparedStatement stmt) throws SQLException {
+ this.stmt = stmt;
+ this.rs = stmt.executeQuery();
+ }
+
+ protected ResultSetIterator(ResultSet rs) {
+ this.stmt = null;
+ this.rs = rs;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!didNext) {
+ hasNext = doNextQuietly();
+ didNext = true;
+ }
+ return hasNext;
+ }
+
+ @Override
+ @CheckForNull
+ public E next() {
+ if (!didNext) {
+ doNextQuietly();
+ }
+ didNext = false;
+ try {
+ return read(rs);
+ } catch (SQLException e) {
+ // TODO add SQL request to context
+ throw new IllegalStateException("Fail to read result set row", e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ DbUtils.closeQuietly(rs);
+ DbUtils.closeQuietly(stmt);
+ }
+
+ protected abstract E read(ResultSet rs) throws SQLException;
+
+ private boolean doNextQuietly() {
+ try {
+ return rs.next();
+ } catch (SQLException e) {
+ throw new IllegalStateException("Fail to read row of JDBC result set", e);
+ }
+ }
+}
}
static SelectImpl create(Database db, Connection connection, String sql) throws SQLException {
+ // TODO use DbClient#newScrollingSelectStatement()
PreparedStatement pstmt = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
pstmt.setFetchSize(db.getDialect().getScrollDefaultFetchSize());
return new SelectImpl(pstmt);
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Date;
public class SqlUtil {
String s = rs.getString(columnName);
return rs.wasNull() ? null : s;
}
+
+ @CheckForNull
+ public static Long getLong(ResultSet rs, int columnIndex) throws SQLException {
+ long l = rs.getLong(columnIndex);
+ return rs.wasNull() ? null : l;
+ }
+
+ @CheckForNull
+ public static Double getDouble(ResultSet rs, int columnIndex) throws SQLException {
+ double d = rs.getDouble(columnIndex);
+ return rs.wasNull() ? null : d;
+ }
+
+ @CheckForNull
+ public static Integer getInt(ResultSet rs, int columnIndex) throws SQLException {
+ int i = rs.getInt(columnIndex);
+ return rs.wasNull() ? null : i;
+ }
+
+ @CheckForNull
+ public static String getString(ResultSet rs, int columnIndex) throws SQLException {
+ String s = rs.getString(columnIndex);
+ return rs.wasNull() ? null : s;
+ }
+
+ public static Date getDate(ResultSet rs, int columnIndex) throws SQLException {
+ Timestamp t = rs.getTimestamp(columnIndex);
+ return rs.wasNull() ? null : new Date(t.getTime());
+ }
}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import org.elasticsearch.action.ActionRequest;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-public class BulkIndexRequestIterator<INPUT> implements Iterator<ActionRequest> {
-
- public static interface InputConverter<INPUT> {
- List<ActionRequest> convert(INPUT input);
- }
-
- private final Iterator<INPUT> input;
- private final InputConverter<INPUT> converter;
- private Iterator<ActionRequest> currents = null;
-
- public BulkIndexRequestIterator(Iterable<INPUT> input, InputConverter<INPUT> converter) {
- this.input = input.iterator();
- this.converter = converter;
- if (this.input.hasNext()) {
- this.currents = converter.convert(this.input.next()).iterator();
- }
- }
-
- @Override
- public boolean hasNext() {
- return currents != null && currents.hasNext();
- }
-
- @Override
- public ActionRequest next() {
- if (currents == null) {
- throw new NoSuchElementException();
- }
- ActionRequest request = currents.next();
- peekNext();
- return request;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- private void peekNext() {
- if (!currents.hasNext()) {
- if (input.hasNext()) {
- currents = converter.convert(input.next()).iterator();
- } else {
- currents = null;
- }
- }
- }
-}
*/
package org.sonar.server.es;
-import com.google.common.collect.ImmutableMap;
-import org.apache.commons.lang.StringUtils;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
-import org.sonar.server.search.SearchClient;
+import org.picocontainer.Startable;
-import java.util.Iterator;
+import java.util.Map;
/**
- *
+ * Helper to bulk requests in an efficient way :
+ * <ul>
+ * <li>bulk request is sent on the wire when its size is higher than 5Mb</li>
+ * <li>on large table indexing, replicas and automatic refresh can be temporarily disabled</li>
+ * <li>index refresh is optional (enabled by default)</li>
+ * </ul>
*/
-public class BulkIndexer {
+public class BulkIndexer implements Startable {
+
+ public static final long FLUSH_BYTE_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytes();
- private static final long FLUSH_BYTE_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytes();
-
- private final SearchClient client;
+ private final EsClient client;
+ private final String indexName;
+ private boolean large = false;
+ private boolean refresh = true;
+ private long flushByteSize = FLUSH_BYTE_SIZE;
+ private BulkRequestBuilder bulkRequest = null;
+ private Map<String, Object> largeInitialSettings = null;
- public BulkIndexer(SearchClient client) {
+ public BulkIndexer(EsClient client, String indexName) {
this.client = client;
+ this.indexName = indexName;
}
/**
- * Heavy operation that populates an index from scratch. Replicas are disabled during
- * the bulk indexation and lucene segments are optimized at the end. No need
- * to call {@link #refresh(String)} after this method.
- *
- * @see BulkIndexRequestIterator
+ * Large indexing is an heavy operation that populates an index generally from scratch. Replicas and
+ * automatic refresh are disabled during bulk indexing and lucene segments are optimized at the end.
*/
- public void fullIndex(String index, Iterator<ActionRequest> requests) {
- // deactivate replicas
- GetSettingsRequestBuilder replicaRequest = client.admin().indices().prepareGetSettings(index);
- String initialRequestSetting = replicaRequest.get().getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS);
- int initialReplicas = Integer.parseInt(StringUtils.defaultIfEmpty(initialRequestSetting, "0"));
- if (initialReplicas > 0) {
- setNumberOfReplicas(index, 0);
- }
- index(requests);
- refresh(index);
- optimize(index);
-
- if (initialReplicas > 0) {
- // re-enable replicas
- setNumberOfReplicas(index, initialReplicas);
- }
+ public BulkIndexer setLarge(boolean b) {
+ Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started");
+ this.large = b;
+ return this;
}
- private void setNumberOfReplicas(String index, int replicas) {
- UpdateSettingsRequestBuilder req = client.admin().indices().prepareUpdateSettings(index);
- req.setSettings(ImmutableMap.<String, Object>of(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, String.valueOf(replicas)));
- req.get();
+ public BulkIndexer setRefresh(boolean b) {
+ Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started");
+ this.refresh = b;
+ return this;
}
/**
- * @see BulkIndexRequestIterator
+ * Default value is {@link org.sonar.server.es.BulkIndexer#FLUSH_BYTE_SIZE}
+ * @see org.elasticsearch.common.unit.ByteSizeValue
*/
- public void index(Iterator<ActionRequest> requests) {
- BulkRequest bulkRequest = client.prepareBulk().request();
- while (requests.hasNext()) {
- ActionRequest request = requests.next();
- bulkRequest.add(request);
- if (bulkRequest.estimatedSizeInBytes() >= FLUSH_BYTE_SIZE) {
- executeBulk(bulkRequest);
- bulkRequest = client.prepareBulk().request();
+ public BulkIndexer setFlushByteSize(long l) {
+ this.flushByteSize = l;
+ return this;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started");
+ if (large) {
+ largeInitialSettings = Maps.newHashMap();
+ Map<String, Object> bulkSettings = Maps.newHashMap();
+ GetSettingsResponse settingsResp = client.nativeClient().admin().indices().prepareGetSettings(indexName).get();
+
+ // deactivate replicas
+ int initialReplicas = Integer.parseInt(settingsResp.getSetting(indexName, IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
+ if (initialReplicas > 0) {
+ largeInitialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas);
+ bulkSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
}
+
+ // deactivate periodical refresh
+ String refreshInterval = settingsResp.getSetting(indexName, "index.refresh_interval");
+ largeInitialSettings.put("index.refresh_interval", refreshInterval);
+ bulkSettings.put("index.refresh_interval", "-1");
+
+ updateSettings(bulkSettings);
}
- if (bulkRequest.numberOfActions() > 0) {
+ bulkRequest = client.prepareBulk();
+ }
+
+ public void add(ActionRequest request) {
+ bulkRequest.request().add(request);
+ if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) {
executeBulk(bulkRequest);
}
}
- private void executeBulk(BulkRequest bulkRequest) {
- try {
- BulkResponse response = client.bulk(bulkRequest).get();
+ @Override
+ public void stop() {
+ if (bulkRequest.numberOfActions() > 0) {
+ executeBulk(bulkRequest);
+ }
+ if (refresh) {
+ client.prepareRefresh(indexName).get();
+ }
+ if (large) {
+ // optimize lucene segments and revert index settings
+ // Optimization must be done before re-applying replicas:
+ // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/
+ // TODO do not use nativeClient, else request is not profiled
+ client.nativeClient().admin().indices().prepareOptimize(indexName)
+ .setMaxNumSegments(1)
+ .setWaitForMerge(true)
+ .get();
- // TODO check failures
- // WARNING - complexity of response#hasFailures() and #buildFailureMessages() is O(n)
- } catch (Exception e) {
- throw new IllegalStateException("TODO", e);
+ updateSettings(largeInitialSettings);
}
+ bulkRequest = null;
}
- public void refresh(String index) {
- client.prepareRefresh(index).get();
+ private void updateSettings(Map<String, Object> settings) {
+ UpdateSettingsRequestBuilder req = client.nativeClient().admin().indices().prepareUpdateSettings(indexName);
+ req.setSettings(settings);
+ req.get();
}
- private void optimize(String index) {
- client.admin().indices().prepareOptimize(index)
- .setMaxNumSegments(1)
- .setWaitForMerge(true)
- .get();
- }
+ private void executeBulk(BulkRequestBuilder bulkRequest) {
+ bulkRequest.get();
+ // TODO check failures
+ // WARNING - complexity of response#hasFailures() and #buildFailureMessages() is O(n)
+ }
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.ImmutableSettings;
+
+class DefaultIndexSettings {
+
+ private DefaultIndexSettings() {
+ // only static stuff
+ }
+
+ static ImmutableSettings.Builder defaults() {
+ return ImmutableSettings.builder()
+ .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+ .put("index.refresh_interval", "30s")
+ .put("index.mapper.dynamic", false)
+
+ // Sortable text analyzer
+ .put("index.analysis.analyzer.sortable.type", "custom")
+ .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
+ .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase")
+
+ // Edge NGram index-analyzer
+ .put("index.analysis.analyzer.index_grams.type", "custom")
+ .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
+ .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
+
+ // Edge NGram search-analyzer
+ .put("index.analysis.analyzer.search_grams.type", "custom")
+ .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
+ .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
+
+ // Word index-analyzer
+ .put("index.analysis.analyzer.index_words.type", "custom")
+ .put("index.analysis.analyzer.index_words.tokenizer", "standard")
+ .putArray("index.analysis.analyzer.index_words.filter",
+ "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
+
+ // Word search-analyzer
+ .put("index.analysis.analyzer.search_words.type", "custom")
+ .put("index.analysis.analyzer.search_words.tokenizer", "standard")
+ .putArray("index.analysis.analyzer.search_words.filter",
+ "standard", "lowercase", "stop", "asciifolding", "porter_stem")
+
+ // Edge NGram filter
+ .put("index.analysis.filter.gram_filter.type", "edgeNGram")
+ .put("index.analysis.filter.gram_filter.min_gram", 2)
+ .put("index.analysis.filter.gram_filter.max_gram", 15)
+ .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
+
+ // Word filter
+ .put("index.analysis.filter.word_filter.type", "word_delimiter")
+ .put("index.analysis.filter.word_filter.generate_word_parts", true)
+ .put("index.analysis.filter.word_filter.catenate_words", true)
+ .put("index.analysis.filter.word_filter.catenate_numbers", true)
+ .put("index.analysis.filter.word_filter.catenate_all", true)
+ .put("index.analysis.filter.word_filter.split_on_case_change", true)
+ .put("index.analysis.filter.word_filter.preserve_original", true)
+ .put("index.analysis.filter.word_filter.split_on_numerics", true)
+ .put("index.analysis.filter.word_filter.stem_english_possessive", true)
+
+ // Path Analyzer
+ .put("index.analysis.analyzer.path_analyzer.type", "custom")
+ .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy")
+
+ // UUID Module analyzer
+ .put("index.analysis.tokenizer.dot_tokenizer.type", "pattern")
+ .put("index.analysis.tokenizer.dot_tokenizer.pattern", "\\.")
+ .put("index.analysis.analyzer.uuid_analyzer.type", "custom")
+ .putArray("index.analysis.analyzer.uuid_analyzer.filter", "trim", "lowercase")
+ .put("index.analysis.analyzer.uuid_analyzer.tokenizer", "dot_tokenizer");
+
+ }
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.ImmutableSettings;
-
-class DefaultMappingSettings {
-
- private DefaultMappingSettings() {
- // only static stuff
- }
-
- static ImmutableSettings.Builder defaults() {
- return ImmutableSettings.builder()
- .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
- .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
- .put("index.refresh_interval", "30s")
- .put("index.mapper.dynamic", false)
-
- // Sortable text analyzer
- .put("index.analysis.analyzer.sortable.type", "custom")
- .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
- .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase")
-
- // Edge NGram index-analyzer
- .put("index.analysis.analyzer.index_grams.type", "custom")
- .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
- .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
-
- // Edge NGram search-analyzer
- .put("index.analysis.analyzer.search_grams.type", "custom")
- .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
- .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
-
- // Word index-analyzer
- .put("index.analysis.analyzer.index_words.type", "custom")
- .put("index.analysis.analyzer.index_words.tokenizer", "standard")
- .putArray("index.analysis.analyzer.index_words.filter",
- "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
-
- // Word search-analyzer
- .put("index.analysis.analyzer.search_words.type", "custom")
- .put("index.analysis.analyzer.search_words.tokenizer", "standard")
- .putArray("index.analysis.analyzer.search_words.filter",
- "standard", "lowercase", "stop", "asciifolding", "porter_stem")
-
- // Edge NGram filter
- .put("index.analysis.filter.gram_filter.type", "edgeNGram")
- .put("index.analysis.filter.gram_filter.min_gram", 2)
- .put("index.analysis.filter.gram_filter.max_gram", 15)
- .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
-
- // Word filter
- .put("index.analysis.filter.word_filter.type", "word_delimiter")
- .put("index.analysis.filter.word_filter.generate_word_parts", true)
- .put("index.analysis.filter.word_filter.catenate_words", true)
- .put("index.analysis.filter.word_filter.catenate_numbers", true)
- .put("index.analysis.filter.word_filter.catenate_all", true)
- .put("index.analysis.filter.word_filter.split_on_case_change", true)
- .put("index.analysis.filter.word_filter.preserve_original", true)
- .put("index.analysis.filter.word_filter.split_on_numerics", true)
- .put("index.analysis.filter.word_filter.stem_english_possessive", true)
-
- // Path Analyzer
- .put("index.analysis.analyzer.path_analyzer.type", "custom")
- .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy")
-
- // UUID Module analyzer
- .put("index.analysis.tokenizer.dot_tokenizer.type", "pattern")
- .put("index.analysis.tokenizer.dot_tokenizer.pattern", "\\.")
- .put("index.analysis.analyzer.uuid_analyzer.type", "custom")
- .putArray("index.analysis.analyzer.uuid_analyzer.filter", "trim", "lowercase")
- .put("index.analysis.analyzer.uuid_analyzer.tokenizer", "dot_tokenizer");
-
- }
-}
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.Client;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.picocontainer.Startable;
import org.sonar.core.profiling.Profiling;
import org.sonar.server.search.ClusterHealth;
this.client = deprecatedClient;
}
- public EsClient(Profiling profiling, Client client) {
+ EsClient(Profiling profiling, Client client) {
this.profiling = profiling;
this.client = client;
}
return new ProxyDeleteByQueryRequestBuilder(client, profiling).setIndices(indices);
}
+ public long getLastUpdatedAt(String indexName, String typeName) {
+ SearchRequestBuilder request = prepareSearch(indexName)
+ .setTypes(typeName)
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setSize(0)
+ .addAggregation(AggregationBuilders.max("latest").field("updatedAt"));
+
+ Max max = request.get().getAggregations().get("latest");
+ return (long) max.getValue();
+ }
+
@Override
public void start() {
// nothing to do
@Override
public void stop() {
- client.close();
+ // TODO re-enable when SearchClient is dropped
+ //client.close();
}
protected Client nativeClient() {
import org.picocontainer.Startable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.sonar.api.ServerComponent;
import java.util.Map;
/**
* Create registered indices in Elasticsearch.
*/
-public class IndexCreator implements Startable {
+public class IndexCreator implements ServerComponent, Startable {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexCreator.class);
private static final String SETTING_HASH = "sonar_hash";
private final EsClient client;
- private final IndexDefinition[] definitions;
+ private final IndexRegistry registry;
- public IndexCreator(EsClient client, IndexDefinition[] definitions) {
+ public IndexCreator(EsClient client, IndexRegistry registry) {
this.client = client;
- this.definitions = definitions;
+ this.registry = registry;
}
@Override
public void start() {
- create();
- }
-
- @Override
- public void stop() {
- // nothing to do
- }
-
- public void create() {
- // collect definitions
- IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
- for (IndexDefinition definition : definitions) {
- definition.define(context);
- }
-
// create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...)
- for (NewIndex newIndex : context.getIndices().values()) {
- boolean exists = client.prepareExists(newIndex.getName()).get().isExists();
+ for (IndexRegistry.Index index : registry.getIndices().values()) {
+ boolean exists = client.prepareExists(index.getName()).get().isExists();
if (exists) {
- if (needsToDeleteIndex(newIndex)) {
- LOGGER.info(String.format("Delete index %s (settings changed)", newIndex.getName()));
- deleteIndex(newIndex.getName());
+ if (needsToDeleteIndex(index)) {
+ LOGGER.info(String.format("Delete index %s (settings changed)", index.getName()));
+ deleteIndex(index.getName());
exists = false;
}
}
if (!exists) {
- createIndex(newIndex);
+ createIndex(index);
}
}
}
- private void createIndex(NewIndex newIndex) {
- LOGGER.info(String.format("Create index %s", newIndex.getName()));
- ImmutableSettings.Builder settings = newIndex.getSettings();
- settings.put(SETTING_HASH, new IndexHash().of(newIndex));
+ @Override
+ public void stop() {
+ // nothing to do
+ }
+
+ private void createIndex(IndexRegistry.Index index) {
+ LOGGER.info(String.format("Create index %s", index.getName()));
+ ImmutableSettings.Builder settings = ImmutableSettings.builder();
+ settings.put(index.getSettings());
+ settings.put(SETTING_HASH, new IndexHash().of(index));
client
- .prepareCreate(newIndex.getName())
+ .prepareCreate(index.getName())
.setSettings(settings)
.get();
// create types
- for (Map.Entry<String, NewIndex.NewMapping> entry : newIndex.getMappings().entrySet()) {
- LOGGER.info(String.format("Create type %s/%s", newIndex.getName(), entry.getKey()));
- client.preparePutMapping(newIndex.getName())
+ for (Map.Entry<String, IndexRegistry.IndexType> entry : index.getTypes().entrySet()) {
+ LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey()));
+ client.preparePutMapping(index.getName())
.setType(entry.getKey())
.setIgnoreConflicts(false)
.setSource(entry.getValue().getAttributes())
client.nativeClient().admin().indices().prepareDelete(indexName).get();
}
- private boolean needsToDeleteIndex(NewIndex index) {
+ private boolean needsToDeleteIndex(IndexRegistry.Index index) {
boolean toBeDeleted = false;
String hash = client.nativeClient().admin().indices().prepareGetSettings(index.getName()).get().getSetting(index.getName(), "index." + SETTING_HASH);
if (hash != null) {
*/
package org.sonar.server.es;
-import java.util.HashMap;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.sonar.api.ServerComponent;
+
import java.util.Map;
-public interface IndexDefinition {
+public interface IndexDefinition extends ServerComponent {
public static class IndexDefinitionContext {
- private final Map<String, NewIndex> byKey = new HashMap<String, NewIndex>();
+ private final Map<String, NewIndex> byKey = Maps.newHashMap();
public NewIndex create(String key) {
- NewIndex index = byKey.get(key);
- if (index == null) {
- index = new NewIndex(key);
- byKey.put(key, index);
- }
+ Preconditions.checkArgument(!byKey.containsKey(key), String.format("Index already exists: %s", key));
+ NewIndex index = new NewIndex(key);
+ byKey.put(key, index);
return index;
}
}
}
+
void define(IndexDefinitionContext context);
+
}
import com.google.common.collect.Lists;
import org.apache.commons.codec.digest.DigestUtils;
-import javax.annotation.Nullable;
-
import java.util.Collections;
import java.util.List;
import java.util.Map;
private static final char DELIMITER = ',';
- String of(NewIndex index) {
- return of(index.getSettings().internalMap(), index.getMappings());
+ String of(IndexRegistry.Index index) {
+ return of(index.getSettings().getAsMap(), index.getTypes());
}
String of(Map... maps) {
}
private void appendObject(StringBuilder sb, Object value) {
- if (value instanceof NewIndex.NewMapping) {
- appendMapping(sb, (NewIndex.NewMapping) value);
+ if (value instanceof IndexRegistry.IndexType) {
+ appendIndexType(sb, (IndexRegistry.IndexType) value);
} else if (value instanceof Map) {
appendMap(sb, (Map) value);
} else if (value instanceof Iterable) {
}
}
- private void appendMapping(StringBuilder sb, NewIndex.NewMapping mapping) {
- appendMap(sb, mapping.getAttributes());
+ private void appendIndexType(StringBuilder sb, IndexRegistry.IndexType type) {
+ appendMap(sb, type.getAttributes());
}
private void appendMap(StringBuilder sb, Map attributes) {
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.elasticsearch.common.settings.Settings;
+import org.picocontainer.Startable;
+import org.sonar.api.ServerComponent;
+
+import java.util.Map;
+
+public class IndexRegistry implements ServerComponent, Startable {
+
+ /**
+ * Immutable copy of {@link org.sonar.server.es.NewIndex}
+ */
+ public static class Index {
+ private final String name;
+ private final Settings settings;
+ private final Map<String, IndexType> types;
+
+ Index(NewIndex newIndex) {
+ this.name = newIndex.getName();
+ this.settings = newIndex.getSettings().build();
+ ImmutableMap.Builder<String, IndexType> builder = ImmutableMap.builder();
+ for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) {
+ IndexType type = new IndexType(newIndexType);
+ builder.put(type.getName(), type);
+ }
+ this.types = builder.build();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Settings getSettings() {
+ return settings;
+ }
+
+ public Map<String, IndexType> getTypes() {
+ return types;
+ }
+ }
+
+ /**
+ * Immutable copy of {@link org.sonar.server.es.NewIndex.NewIndexType}
+ */
+ public static class IndexType {
+ private final String name;
+ private final Map<String, Object> attributes;
+
+ private IndexType(NewIndex.NewIndexType newType) {
+ this.name = newType.getName();
+ this.attributes = ImmutableMap.copyOf(newType.getAttributes());
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+ }
+
+ private final Map<String, Index> byKey = Maps.newHashMap();
+ private final IndexDefinition[] defs;
+
+ public IndexRegistry(IndexDefinition[] defs) {
+ this.defs = defs;
+ }
+
+ public Map<String, Index> getIndices() {
+ return byKey;
+ }
+
+ @Override
+ public void start() {
+ // collect definitions
+ IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
+ for (IndexDefinition definition : defs) {
+ definition.define(context);
+ }
+
+ for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) {
+ byKey.put(entry.getKey(), new Index(entry.getValue()));
+ }
+ }
+
+ @Override
+ public void stop() {
+ // nothing to do
+ }
+}
public class IssueIndexDefinition implements IndexDefinition {
public static final String INDEX_ISSUES = "issues";
+
public static final String TYPE_ISSUE_AUTHORIZATION = "issueAuthorization";
public static final String TYPE_ISSUE = "issue";
+ public static final String FIELD_ISSUE_ACTION_PLAN = "actionPlan";
+ public static final String FIELD_ISSUE_ASSIGNEE = "assignee";
+ public static final String FIELD_ISSUE_ATTRIBUTES = "attributes";
+ public static final String FIELD_ISSUE_AUTHOR_LOGIN = "authorLogin";
+ public static final String FIELD_ISSUE_COMPONENT_UUID = "component";
+ public static final String FIELD_ISSUE_CREATED_AT = "createdAt";
+ public static final String FIELD_ISSUE_DEBT = "debt";
+ public static final String FIELD_ISSUE_EFFORT = "effort";
+ public static final String FIELD_ISSUE_FILE_PATH = "filePath";
+ public static final String FIELD_ISSUE_FUNC_CREATED_AT = "issueCreatedAt";
+ public static final String FIELD_ISSUE_FUNC_UPDATED_AT = "issueUpdatedAt";
+ public static final String FIELD_ISSUE_FUNC_CLOSED_AT = "issueClosedAt";
+ public static final String FIELD_ISSUE_KEY = "key";
+ public static final String FIELD_ISSUE_LANGUAGE = "language";
+ public static final String FIELD_ISSUE_LINE = "line";
+ public static final String FIELD_ISSUE_MESSAGE = "message";
+ public static final String FIELD_ISSUE_MODULE_UUID = "module";
+ public static final String FIELD_ISSUE_MODULE_PATH = "modulePath";
+ public static final String FIELD_ISSUE_PROJECT_UUID = "project";
+ public static final String FIELD_ISSUE_REPORTER = "reporter";
+ public static final String FIELD_ISSUE_RESOLUTION = "resolution";
+ public static final String FIELD_ISSUE_RULE_KEY = "ruleKey";
+ public static final String FIELD_ISSUE_SEVERITY = "severity";
+ public static final String FIELD_ISSUE_SEVERITY_VALUE = "severityValue";
+ public static final String FIELD_ISSUE_STATUS = "status";
+ public static final String FIELD_ISSUE_UPDATED_AT = "updatedAt";
+
private final Settings settings;
public IssueIndexDefinition(Settings settings) {
}
// type "issueAuthorization"
- NewIndex.NewMapping authorizationMapping = index.createMapping(TYPE_ISSUE_AUTHORIZATION);
+ NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_ISSUE_AUTHORIZATION);
authorizationMapping.setAttribute("_id", ImmutableMap.of("path", IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field()));
authorizationMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD);
authorizationMapping.stringFieldBuilder("project").build();
authorizationMapping.stringFieldBuilder("users").build();
// type "issue"
- NewIndex.NewMapping issueMapping = index.createMapping(TYPE_ISSUE);
+ NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE);
issueMapping.setAttribute("_id", ImmutableMap.of("path", IssueNormalizer.IssueField.KEY.field()));
issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_ISSUE_AUTHORIZATION));
issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", IssueNormalizer.IssueField.PROJECT.field()));
- issueMapping.stringFieldBuilder("component").build();
- issueMapping.stringFieldBuilder("actionPlan").build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_ACTION_PLAN).build();
// TODO do we really sort by assignee ?
- issueMapping.stringFieldBuilder("assignee").enableSorting().build();
- issueMapping.stringFieldBuilder("attributes").build();
- issueMapping.stringFieldBuilder("authorLogin").build();
- issueMapping.createDateTimeField("createdAt");
- issueMapping.createDoubleField("debt");
- issueMapping.createDoubleField("effort");
- issueMapping.stringFieldBuilder("filePath").enableSorting().build();
- issueMapping.createDateTimeField("issueCreatedAt");
- issueMapping.createDateTimeField("issueUpdatedAt");
- issueMapping.createDateTimeField("issueClosedAt");
- issueMapping.stringFieldBuilder("key").enableSorting().build();
- issueMapping.stringFieldBuilder("language").build();
- issueMapping.createIntegerField("line");
- issueMapping.stringFieldBuilder("message").build();
- issueMapping.stringFieldBuilder("module").build();
- issueMapping.createUuidPathField("modulePath");
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).build();
+ // TODO rename into componentUuid ? or restrict to fileUuid ?
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).build();
+ issueMapping.createDoubleField(FIELD_ISSUE_DEBT);
+ issueMapping.createDoubleField(FIELD_ISSUE_EFFORT);
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_FILE_PATH).enableSorting().build();
+ issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT);
+ issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT);
+ issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT);
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_KEY).enableSorting().build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_LANGUAGE).build();
+ issueMapping.createIntegerField(FIELD_ISSUE_LINE);
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).build();
+ issueMapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH);
// TODO do we need to sort by project ?
- issueMapping.stringFieldBuilder("project").enableSorting().build();
- issueMapping.stringFieldBuilder("reporter").build();
- issueMapping.stringFieldBuilder("resolution").build();
- issueMapping.stringFieldBuilder("ruleKey").build();
- issueMapping.stringFieldBuilder("severity").build();
+ // TODO rename into projectUuid ?
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_PROJECT_UUID).enableSorting().build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_REPORTER).build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_RESOLUTION).build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_RULE_KEY).build();
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_SEVERITY).build();
// TODO do we need to sort by severity ?
- issueMapping.createByteField("severityValue");
+ issueMapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE);
// TODO do we really sort by status ? If yes, then we should sort by "int value", but not by string key
- issueMapping.stringFieldBuilder("status").enableSorting().build();
- issueMapping.createDateTimeField("updatedAt");
+ issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).enableSorting().build();
+ // TODO is createdAt required ?
+ issueMapping.createDateTimeField(FIELD_ISSUE_CREATED_AT);
+ issueMapping.createDateTimeField(FIELD_ISSUE_UPDATED_AT);
}
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.elasticsearch.action.update.UpdateRequest;
+import org.sonar.api.ServerComponent;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.issue.index.IssueDoc;
+
+import java.sql.Connection;
+import java.util.Iterator;
+
+/**
+ * Not thread-safe
+ */
+public class IssueIndexer implements ServerComponent {
+
+ private final DbClient dbClient;
+ private final EsClient esClient;
+ private long lastUpdatedAt = 0L;
+
+ public IssueIndexer(DbClient dbClient, EsClient esClient) {
+ this.dbClient = dbClient;
+ this.esClient = esClient;
+ }
+
+ public void indexProjectPermissions() {
+ final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES);
+
+ DbSession dbSession = dbClient.openSession(false);
+ Connection dbConnection = dbSession.getConnection();
+ try {
+ IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbConnection, getLastUpdatedAt());
+ indexIssues(bulk, rowIt);
+ rowIt.close();
+
+ } finally {
+ dbSession.close();
+ }
+ }
+
+ public void indexIssues(boolean large) {
+ // TODO support timezones
+ final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES);
+ bulk.setLarge(large);
+
+ DbSession dbSession = dbClient.openSession(false);
+ Connection dbConnection = dbSession.getConnection();
+ try {
+ IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbConnection, getLastUpdatedAt());
+ indexIssues(bulk, rowIt);
+ rowIt.close();
+
+ } finally {
+ dbSession.close();
+ }
+ }
+
+ public void indexIssues(BulkIndexer bulk, Iterator<IssueDoc> issues) {
+ bulk.start();
+ while (issues.hasNext()) {
+ IssueDoc issue = issues.next();
+ bulk.add(newUpsertRequest(issue));
+
+ // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance)
+ long dtoUpdatedAt = issue.updateDate().getTime();
+ if (lastUpdatedAt < dtoUpdatedAt) {
+ lastUpdatedAt = dtoUpdatedAt;
+ }
+ }
+ bulk.stop();
+ }
+
+ private long getLastUpdatedAt() {
+ long result;
+ if (lastUpdatedAt <= 0L) {
+ // request ES to get the max(updatedAt)
+ result = esClient.getLastUpdatedAt(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE);
+ } else {
+ // use cache. Will not work with Tomcat cluster.
+ result = lastUpdatedAt;
+ }
+ return result;
+ }
+
+ private UpdateRequest newUpsertRequest(IssueDoc issue) {
+ String projectUuid = issue.projectUuid();
+ issue.setField("_parent", projectUuid);
+ return new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE, issue.key())
+ .routing(projectUuid)
+ .parent(projectUuid)
+ .doc(issue.getFields())
+ .upsert(issue.getFields());
+ }
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.db.ResultSetIterator;
+import org.sonar.server.db.migrations.SqlUtil;
+import org.sonar.server.issue.index.IssueDoc;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Date;
+
+/**
+ * Scroll over table ISSUES and directly read the maps required to
+ * post index requests
+ */
+class IssueResultSetIterator extends ResultSetIterator<IssueDoc> {
+
+ private static final String[] FIELDS = {
+ // column 1
+ "i.kee",
+ "root.uuid",
+ "i.updated_at",
+ "i.created_at",
+ "i.action_plan_key",
+ "i.assignee",
+ "i.effort_to_fix",
+ "i.issue_attributes",
+ "i.line",
+ "i.message",
+
+ // column 11
+ "i.resolution",
+ "i.severity",
+ "i.status",
+ "i.technical_debt",
+ "i.reporter",
+ "i.author_login",
+ "i.issue_close_date",
+ "i.issue_creation_date",
+ "i.issue_update_date",
+ "r.plugin_rule_key",
+
+ // column 21
+ "r.language",
+ "p.uuid",
+ "p.module_uuid",
+ "p.module_uuid_path",
+ "p.path"
+ };
+
+ private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " +
+ "inner join rules r on r.id=i.rule_id " +
+ "inner join projects p on p.id=i.component_id " +
+ "inner join projects root on root.id=i.root_component_id";
+
+ private static final String SQL_AFTER_DATE = SQL_ALL + " where i.updated_at>=?";
+
+ static IssueResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) {
+ try {
+ String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
+ PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql);
+ if (afterDate > 0L) {
+ stmt.setTimestamp(0, new Timestamp(afterDate));
+ }
+ return new IssueResultSetIterator(stmt);
+ } catch (SQLException e) {
+ throw new IllegalStateException("Fail to prepare SQL request to select all issues", e);
+ }
+ }
+
+ private IssueResultSetIterator(PreparedStatement stmt) throws SQLException {
+ super(stmt);
+ }
+
+ @Override
+ protected IssueDoc read(ResultSet rs) throws SQLException {
+ IssueDoc doc = new IssueDoc(Maps.<String, Object>newHashMapWithExpectedSize(30));
+
+ String key = rs.getString(1);
+ String projectUuid = rs.getString(2);
+
+ // all the keys must be present, even if value is null
+ doc.setKey(key);
+ doc.setProjectUuid(projectUuid);
+ doc.setUpdateDate(SqlUtil.getDate(rs, 3));
+ doc.setCreationDate(new Date(rs.getTimestamp(4).getTime()));
+ doc.setActionPlanKey(rs.getString(5));
+ doc.setAssignee(rs.getString(6));
+ doc.setEffortToFix(SqlUtil.getDouble(rs, 7));
+ doc.setAttributes(rs.getString(8));
+ doc.setLine(SqlUtil.getInt(rs, 9));
+ doc.setMessage(rs.getString(10));
+ doc.setResolution(rs.getString(11));
+ doc.setSeverity(rs.getString(12));
+ doc.setStatus(rs.getString(13));
+ doc.setDebt(SqlUtil.getLong(rs, 14));
+ doc.setReporter(rs.getString(15));
+ doc.setAuthorLogin(rs.getString(16));
+ doc.setFuncCloseDate(SqlUtil.getDate(rs, 17));
+ doc.setFuncCreationDate(SqlUtil.getDate(rs, 18));
+ doc.setFuncUpdateDate(SqlUtil.getDate(rs, 19));
+ doc.setRuleKey(rs.getString(20));
+ doc.setLanguage(rs.getString(21));
+ doc.setComponentUuid(rs.getString(22));
+ doc.setModuleUuid(rs.getString(23));
+ doc.setModuleUuidPath(rs.getString(24));
+ doc.setFilePath(rs.getString(25));
+ return doc;
+ }
+}
public class NewIndex {
- public static class NewMapping {
+ public static class NewIndexType {
+ private final String name;
private final Map<String, Object> attributes = new TreeMap<String, Object>();
private final Map<String, Object> properties = new TreeMap<String, Object>();
- private NewMapping() {
+ private NewIndexType(String typeName) {
+ this.name = typeName;
// defaults
attributes.put("dynamic", false);
attributes.put("_all", ImmutableSortedMap.of("enabled", false));
attributes.put("properties", properties);
}
+ public String getName() {
+ return name;
+ }
+
/**
* Complete the root json hash of mapping type, for example to set the attribute "_id"
*/
- public NewMapping setAttribute(String key, Object value) {
+ public NewIndexType setAttribute(String key, Object value) {
attributes.put(key, value);
return this;
}
/**
* Complete the json hash named "properties" in mapping type, usually to declare fields
*/
- public NewMapping setProperty(String key, Object value) {
+ public NewIndexType setProperty(String key, Object value) {
properties.put(key, value);
return this;
}
return new StringFieldBuilder(this, fieldName);
}
- public NewMapping createBooleanField(String fieldName) {
+ public NewIndexType createBooleanField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "boolean"));
}
- public NewMapping createByteField(String fieldName) {
+ public NewIndexType createByteField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "byte"));
}
- public NewMapping createDateTimeField(String fieldName) {
+ public NewIndexType createDateTimeField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time"));
}
- public NewMapping createDoubleField(String fieldName) {
+ public NewIndexType createDoubleField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "double"));
}
- public NewMapping createIntegerField(String fieldName) {
+ public NewIndexType createIntegerField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "integer"));
}
- public NewMapping createLongField(String fieldName) {
+ public NewIndexType createLongField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "long"));
}
- public NewMapping createShortField(String fieldName) {
+ public NewIndexType createShortField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "short"));
}
- public NewMapping createUuidPathField(String fieldName) {
+ public NewIndexType createUuidPathField(String fieldName) {
return setProperty(fieldName, ImmutableSortedMap.of(
"type", "string",
"index", "analyzed",
"type", "string",
"index", "not_analyzed");
- private final NewMapping newMapping;
+ private final NewIndexType indexType;
private final String fieldName;
private boolean sortable = false, wordSearch = false, gramSearch = false;
- private StringFieldBuilder(NewMapping newMapping, String fieldName) {
- this.newMapping = newMapping;
+ private StringFieldBuilder(NewIndexType indexType, String fieldName) {
+ this.indexType = indexType;
this.fieldName = fieldName;
}
hash.putAll(NOT_ANALYZED);
}
- newMapping.setProperty(fieldName, hash);
+ indexType.setProperty(fieldName, hash);
}
}
private final String indexName;
- private final ImmutableSettings.Builder settings = DefaultMappingSettings.defaults();
- private final SortedMap<String, NewMapping> mappings = new TreeMap<String, NewMapping>();
+ private final ImmutableSettings.Builder settings = DefaultIndexSettings.defaults();
+ private final SortedMap<String, NewIndexType> types = new TreeMap<String, NewIndexType>();
NewIndex(String indexName) {
Preconditions.checkArgument(StringUtils.isAllLowerCase(indexName), "Index name must be lower-case: " + indexName);
return settings;
}
- public NewMapping createMapping(String typeName) {
- NewMapping type = new NewMapping();
- mappings.put(typeName, type);
+ public NewIndexType createType(String typeName) {
+ NewIndexType type = new NewIndexType(typeName);
+ types.put(typeName, type);
return type;
}
- public SortedMap<String, NewMapping> getMappings() {
- return mappings;
+ public SortedMap<String, NewIndexType> getTypes() {
+ return types;
}
}
import org.sonar.api.issue.Issue;
import org.sonar.api.issue.IssueComment;
import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.Severity;
import org.sonar.api.utils.Duration;
import org.sonar.api.utils.KeyValueFormat;
+import org.sonar.server.es.IssueIndexDefinition;
import org.sonar.server.search.BaseDoc;
import org.sonar.server.search.IndexUtils;
import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
import java.util.Collections;
import java.util.Date;
@Override
public String key() {
- return getField(IssueNormalizer.IssueField.KEY.field());
+ return getField(IssueIndexDefinition.FIELD_ISSUE_KEY);
}
@Override
public String filePath() {
return getNullableField(IssueNormalizer.IssueField.FILE_PATH.field());
}
+
+ public void setKey(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_KEY, s);
+ }
+
+ public void setComponentUuid(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, s);
+ }
+
+ public void setModuleUuid(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, s);
+ }
+
+ public void setProjectUuid(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, s);
+ }
+
+ public void setRuleKey(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, s);
+ }
+
+ public void setLanguage(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_LANGUAGE, s);
+ }
+
+ public void setSeverity(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_SEVERITY, s);
+ setField(IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE, Severity.ALL.indexOf(s));
+ }
+
+ public void setMessage(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_MESSAGE, s);
+ }
+
+ public void setLine(@Nullable Integer i) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_LINE, i);
+ }
+
+ public void setEffortToFix(@Nullable Double d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_EFFORT, d);
+ }
+
+ public void setStatus(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_STATUS, s);
+ }
+
+ public void setResolution(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION, s);
+ }
+
+ public void setReporter(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_REPORTER, s);
+ }
+
+ public void setAssignee(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, s);
+ }
+
+ public void setCreationDate(Date d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_CREATED_AT, d);
+ }
+
+ public void setUpdateDate(Date d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_UPDATED_AT, d);
+ }
+
+ public void setFuncCreationDate(Date d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, d);
+ }
+
+ public void setFuncUpdateDate(Date d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT, d);
+ }
+
+ public void setFuncCloseDate(Date d) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_CLOSED_AT, d);
+ }
+
+ public void setAttributes(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_ATTRIBUTES, s);
+ }
+
+ public void setAuthorLogin(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, s);
+ }
+
+ public void setActionPlanKey(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_ACTION_PLAN, s);
+ }
+
+ public void setDebt(Long l) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_DEBT, l);
+ }
+
+ public void setFilePath(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_FILE_PATH, s);
+ }
+
+ public void setModuleUuidPath(String s) {
+ setField(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, s);
+ }
}
import org.sonar.server.duplication.ws.DuplicationsWs;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexCreator;
+import org.sonar.server.es.IndexRegistry;
import org.sonar.server.es.IssueIndexDefinition;
+import org.sonar.server.es.IssueIndexer;
import org.sonar.server.issue.ActionService;
import org.sonar.server.issue.AssignAction;
import org.sonar.server.issue.CommentAction;
pico.addSingleton(Periods.class);
pico.addSingleton(ServerWs.class);
pico.addSingleton(BackendCleanup.class);
+ pico.addSingleton(IndexRegistry.class);
pico.addSingleton(IndexCreator.class);
// batch
// issues
pico.addSingleton(IssueIndexDefinition.class);
+ pico.addSingleton(IssueIndexer.class);
pico.addSingleton(ServerIssueStorage.class);
pico.addSingleton(IssueUpdater.class);
pico.addSingleton(FunctionExecutor.class);
package org.sonar.server.search;
import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
import java.util.Map;
/**
}
return value;
}
+
+ public void setField(String key, @Nullable Object value) {
+ fields.put(key, value);
+ }
+
+ public Map<String, Object> getFields() {
+ return fields;
+ }
}
@Rule
public ExpectedException throwable = ExpectedException.none();
- @Test(timeout = 10000)
+ @Test(timeout = 5000)
public void should_start_and_stop() throws IOException {
int port = NetworkUtils.freePort();
database.stop();
}
- @Test(timeout = 10000)
- public void should_support_memory_database() throws IOException {
- int port = NetworkUtils.freePort();
-
- EmbeddedDatabase database = new EmbeddedDatabase(testSettings(port)
- .setProperty(DatabaseProperties.PROP_URL, "jdbc:h2:tcp://localhost:" + port + "/mem:sonarIT;USER=sonar;PASSWORD=sonar"));
- database.start();
-
- try {
- String driverUrl = String.format("jdbc:h2:tcp://localhost:%d/mem:sonarIT;USER=sonar;PASSWORD=sonar", port);
- DriverManager.registerDriver(new Driver());
- DriverManager.getConnection(driverUrl).close();
- } catch (Exception ex) {
- fail("Unable to connect after start");
- }
-
- database.stop();
- }
-
@Test
public void should_return_embedded_data_directory() throws Exception {
-
Settings settings = testSettings(0);
EmbeddedDatabase database = new EmbeddedDatabase(settings);
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.db;
+
+import org.apache.commons.dbutils.DbUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.core.persistence.TestDatabase;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.fest.assertions.Assertions.assertThat;
+import static org.fest.assertions.Fail.fail;
+
+public class ResultSetIteratorTest {
+
+ @Rule
+ public TestDatabase dbTester = new TestDatabase().schema(ResultSetIteratorTest.class, "schema.sql");
+
+ Connection connection = null;
+
+ @Before
+ public void setUp() throws Exception {
+ connection = dbTester.openConnection();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ DbUtils.closeQuietly(connection);
+ }
+
+ @Test
+ public void iterate_through_statement() throws Exception {
+ dbTester.prepareDbUnit(getClass(), "feed.xml");
+
+ PreparedStatement stmt = connection.prepareStatement("select * from fake order by id");
+ FirstIntColumnIterator iterator = new FirstIntColumnIterator(stmt);
+
+ assertThat(iterator.hasNext()).isTrue();
+
+ // calling multiple times hasNext() is ok
+ assertThat(iterator.hasNext()).isTrue();
+
+ assertThat(iterator.next()).isEqualTo(10);
+ assertThat(iterator.hasNext()).isTrue();
+ assertThat(iterator.next()).isEqualTo(20);
+
+ // call next() without calling hasNext()
+ assertThat(iterator.next()).isEqualTo(30);
+ assertThat(iterator.hasNext()).isFalse();
+
+ iterator.close();
+ // statement is closed by ResultSetIterator
+ assertThat(stmt.isClosed()).isTrue();
+ }
+
+ @Test
+ public void iterate_through_rs() throws Exception {
+ dbTester.prepareDbUnit(getClass(), "feed.xml");
+
+ PreparedStatement stmt = connection.prepareStatement("select * from fake order by id");
+ ResultSet rs = stmt.executeQuery();
+ FirstIntColumnIterator iterator = new FirstIntColumnIterator(rs);
+
+ assertThat(iterator.next()).isEqualTo(10);
+ assertThat(iterator.next()).isEqualTo(20);
+ assertThat(iterator.next()).isEqualTo(30);
+
+ iterator.close();
+ assertThat(rs.isClosed()).isTrue();
+ stmt.close();
+ }
+
+ @Test
+ public void remove_row_is_not_supported() throws Exception {
+ PreparedStatement stmt = connection.prepareStatement("select * from fake order by id");
+ FirstIntColumnIterator iterator = new FirstIntColumnIterator(stmt);
+
+ try {
+ iterator.remove();
+ fail();
+ } catch (UnsupportedOperationException ok) {
+ }
+
+ iterator.close();
+ }
+
+ @Test
+ public void fail_to_read_row() throws Exception {
+ dbTester.prepareDbUnit(getClass(), "feed.xml");
+
+ PreparedStatement stmt = connection.prepareStatement("select * from fake order by id");
+ FailIterator iterator = new FailIterator(stmt);
+
+ assertThat(iterator.hasNext()).isTrue();
+ try {
+ iterator.next();
+ fail();
+ } catch (IllegalStateException e) {
+ assertThat(e.getCause()).isInstanceOf(SQLException.class);
+ }
+ iterator.close();
+ }
+
+ private static class FirstIntColumnIterator extends ResultSetIterator<Integer> {
+
+ public FirstIntColumnIterator(PreparedStatement stmt) throws SQLException {
+ super(stmt);
+ }
+
+ public FirstIntColumnIterator(ResultSet rs) {
+ super(rs);
+ }
+
+ @Override
+ protected Integer read(ResultSet rs) throws SQLException {
+ return rs.getInt(1);
+ }
+ }
+
+ private static class FailIterator extends ResultSetIterator<Integer> {
+
+ public FailIterator(PreparedStatement stmt) throws SQLException {
+ super(stmt);
+ }
+
+ @Override
+ protected Integer read(ResultSet rs) throws SQLException {
+ // column does not exist
+ return rs.getInt(1234);
+ }
+ }
+}
import org.junit.Test;
import org.slf4j.Logger;
-import org.sonar.server.db.migrations.SqlUtil;
import java.sql.SQLException;
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import static org.fest.assertions.Assertions.assertThat;
-import static org.fest.assertions.Fail.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verifyZeroInteractions;
-
-public class BulkIndexRequestIteratorTest {
-
- @Test
- public void iterate_over_requests() throws Exception {
- List<String> input = Arrays.asList("foo", "bar", "3requests", "baz");
- BulkIndexRequestIterator.InputConverter converter = new BulkIndexRequestIterator.InputConverter<String>() {
- @Override
- public List<ActionRequest> convert(String input) {
- if ("3requests".equals(input)) {
- return Collections.nCopies(3, (ActionRequest) new IndexRequest(input));
- }
- return Arrays.asList((ActionRequest) new IndexRequest(input));
- }
- };
-
- BulkIndexRequestIterator<String> it = new BulkIndexRequestIterator<String>(input, converter);
-
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "foo");
-
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "bar");
-
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "3requests");
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "3requests");
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "3requests");
-
- assertThat(it.hasNext()).isTrue();
- assertIndex(it.next(), "baz");
-
- assertThat(it.hasNext()).isFalse();
- try {
- it.next();
- fail();
- } catch (NoSuchElementException e) {
- // expected
- }
- }
-
- @Test
- public void empty() throws Exception {
- List<String> input = Collections.emptyList();
- BulkIndexRequestIterator.InputConverter converter = mock(BulkIndexRequestIterator.InputConverter.class);
-
- BulkIndexRequestIterator<String> it = new BulkIndexRequestIterator<String>(input, converter);
-
- assertThat(it.hasNext()).isFalse();
- verifyZeroInteractions(converter);
- }
-
- @Test
- public void removal_is_not_supported() throws Exception {
- List<String> input = Arrays.asList("foo");
- BulkIndexRequestIterator.InputConverter converter = mock(BulkIndexRequestIterator.InputConverter.class);
-
- BulkIndexRequestIterator<String> it = new BulkIndexRequestIterator<String>(input, converter);
-
- try {
- it.remove();
- fail();
- } catch (UnsupportedOperationException e) {
- // expected
- }
- }
-
- private void assertIndex(ActionRequest req, String indexName) {
- assertThat(req).isNotNull();
- assertThat(req).isInstanceOf(IndexRequest.class);
- assertThat(((IndexRequest) req).index()).isEqualTo(indexName);
- }
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class BulkIndexerTest {
+
+ @Rule
+ public EsTester esTester = new EsTester().addDefinitions(new FakeIndexDefinition().setReplicas(1));
+
+ @Test
+ public void index_nothing() throws Exception {
+ BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX);
+ indexer.start();
+ indexer.stop();
+
+ assertThat(count()).isEqualTo(0);
+ }
+
+ @Test
+ public void index_documents() throws Exception {
+ BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX)
+ .setRefresh(true);
+ indexer.start();
+ indexer.add(newIndexRequest(42));
+ indexer.add(newIndexRequest(78));
+
+ // request is not sent yet
+ assertThat(count()).isEqualTo(0);
+
+ // send remaining requests
+ indexer.stop();
+ assertThat(count()).isEqualTo(2);
+ }
+
+ @Test
+ public void large_indexing() throws Exception {
+ // index has one replica
+ assertThat(replicas()).isEqualTo(1);
+
+ BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX)
+ .setLarge(true)
+ .setFlushByteSize(new ByteSizeValue(1, ByteSizeUnit.BYTES).bytes());
+ indexer.start();
+
+ // replicas are temporarily disabled
+ assertThat(replicas()).isEqualTo(0);
+
+ for (int i = 0; i < 10; i++) {
+ indexer.add(newIndexRequest(i));
+ }
+ indexer.stop();
+
+ assertThat(count()).isEqualTo(10);
+
+ // replicas are re-enabled
+ assertThat(replicas()).isEqualTo(1);
+ }
+
+ private long count() {
+ return esTester.client().prepareCount(FakeIndexDefinition.INDEX).get().getCount();
+ }
+
+ private int replicas() {
+ GetSettingsResponse settingsResp = esTester.client().nativeClient().admin().indices()
+ .prepareGetSettings(FakeIndexDefinition.INDEX).get();
+ return Integer.parseInt(settingsResp.getSetting(FakeIndexDefinition.INDEX, IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
+ }
+
+ private IndexRequest newIndexRequest(int intField) {
+ return new IndexRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.TYPE)
+ .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, intField));
+ }
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.elasticsearch.common.collect.ImmutableMap;
+import org.junit.Test;
+import org.sonar.test.TestUtils;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class DefaultIndexSettingsTest {
+
+ @Test
+ public void defaults() throws Exception {
+ ImmutableMap<String, String> map = DefaultIndexSettings.defaults().build().getAsMap();
+ assertThat(map).isNotEmpty();
+
+ // test some values
+ assertThat(map.get("index.number_of_shards")).isEqualTo("1");
+ assertThat(map.get("index.number_of_replicas")).isEqualTo("0");
+ assertThat(map.get("index.analysis.analyzer.sortable.type")).isEqualTo("custom");
+ }
+
+ @Test
+ public void only_statics() throws Exception {
+ TestUtils.hasOnlyPrivateConstructors(DefaultIndexSettings.class);
+
+ }
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import org.elasticsearch.common.collect.ImmutableMap;
-import org.junit.Test;
-import org.sonar.test.TestUtils;
-
-import static org.fest.assertions.Assertions.assertThat;
-
-public class DefaultMappingSettingsTest {
-
- @Test
- public void defaults() throws Exception {
- ImmutableMap<String, String> map = DefaultMappingSettings.defaults().build().getAsMap();
- assertThat(map).isNotEmpty();
-
- // test some values
- assertThat(map.get("index.number_of_shards")).isEqualTo("1");
- assertThat(map.get("index.number_of_replicas")).isEqualTo("0");
- assertThat(map.get("index.analysis.analyzer.sortable.type")).isEqualTo("custom");
- }
-
- @Test
- public void only_statics() throws Exception {
- TestUtils.hasOnlyPrivateConstructors(DefaultMappingSettings.class);
-
- }
-}
*/
package org.sonar.server.es;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.math.RandomUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.node.NodeBuilder;
import org.junit.rules.ExternalResource;
import org.sonar.api.config.Settings;
+import org.sonar.api.platform.ComponentContainer;
import org.sonar.core.profiling.Profiling;
-import java.util.Date;
+import java.util.Collections;
+import java.util.List;
import static org.fest.assertions.Assertions.assertThat;
public class EsTester extends ExternalResource {
- /**
- * This instance is shared for performance reasons. Never stopped.
- */
- private static Node sharedNode;
- private static EsClient client;
+ private static int PROCESS_ID = RandomUtils.nextInt(Integer.MAX_VALUE);
+ private Node node;
+ private EsClient client;
+ private final List<IndexDefinition> definitions = Lists.newArrayList();
+
+ public EsTester addDefinitions(IndexDefinition... defs) {
+ Collections.addAll(definitions, defs);
+ return this;
+ }
- @Override
protected void before() throws Throwable {
- if (sharedNode == null) {
- String nodeName = EsTester.class.getName();
- sharedNode = NodeBuilder.nodeBuilder().local(true).data(true).settings(ImmutableSettings.builder()
- .put(ClusterName.SETTING, nodeName)
- .put("node.name", nodeName)
- // the two following properties are probably not used because they are
- // declared on indices too
- .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
- .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
- // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors)
- .put("processors", 1)
- .put("http.enabled", false)
- .put("index.store.type", "ram")
- .put("config.ignore_system_properties", true)
- .put("gateway.type", "none"))
- .build();
- sharedNode.start();
- assertThat(DiscoveryNode.localNode(sharedNode.settings())).isTrue();
- client = new EsClient(new Profiling(new Settings()), sharedNode.client());
+ String nodeName = "es-ram-" + PROCESS_ID;
+ node = NodeBuilder.nodeBuilder().local(true).data(true).settings(ImmutableSettings.builder()
+ .put("cluster.name", nodeName)
+ .put("node.name", nodeName)
+ // the two following properties are probably not used because they are
+ // declared on indices too
+ .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+ // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors)
+ .put("processors", 1)
+ .put("http.enabled", false)
+ .put("index.store.type", "memory")
+ .put("config.ignore_system_properties", true)
+ // reuse the same directory than other tests for faster initialization
+ .put("path.home", "target/es-" + PROCESS_ID)
+ .put("gateway.type", "none"))
+ .build();
+ node.start();
+ assertThat(DiscoveryNode.localNode(node.settings())).isTrue();
+
+ // delete the indices created by previous tests
+ DeleteIndexResponse response = node.client().admin().indices().prepareDelete("_all").get();
+ assertThat(response.isAcknowledged()).isTrue();
+
+ client = new EsClient(new Profiling(new Settings()), node.client());
+ client.start();
- } else {
- // delete the indices created by previous tests
- DeleteIndexResponse response = sharedNode.client().admin().indices().prepareDelete("_all").get();
- assertThat(response.isAcknowledged()).isTrue();
+ if (!definitions.isEmpty()) {
+ ComponentContainer container = new ComponentContainer();
+ container.addSingletons(definitions);
+ container.addSingleton(client);
+ container.addSingleton(IndexRegistry.class);
+ container.addSingleton(IndexCreator.class);
+ container.startComponents();
+ }
+ }
+
+ @Override
+ protected void after() {
+ if (client != null) {
+ client.stop();
+ }
+ if (node != null) {
+ node.stop();
+ node.close();
}
}
}
public Node node() {
- return sharedNode;
+ return node;
}
public EsClient client() {
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+public class FakeIndexDefinition implements IndexDefinition {
+
+ public static final String INDEX = "fakes";
+ public static final String TYPE = "fake";
+ public static final String INT_FIELD = "intField";
+
+ private int replicas = 0;
+
+ public FakeIndexDefinition setReplicas(int replicas) {
+ this.replicas = replicas;
+ return this;
+ }
+
+ @Override
+ public void define(IndexDefinitionContext context) {
+ NewIndex index = context.create(INDEX);
+ index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
+ NewIndex.NewIndexType type = index.createType(TYPE);
+ type.createIntegerField(INT_FIELD);
+ }
+}
import org.junit.Test;
import javax.annotation.CheckForNull;
+
import java.io.IOException;
import java.util.Map;
public void create_index() throws Exception {
assertThat(mappings()).isEmpty();
- IndexCreator creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinition()});
+ IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()});
+ registry.start();
+ IndexCreator creator = new IndexCreator(es.client(), registry);
creator.start();
// check that index is created with related mapping
assertThat(mappings()).isNotEmpty();
}
- private String setting(String indexName, String settingKey) {
- GetSettingsResponse indexSettings = es.client().nativeClient().admin().indices().prepareGetSettings(indexName).get();
- return indexSettings.getSetting(indexName, settingKey);
- }
-
@Test
public void recreate_index_on_definition_changes() throws Exception {
assertThat(mappings()).isEmpty();
// v1
- IndexCreator creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinition()});
+ IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()});
+ registry.start();
+ IndexCreator creator = new IndexCreator(es.client(), registry);
creator.start();
creator.stop();
String hashV1 = setting("fakes", "index.sonar_hash");
assertThat(hashV1).isNotEmpty();
// v2
- creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinitionV2()});
+ registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinitionV2()});
+ registry.start();
+ creator = new IndexCreator(es.client(), registry);
creator.start();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings();
MappingMetaData mapping = mappings.get("fakes").get("fake");
creator.stop();
}
+ private String setting(String indexName, String settingKey) {
+ GetSettingsResponse indexSettings = es.client().nativeClient().admin().indices().prepareGetSettings(indexName).get();
+ return indexSettings.getSetting(indexName, settingKey);
+ }
+
private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() {
return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings();
}
@Override
public void define(IndexDefinitionContext context) {
NewIndex index = context.create("fakes");
- NewIndex.NewMapping mapping = index.createMapping("fake");
+ NewIndex.NewIndexType mapping = index.createType("fake");
mapping.stringFieldBuilder("key").build();
mapping.createDateTimeField("updatedAt");
}
@Override
public void define(IndexDefinitionContext context) {
NewIndex index = context.create("fakes");
- NewIndex.NewMapping mapping = index.createMapping("fake");
+ NewIndex.NewIndexType mapping = index.createType("fake");
mapping.stringFieldBuilder("key").build();
mapping.createDateTimeField("updatedAt");
mapping.createIntegerField("newField");
@Test
public void of() throws Exception {
- NewIndex indexV1 = createIndex();
+ IndexRegistry.Index indexV1 = new IndexRegistry.Index(createIndex());
String hashV1 = new IndexHash().of(indexV1);
assertThat(hashV1).isNotEmpty();
// always the same
assertThat(hashV1).isEqualTo(new IndexHash().of(indexV1));
- NewIndex indexV2 = createIndex();
- indexV2.getMappings().get("fake").createIntegerField("max");
- String hashV2 = new IndexHash().of(indexV2);
+ NewIndex newIndexV2 = createIndex();
+ newIndexV2.getTypes().get("fake").createIntegerField("max");
+ String hashV2 = new IndexHash().of(new IndexRegistry.Index(newIndexV2));
assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1);
}
private NewIndex createIndex() {
- NewIndex index = new NewIndex("fakes");
- NewIndex.NewMapping mapping = index.createMapping("fake");
+ NewIndex newIndex = new NewIndex("fakes");
+ NewIndex.NewIndexType mapping = newIndex.createType("fake");
mapping.setAttribute("list_attr", Arrays.asList("foo", "bar"));
mapping.stringFieldBuilder("key").build();
mapping.createDateTimeField("updatedAt");
- return index;
+ return newIndex;
}
}
assertThat(context.getIndices()).hasSize(1);
NewIndex issuesIndex = context.getIndices().get("issues");
assertThat(issuesIndex).isNotNull();
- assertThat(issuesIndex.getMappings().keySet()).containsOnly("issue", "issueAuthorization");
+ assertThat(issuesIndex.getTypes().keySet()).containsOnly("issue", "issueAuthorization");
// no cluster by default
assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("1");
public void most_basic_index() throws Exception {
NewIndex index = new NewIndex("issues");
assertThat(index.getName()).isEqualTo("issues");
- assertThat(index.getMappings()).isEmpty();
+ assertThat(index.getTypes()).isEmpty();
Settings settings = index.getSettings().build();
// test some basic settings
assertThat(settings.get("index.number_of_shards")).isNotEmpty();
@Test
public void define_fields() throws Exception {
NewIndex index = new NewIndex("issues");
- NewIndex.NewMapping mapping = index.createMapping("issue");
+ NewIndex.NewIndexType mapping = index.createType("issue");
mapping.setAttribute("dynamic", "true");
mapping.setProperty("foo_field", ImmutableMap.of("type", "string"));
mapping.createBooleanField("boolean_field");
mapping.createShortField("short_field");
mapping.createUuidPathField("uuid_path_field");
- mapping = index.getMappings().get("issue");
+ mapping = index.getTypes().get("issue");
assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true");
assertThat(mapping).isNotNull();
assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class);
@Test
public void define_string_field() throws Exception {
NewIndex index = new NewIndex("issues");
- NewIndex.NewMapping mapping = index.createMapping("issue");
+ NewIndex.NewIndexType mapping = index.createType("issue");
mapping.stringFieldBuilder("basic_field").build();
mapping.stringFieldBuilder("all_capabilities_field")
.enableGramSearch()
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.common.settings.ImmutableSettings;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
searchClient = new SearchClient(settings);
}
+ @After
+ public void tearDown() throws Exception {
+ if (searchClient != null) {
+ searchClient.stop();
+ }
+ }
+
@Test
public void can_load() {
BaseIndex index = getIndex(this.searchClient);
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void bulk() {
BulkRequestBuilder bulkRequestBuilder = searchClient.prepareBulk();
}
// TODO assert profiling
+ searchClient.stop();
}
@Test(expected = UnsupportedOperationException.class)
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void state() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void state() {
try {
@Test
public void with_profiling_basic() {
+ Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name()));
+ SearchClient searchClient = new SearchClient(new Settings(), profiling);
try {
- Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name()));
- SearchClient searchClient = new SearchClient(new Settings(), profiling);
ClusterStateRequestBuilder requestBuilder = searchClient.prepareState();
requestBuilder.get();
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void stats() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void count() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void create_index() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void bulk() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void flush() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void get() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void exists() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void stats() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.fetch.source.FetchSourceContext;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void multi_get() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void stats() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void put_mapping() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void refresh() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
package org.sonar.server.search.request;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void search() {
try {
}
// TODO assert profiling
+ searchClient.stop();
}
@Test
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.unit.TimeValue;
+import org.junit.After;
import org.junit.Test;
import org.sonar.api.config.Settings;
import org.sonar.core.profiling.Profiling;
Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name()));
SearchClient searchClient = new SearchClient(new Settings(), profiling);
+ @After
+ public void tearDown() throws Exception {
+ searchClient.stop();
+ }
+
@Test
public void search_scroll() {
try {
@Test
public void with_profiling_basic() {
+ Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name()));
+ SearchClient searchClient = new SearchClient(new Settings(), profiling);
try {
- Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name()));
- SearchClient searchClient = new SearchClient(new Settings(), profiling);
SearchResponse search = searchClient.prepareSearch(IndexDefinition.RULE.getIndexName())
.setSearchType(SearchType.SCAN)
.setScroll(TimeValue.timeValueSeconds(3L))
assertThat(e).isInstanceOf(IllegalStateException.class);
assertThat(e.getMessage()).contains("Fail to execute ES search request '{}' on indices '[rules]'");
}
+ searchClient.stop();
}
@Test
}
}
- searchServer.start();
- platform.init(properties);
- platform.addComponents(components);
- platform.doStart();
+ try {
+ searchServer.start();
+ platform.init(properties);
+ platform.addComponents(components);
+ platform.doStart();
+ } catch (RuntimeException e) {
+ stop();
+ throw e;
+ }
if (!platform.isStarted()) {
throw new IllegalStateException("Server not started. You should check that db migrations " +
"are correctly declared, for example in schema-h2.sql or DatabaseVersion");
private File createTempDir() {
try {
// Technique to create a temp directory from a temp file
- File f = File.createTempFile("SonarQube", "");
+ File f = File.createTempFile("tmp-sq", "");
f.delete();
f.mkdir();
return f;
* This method should not be called by test when ServerTester is annotated with {@link org.junit.Rule}
*/
public void stop() {
- platform.doStop();
- searchServer.stop();
+ try {
+ if (platform != null) {
+ platform.doStop();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ try {
+ if (searchServer != null) {
+ searchServer.stop();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
FileUtils.deleteQuietly(homeDir);
}
--- /dev/null
+<dataset>
+ <fake id="10" kee="AB" />
+ <fake id="20" kee="CD" />
+ <fake id="30" kee="EF" />
+</dataset>
--- /dev/null
+CREATE TABLE "FAKE" (
+ "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1),
+ "KEE" VARCHAR(200) NOT NULL
+);
projects / sonarqube.git / commitdiff