From aa91479cf9de3f172af489e0cf3ff885b77aeb42 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Wed, 19 Nov 2014 08:26:55 +0100 Subject: [PATCH] Synchronize database and ES with a journal approach --- .../java/org/sonar/search/SearchServer.java | 18 -- .../org/sonar/search/SearchServerTest.java | 44 ----- .../java/org/sonar/server/db/DbClient.java | 17 ++ .../sonar/server/db/ResultSetIterator.java | 98 +++++++++++ .../server/db/migrations/SelectImpl.java | 1 + .../sonar/server/db/migrations/SqlUtil.java | 31 ++++ .../server/es/BulkIndexRequestIterator.java | 75 --------- .../java/org/sonar/server/es/BulkIndexer.java | 158 +++++++++++------- ...ettings.java => DefaultIndexSettings.java} | 4 +- .../java/org/sonar/server/es/EsClient.java | 19 ++- .../org/sonar/server/es/IndexCreator.java | 60 +++---- .../org/sonar/server/es/IndexDefinition.java | 19 ++- .../java/org/sonar/server/es/IndexHash.java | 14 +- .../org/sonar/server/es/IndexRegistry.java | 113 +++++++++++++ .../sonar/server/es/IssueIndexDefinition.java | 87 ++++++---- .../org/sonar/server/es/IssueIndexer.java | 113 +++++++++++++ .../server/es/IssueResultSetIterator.java | 134 +++++++++++++++ .../java/org/sonar/server/es/NewIndex.java | 52 +++--- .../sonar/server/issue/index/IssueDoc.java | 106 +++++++++++- .../server/platform/ServerComponents.java | 4 + .../java/org/sonar/server/search/BaseDoc.java | 9 + .../sonar/server/db/EmbeddedDatabaseTest.java | 22 +-- .../server/db/ResultSetIteratorTest.java | 155 +++++++++++++++++ .../server/db/migrations/SqlUtilTest.java | 1 - .../es/BulkIndexRequestIteratorTest.java | 109 ------------ .../org/sonar/server/es/BulkIndexerTest.java | 101 +++++++++++ ...est.java => DefaultIndexSettingsTest.java} | 6 +- .../java/org/sonar/server/es/EsTester.java | 91 ++++++---- .../sonar/server/es/FakeIndexDefinition.java | 44 +++++ .../org/sonar/server/es/IndexCreatorTest.java | 27 +-- .../org/sonar/server/es/IndexHashTest.java | 14 +- .../server/es/IssueIndexDefinitionTest.java | 2 +- .../org/sonar/server/es/NewIndexTest.java | 8 +- .../sonar/server/search/BaseIndexTest.java | 8 + .../request/ProxyBulkRequestBuilderTest.java | 7 + .../ProxyClusterHealthRequestBuilderTest.java | 7 + .../ProxyClusterStateRequestBuilderText.java | 11 +- .../ProxyClusterStatsRequestBuilderText.java | 7 + .../request/ProxyCountRequestBuilderTest.java | 7 + .../ProxyCreateIndexRequestBuilderText.java | 7 + .../ProxyDeleteByQueryRequestBuilderTest.java | 7 + .../request/ProxyFlushRequestBuilderTest.java | 7 + .../request/ProxyGetRequestBuilderTest.java | 7 + .../ProxyIndicesExistsRequestBuilderTest.java | 7 + .../ProxyIndicesStatsRequestBuilderTest.java | 7 + .../ProxyMultiGetRequestBuilderTest.java | 7 + .../ProxyNodesStatsRequestBuilderTest.java | 7 + .../ProxyPutMappingRequestBuilderTest.java | 7 + .../ProxyRefreshRequestBuilderTest.java | 7 + .../ProxySearchRequestBuilderTest.java | 7 + .../ProxySearchScrollRequestBuilderTest.java | 11 +- .../org/sonar/server/tester/ServerTester.java | 31 +++- .../server/db/ResultSetIteratorTest/feed.xml | 5 + .../db/ResultSetIteratorTest/schema.sql | 4 + 54 files changed, 1426 insertions(+), 505 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/db/ResultSetIterator.java delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexRequestIterator.java rename server/sonar-server/src/main/java/org/sonar/server/es/{DefaultMappingSettings.java => DefaultIndexSettings.java} (98%) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/db/ResultSetIteratorTest.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexRequestIteratorTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java rename server/sonar-server/src/test/java/org/sonar/server/es/{DefaultMappingSettingsTest.java => DefaultIndexSettingsTest.java} (87%) create mode 100644 server/sonar-server/src/test/java/org/sonar/server/es/FakeIndexDefinition.java create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/feed.xml create mode 100644 server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/schema.sql diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java index c1f8a1e329f..18909b942ae 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java +++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java @@ -20,13 +20,9 @@ package org.sonar.search; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.hppc.cursors.ObjectCursor; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; import org.slf4j.LoggerFactory; -import org.sonar.process.MessageException; import org.sonar.process.MinimumViableSystem; import org.sonar.process.Monitored; import org.sonar.process.ProcessEntryPoint; @@ -49,20 +45,6 @@ public class SearchServer implements Monitored { node = new InternalNode(settings.build(), true); node.start(); - - // When joining a cluster, make sur the master(s) have a - // replication factor on all indices > 0 - if (settings.inCluster() && !settings.isMaster()) { - for (ObjectCursor settingCursor : node.client().admin().indices() - .prepareGetSettings().get().getIndexToSettings().values()) { - Settings settings = settingCursor.value; - String clusterReplicationFactor = settings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "-1"); - if (Integer.parseInt(clusterReplicationFactor) <= 0) { - node.stop(); - throw new MessageException("Invalid number of Elasticsearch replicas: " + clusterReplicationFactor); - } - } - } } @Override diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java index 62953cb7726..122ff0431d0 100644 --- a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java +++ b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java @@ -38,7 +38,6 @@ import org.sonar.process.Props; import java.util.Properties; import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; public class SearchServerTest { @@ -130,49 +129,6 @@ public class SearchServerTest { slaveServer.awaitStop(); } - @Test - public void slave_failed_replication() throws Exception { - Props props = new Props(new Properties()); - props.set(ProcessConstants.SEARCH_PORT, String.valueOf(port)); - props.set(ProcessConstants.CLUSTER_ACTIVATE, "false"); - props.set(ProcessConstants.CLUSTER_NAME, CLUSTER_NAME); - props.set(ProcessConstants.CLUSTER_NODE_NAME, "NOT_MASTER"); - props.set(ProcessConstants.PATH_HOME, temp.newFolder().getAbsolutePath()); - searchServer = new SearchServer(props); - assertThat(searchServer).isNotNull(); - - searchServer.start(); - assertThat(searchServer.isReady()).isTrue(); - - client = getSearchClient(); - client.admin().indices().prepareCreate("test").get(); - - // start a slave - props = new Props(new Properties()); - props.set(ProcessConstants.CLUSTER_ACTIVATE, "true"); - props.set(ProcessConstants.CLUSTER_MASTER, "false"); - props.set(ProcessConstants.CLUSTER_MASTER_HOST, "localhost:" + port); - props.set(ProcessConstants.CLUSTER_NAME, CLUSTER_NAME); - props.set(ProcessConstants.CLUSTER_NODE_NAME, "SLAVE"); - props.set(ProcessConstants.SEARCH_PORT, String.valueOf(NetworkUtils.freePort())); - props.set(ProcessConstants.PATH_HOME, temp.newFolder().getAbsolutePath()); - SearchServer slaveServer = new SearchServer(props); - assertThat(slaveServer).isNotNull(); - - try { - slaveServer.start(); - fail(); - } catch (Exception e) { - assertThat(e).hasMessage("Invalid number of Elasticsearch replicas: 0"); - } - - assertThat(client.admin().cluster().prepareClusterStats().get() - .getNodesStats().getCounts().getTotal()).isEqualTo(1); - - slaveServer.stop(); - slaveServer.awaitStop(); - } - private Client getSearchClient() { Settings settings = ImmutableSettings.settingsBuilder() .put("cluster.name", CLUSTER_NAME).build(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java index 256d76ab6ff..bfb143566b9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/DbClient.java @@ -47,6 +47,10 @@ import org.sonar.server.qualityprofile.db.ActiveRuleDao; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.user.db.GroupDao; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.IdentityHashMap; import java.util.Map; @@ -211,4 +215,17 @@ public class DbClient implements ServerComponent { private K getDao(Map map, Class clazz) { return (K) map.get(clazz); } + + /** + * Create a PreparedStatement for SELECT requests with scrolling of results + */ + public final PreparedStatement newScrollingSelectStatement(Connection connection, String sql) { + try { + PreparedStatement stmt = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + stmt.setFetchSize(database().getDialect().getScrollDefaultFetchSize()); + return stmt; + } catch (SQLException e) { + throw new IllegalStateException("Fail to create SQL statement: " + sql, e); + } + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/ResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/db/ResultSetIterator.java new file mode 100644 index 00000000000..a8f882e4f30 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/db/ResultSetIterator.java @@ -0,0 +1,98 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.db; + +import org.apache.commons.dbutils.DbUtils; + +import javax.annotation.CheckForNull; + +import java.io.Closeable; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Iterator; + +/** + * {@link java.util.Iterator} applied to {@link java.sql.ResultSet} + */ +public abstract class ResultSetIterator implements Iterator, Closeable { + + private final ResultSet rs; + private final PreparedStatement stmt; + + // TODO can be simpler by using rs.isLast(). See ResultSetIterator from commons-dbutils + private boolean didNext = false; + private boolean hasNext = false; + + public ResultSetIterator(PreparedStatement stmt) throws SQLException { + this.stmt = stmt; + this.rs = stmt.executeQuery(); + } + + protected ResultSetIterator(ResultSet rs) { + this.stmt = null; + this.rs = rs; + } + + @Override + public boolean hasNext() { + if (!didNext) { + hasNext = doNextQuietly(); + didNext = true; + } + return hasNext; + } + + @Override + @CheckForNull + public E next() { + if (!didNext) { + doNextQuietly(); + } + didNext = false; + try { + return read(rs); + } catch (SQLException e) { + // TODO add SQL request to context + throw new IllegalStateException("Fail to read result set row", e); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + DbUtils.closeQuietly(rs); + DbUtils.closeQuietly(stmt); + } + + protected abstract E read(ResultSet rs) throws SQLException; + + private boolean doNextQuietly() { + try { + return rs.next(); + } catch (SQLException e) { + throw new IllegalStateException("Fail to read row of JDBC result set", e); + } + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/SelectImpl.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/SelectImpl.java index 11243866258..81a8ac9103e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/SelectImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/SelectImpl.java @@ -81,6 +81,7 @@ class SelectImpl extends BaseSqlStatement implements Iterator { - - public static interface InputConverter { - List convert(INPUT input); - } - - private final Iterator input; - private final InputConverter converter; - private Iterator currents = null; - - public BulkIndexRequestIterator(Iterable input, InputConverter converter) { - this.input = input.iterator(); - this.converter = converter; - if (this.input.hasNext()) { - this.currents = converter.convert(this.input.next()).iterator(); - } - } - - @Override - public boolean hasNext() { - return currents != null && currents.hasNext(); - } - - @Override - public ActionRequest next() { - if (currents == null) { - throw new NoSuchElementException(); - } - ActionRequest request = currents.next(); - peekNext(); - return request; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - private void peekNext() { - if (!currents.hasNext()) { - if (input.hasNext()) { - currents = converter.convert(input.next()).iterator(); - } else { - currents = null; - } - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 895e7d74b3c..07eb4bb8a82 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -19,103 +19,135 @@ */ package org.sonar.server.es; -import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang.StringUtils; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.sonar.server.search.SearchClient; +import org.picocontainer.Startable; -import java.util.Iterator; +import java.util.Map; /** - * + * Helper to bulk requests in an efficient way : + *
    + *
  • bulk request is sent on the wire when its size is higher than 5Mb
    • + *
  • on large table indexing, replicas and automatic refresh can be temporarily disabled
    • + *
  • index refresh is optional (enabled by default)
    • + *
*/ -public class BulkIndexer { +public class BulkIndexer implements Startable { + + public static final long FLUSH_BYTE_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytes(); - private static final long FLUSH_BYTE_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytes(); - - private final SearchClient client; + private final EsClient client; + private final String indexName; + private boolean large = false; + private boolean refresh = true; + private long flushByteSize = FLUSH_BYTE_SIZE; + private BulkRequestBuilder bulkRequest = null; + private Map largeInitialSettings = null; - public BulkIndexer(SearchClient client) { + public BulkIndexer(EsClient client, String indexName) { this.client = client; + this.indexName = indexName; } /** - * Heavy operation that populates an index from scratch. Replicas are disabled during - * the bulk indexation and lucene segments are optimized at the end. No need - * to call {@link #refresh(String)} after this method. - * - * @see BulkIndexRequestIterator + * Large indexing is an heavy operation that populates an index generally from scratch. Replicas and + * automatic refresh are disabled during bulk indexing and lucene segments are optimized at the end. */ - public void fullIndex(String index, Iterator requests) { - // deactivate replicas - GetSettingsRequestBuilder replicaRequest = client.admin().indices().prepareGetSettings(index); - String initialRequestSetting = replicaRequest.get().getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS); - int initialReplicas = Integer.parseInt(StringUtils.defaultIfEmpty(initialRequestSetting, "0")); - if (initialReplicas > 0) { - setNumberOfReplicas(index, 0); - } - index(requests); - refresh(index); - optimize(index); - - if (initialReplicas > 0) { - // re-enable replicas - setNumberOfReplicas(index, initialReplicas); - } + public BulkIndexer setLarge(boolean b) { + Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started"); + this.large = b; + return this; } - private void setNumberOfReplicas(String index, int replicas) { - UpdateSettingsRequestBuilder req = client.admin().indices().prepareUpdateSettings(index); - req.setSettings(ImmutableMap.of(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, String.valueOf(replicas))); - req.get(); + public BulkIndexer setRefresh(boolean b) { + Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started"); + this.refresh = b; + return this; } /** - * @see BulkIndexRequestIterator + * Default value is {@link org.sonar.server.es.BulkIndexer#FLUSH_BYTE_SIZE} + * @see org.elasticsearch.common.unit.ByteSizeValue */ - public void index(Iterator requests) { - BulkRequest bulkRequest = client.prepareBulk().request(); - while (requests.hasNext()) { - ActionRequest request = requests.next(); - bulkRequest.add(request); - if (bulkRequest.estimatedSizeInBytes() >= FLUSH_BYTE_SIZE) { - executeBulk(bulkRequest); - bulkRequest = client.prepareBulk().request(); + public BulkIndexer setFlushByteSize(long l) { + this.flushByteSize = l; + return this; + } + + @Override + public void start() { + Preconditions.checkState(bulkRequest == null, "Bulk indexing is already started"); + if (large) { + largeInitialSettings = Maps.newHashMap(); + Map bulkSettings = Maps.newHashMap(); + GetSettingsResponse settingsResp = client.nativeClient().admin().indices().prepareGetSettings(indexName).get(); + + // deactivate replicas + int initialReplicas = Integer.parseInt(settingsResp.getSetting(indexName, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + if (initialReplicas > 0) { + largeInitialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); + bulkSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); } + + // deactivate periodical refresh + String refreshInterval = settingsResp.getSetting(indexName, "index.refresh_interval"); + largeInitialSettings.put("index.refresh_interval", refreshInterval); + bulkSettings.put("index.refresh_interval", "-1"); + + updateSettings(bulkSettings); } - if (bulkRequest.numberOfActions() > 0) { + bulkRequest = client.prepareBulk(); + } + + public void add(ActionRequest request) { + bulkRequest.request().add(request); + if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) { executeBulk(bulkRequest); } } - private void executeBulk(BulkRequest bulkRequest) { - try { - BulkResponse response = client.bulk(bulkRequest).get(); + @Override + public void stop() { + if (bulkRequest.numberOfActions() > 0) { + executeBulk(bulkRequest); + } + if (refresh) { + client.prepareRefresh(indexName).get(); + } + if (large) { + // optimize lucene segments and revert index settings + // Optimization must be done before re-applying replicas: + // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ + // TODO do not use nativeClient, else request is not profiled + client.nativeClient().admin().indices().prepareOptimize(indexName) + .setMaxNumSegments(1) + .setWaitForMerge(true) + .get(); - // TODO check failures - // WARNING - complexity of response#hasFailures() and #buildFailureMessages() is O(n) - } catch (Exception e) { - throw new IllegalStateException("TODO", e); + updateSettings(largeInitialSettings); } + bulkRequest = null; } - public void refresh(String index) { - client.prepareRefresh(index).get(); + private void updateSettings(Map settings) { + UpdateSettingsRequestBuilder req = client.nativeClient().admin().indices().prepareUpdateSettings(indexName); + req.setSettings(settings); + req.get(); } - private void optimize(String index) { - client.admin().indices().prepareOptimize(index) - .setMaxNumSegments(1) - .setWaitForMerge(true) - .get(); - } + private void executeBulk(BulkRequestBuilder bulkRequest) { + bulkRequest.get(); + // TODO check failures + // WARNING - complexity of response#hasFailures() and #buildFailureMessages() is O(n) + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultMappingSettings.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java similarity index 98% rename from server/sonar-server/src/main/java/org/sonar/server/es/DefaultMappingSettings.java rename to server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java index be5c6708d5c..328ec30db51 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultMappingSettings.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java @@ -22,9 +22,9 @@ package org.sonar.server.es; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.ImmutableSettings; -class DefaultMappingSettings { +class DefaultIndexSettings { - private DefaultMappingSettings() { + private DefaultIndexSettings() { // only static stuff } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 3227328f0a3..35cb739359a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -39,6 +39,9 @@ import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.metrics.max.Max; import org.picocontainer.Startable; import org.sonar.core.profiling.Profiling; import org.sonar.server.search.ClusterHealth; @@ -75,7 +78,7 @@ public class EsClient implements Startable { this.client = deprecatedClient; } - public EsClient(Profiling profiling, Client client) { + EsClient(Profiling profiling, Client client) { this.profiling = profiling; this.client = client; } @@ -166,6 +169,17 @@ public class EsClient implements Startable { return new ProxyDeleteByQueryRequestBuilder(client, profiling).setIndices(indices); } + public long getLastUpdatedAt(String indexName, String typeName) { + SearchRequestBuilder request = prepareSearch(indexName) + .setTypes(typeName) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0) + .addAggregation(AggregationBuilders.max("latest").field("updatedAt")); + + Max max = request.get().getAggregations().get("latest"); + return (long) max.getValue(); + } + @Override public void start() { // nothing to do @@ -173,7 +187,8 @@ public class EsClient implements Startable { @Override public void stop() { - client.close(); + // TODO re-enable when SearchClient is dropped + //client.close(); } protected Client nativeClient() { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java index 01008a4cffd..d088c7a9718 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java @@ -24,13 +24,14 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.picocontainer.Startable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.api.ServerComponent; import java.util.Map; /** * Create registered indices in Elasticsearch. */ -public class IndexCreator implements Startable { +public class IndexCreator implements ServerComponent, Startable { private static final Logger LOGGER = LoggerFactory.getLogger(IndexCreator.class); @@ -41,59 +42,50 @@ public class IndexCreator implements Startable { private static final String SETTING_HASH = "sonar_hash"; private final EsClient client; - private final IndexDefinition[] definitions; + private final IndexRegistry registry; - public IndexCreator(EsClient client, IndexDefinition[] definitions) { + public IndexCreator(EsClient client, IndexRegistry registry) { this.client = client; - this.definitions = definitions; + this.registry = registry; } @Override public void start() { - create(); - } - - @Override - public void stop() { - // nothing to do - } - - public void create() { - // collect definitions - IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - for (IndexDefinition definition : definitions) { - definition.define(context); - } - // create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...) - for (NewIndex newIndex : context.getIndices().values()) { - boolean exists = client.prepareExists(newIndex.getName()).get().isExists(); + for (IndexRegistry.Index index : registry.getIndices().values()) { + boolean exists = client.prepareExists(index.getName()).get().isExists(); if (exists) { - if (needsToDeleteIndex(newIndex)) { - LOGGER.info(String.format("Delete index %s (settings changed)", newIndex.getName())); - deleteIndex(newIndex.getName()); + if (needsToDeleteIndex(index)) { + LOGGER.info(String.format("Delete index %s (settings changed)", index.getName())); + deleteIndex(index.getName()); exists = false; } } if (!exists) { - createIndex(newIndex); + createIndex(index); } } } - private void createIndex(NewIndex newIndex) { - LOGGER.info(String.format("Create index %s", newIndex.getName())); - ImmutableSettings.Builder settings = newIndex.getSettings(); - settings.put(SETTING_HASH, new IndexHash().of(newIndex)); + @Override + public void stop() { + // nothing to do + } + + private void createIndex(IndexRegistry.Index index) { + LOGGER.info(String.format("Create index %s", index.getName())); + ImmutableSettings.Builder settings = ImmutableSettings.builder(); + settings.put(index.getSettings()); + settings.put(SETTING_HASH, new IndexHash().of(index)); client - .prepareCreate(newIndex.getName()) + .prepareCreate(index.getName()) .setSettings(settings) .get(); // create types - for (Map.Entry entry : newIndex.getMappings().entrySet()) { - LOGGER.info(String.format("Create type %s/%s", newIndex.getName(), entry.getKey())); - client.preparePutMapping(newIndex.getName()) + for (Map.Entry entry : index.getTypes().entrySet()) { + LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey())); + client.preparePutMapping(index.getName()) .setType(entry.getKey()) .setIgnoreConflicts(false) .setSource(entry.getValue().getAttributes()) @@ -105,7 +97,7 @@ public class IndexCreator implements Startable { client.nativeClient().admin().indices().prepareDelete(indexName).get(); } - private boolean needsToDeleteIndex(NewIndex index) { + private boolean needsToDeleteIndex(IndexRegistry.Index index) { boolean toBeDeleted = false; String hash = client.nativeClient().admin().indices().prepareGetSettings(index.getName()).get().getSetting(index.getName(), "index." + SETTING_HASH); if (hash != null) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinition.java index 93b03587ed2..edf29d61b10 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinition.java @@ -19,20 +19,21 @@ */ package org.sonar.server.es; -import java.util.HashMap; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.sonar.api.ServerComponent; + import java.util.Map; -public interface IndexDefinition { +public interface IndexDefinition extends ServerComponent { public static class IndexDefinitionContext { - private final Map byKey = new HashMap(); + private final Map byKey = Maps.newHashMap(); public NewIndex create(String key) { - NewIndex index = byKey.get(key); - if (index == null) { - index = new NewIndex(key); - byKey.put(key, index); - } + Preconditions.checkArgument(!byKey.containsKey(key), String.format("Index already exists: %s", key)); + NewIndex index = new NewIndex(key); + byKey.put(key, index); return index; } @@ -41,5 +42,7 @@ public interface IndexDefinition { } } + void define(IndexDefinitionContext context); + } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexHash.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexHash.java index 42be03b148b..259e62431eb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexHash.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexHash.java @@ -23,8 +23,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Lists; import org.apache.commons.codec.digest.DigestUtils; -import javax.annotation.Nullable; - import java.util.Collections; import java.util.List; import java.util.Map; @@ -34,8 +32,8 @@ class IndexHash { private static final char DELIMITER = ','; - String of(NewIndex index) { - return of(index.getSettings().internalMap(), index.getMappings()); + String of(IndexRegistry.Index index) { + return of(index.getSettings().getAsMap(), index.getTypes()); } String of(Map... maps) { @@ -47,8 +45,8 @@ class IndexHash { } private void appendObject(StringBuilder sb, Object value) { - if (value instanceof NewIndex.NewMapping) { - appendMapping(sb, (NewIndex.NewMapping) value); + if (value instanceof IndexRegistry.IndexType) { + appendIndexType(sb, (IndexRegistry.IndexType) value); } else if (value instanceof Map) { appendMap(sb, (Map) value); } else if (value instanceof Iterable) { @@ -58,8 +56,8 @@ class IndexHash { } } - private void appendMapping(StringBuilder sb, NewIndex.NewMapping mapping) { - appendMap(sb, mapping.getAttributes()); + private void appendIndexType(StringBuilder sb, IndexRegistry.IndexType type) { + appendMap(sb, type.getAttributes()); } private void appendMap(StringBuilder sb, Map attributes) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java new file mode 100644 index 00000000000..867cfc301f7 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java @@ -0,0 +1,113 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.elasticsearch.common.settings.Settings; +import org.picocontainer.Startable; +import org.sonar.api.ServerComponent; + +import java.util.Map; + +public class IndexRegistry implements ServerComponent, Startable { + + /** + * Immutable copy of {@link org.sonar.server.es.NewIndex} + */ + public static class Index { + private final String name; + private final Settings settings; + private final Map types; + + Index(NewIndex newIndex) { + this.name = newIndex.getName(); + this.settings = newIndex.getSettings().build(); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) { + IndexType type = new IndexType(newIndexType); + builder.put(type.getName(), type); + } + this.types = builder.build(); + } + + public String getName() { + return name; + } + + public Settings getSettings() { + return settings; + } + + public Map getTypes() { + return types; + } + } + + /** + * Immutable copy of {@link org.sonar.server.es.NewIndex.NewIndexType} + */ + public static class IndexType { + private final String name; + private final Map attributes; + + private IndexType(NewIndex.NewIndexType newType) { + this.name = newType.getName(); + this.attributes = ImmutableMap.copyOf(newType.getAttributes()); + } + + public String getName() { + return name; + } + + public Map getAttributes() { + return attributes; + } + } + + private final Map byKey = Maps.newHashMap(); + private final IndexDefinition[] defs; + + public IndexRegistry(IndexDefinition[] defs) { + this.defs = defs; + } + + public Map getIndices() { + return byKey; + } + + @Override + public void start() { + // collect definitions + IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); + for (IndexDefinition definition : defs) { + definition.define(context); + } + + for (Map.Entry entry : context.getIndices().entrySet()) { + byKey.put(entry.getKey(), new Index(entry.getValue())); + } + } + + @Override + public void stop() { + // nothing to do + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java index 63f16081a9c..894bb11c7bf 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java @@ -33,9 +33,37 @@ import org.sonar.server.search.BaseNormalizer; public class IssueIndexDefinition implements IndexDefinition { public static final String INDEX_ISSUES = "issues"; + public static final String TYPE_ISSUE_AUTHORIZATION = "issueAuthorization"; public static final String TYPE_ISSUE = "issue"; + public static final String FIELD_ISSUE_ACTION_PLAN = "actionPlan"; + public static final String FIELD_ISSUE_ASSIGNEE = "assignee"; + public static final String FIELD_ISSUE_ATTRIBUTES = "attributes"; + public static final String FIELD_ISSUE_AUTHOR_LOGIN = "authorLogin"; + public static final String FIELD_ISSUE_COMPONENT_UUID = "component"; + public static final String FIELD_ISSUE_CREATED_AT = "createdAt"; + public static final String FIELD_ISSUE_DEBT = "debt"; + public static final String FIELD_ISSUE_EFFORT = "effort"; + public static final String FIELD_ISSUE_FILE_PATH = "filePath"; + public static final String FIELD_ISSUE_FUNC_CREATED_AT = "issueCreatedAt"; + public static final String FIELD_ISSUE_FUNC_UPDATED_AT = "issueUpdatedAt"; + public static final String FIELD_ISSUE_FUNC_CLOSED_AT = "issueClosedAt"; + public static final String FIELD_ISSUE_KEY = "key"; + public static final String FIELD_ISSUE_LANGUAGE = "language"; + public static final String FIELD_ISSUE_LINE = "line"; + public static final String FIELD_ISSUE_MESSAGE = "message"; + public static final String FIELD_ISSUE_MODULE_UUID = "module"; + public static final String FIELD_ISSUE_MODULE_PATH = "modulePath"; + public static final String FIELD_ISSUE_PROJECT_UUID = "project"; + public static final String FIELD_ISSUE_REPORTER = "reporter"; + public static final String FIELD_ISSUE_RESOLUTION = "resolution"; + public static final String FIELD_ISSUE_RULE_KEY = "ruleKey"; + public static final String FIELD_ISSUE_SEVERITY = "severity"; + public static final String FIELD_ISSUE_SEVERITY_VALUE = "severityValue"; + public static final String FIELD_ISSUE_STATUS = "status"; + public static final String FIELD_ISSUE_UPDATED_AT = "updatedAt"; + private final Settings settings; public IssueIndexDefinition(Settings settings) { @@ -55,7 +83,7 @@ public class IssueIndexDefinition implements IndexDefinition { } // type "issueAuthorization" - NewIndex.NewMapping authorizationMapping = index.createMapping(TYPE_ISSUE_AUTHORIZATION); + NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_ISSUE_AUTHORIZATION); authorizationMapping.setAttribute("_id", ImmutableMap.of("path", IssueAuthorizationNormalizer.IssueAuthorizationField.PROJECT.field())); authorizationMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD); authorizationMapping.stringFieldBuilder("project").build(); @@ -63,39 +91,42 @@ public class IssueIndexDefinition implements IndexDefinition { authorizationMapping.stringFieldBuilder("users").build(); // type "issue" - NewIndex.NewMapping issueMapping = index.createMapping(TYPE_ISSUE); + NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE); issueMapping.setAttribute("_id", ImmutableMap.of("path", IssueNormalizer.IssueField.KEY.field())); issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_ISSUE_AUTHORIZATION)); issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", IssueNormalizer.IssueField.PROJECT.field())); - issueMapping.stringFieldBuilder("component").build(); - issueMapping.stringFieldBuilder("actionPlan").build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_ACTION_PLAN).build(); // TODO do we really sort by assignee ? - issueMapping.stringFieldBuilder("assignee").enableSorting().build(); - issueMapping.stringFieldBuilder("attributes").build(); - issueMapping.stringFieldBuilder("authorLogin").build(); - issueMapping.createDateTimeField("createdAt"); - issueMapping.createDoubleField("debt"); - issueMapping.createDoubleField("effort"); - issueMapping.stringFieldBuilder("filePath").enableSorting().build(); - issueMapping.createDateTimeField("issueCreatedAt"); - issueMapping.createDateTimeField("issueUpdatedAt"); - issueMapping.createDateTimeField("issueClosedAt"); - issueMapping.stringFieldBuilder("key").enableSorting().build(); - issueMapping.stringFieldBuilder("language").build(); - issueMapping.createIntegerField("line"); - issueMapping.stringFieldBuilder("message").build(); - issueMapping.stringFieldBuilder("module").build(); - issueMapping.createUuidPathField("modulePath"); + issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).build(); + // TODO rename into componentUuid ? or restrict to fileUuid ? + issueMapping.stringFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).build(); + issueMapping.createDoubleField(FIELD_ISSUE_DEBT); + issueMapping.createDoubleField(FIELD_ISSUE_EFFORT); + issueMapping.stringFieldBuilder(FIELD_ISSUE_FILE_PATH).enableSorting().build(); + issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT); + issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT); + issueMapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT); + issueMapping.stringFieldBuilder(FIELD_ISSUE_KEY).enableSorting().build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_LANGUAGE).build(); + issueMapping.createIntegerField(FIELD_ISSUE_LINE); + issueMapping.stringFieldBuilder(FIELD_ISSUE_MESSAGE).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_MODULE_UUID).build(); + issueMapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH); // TODO do we need to sort by project ? - issueMapping.stringFieldBuilder("project").enableSorting().build(); - issueMapping.stringFieldBuilder("reporter").build(); - issueMapping.stringFieldBuilder("resolution").build(); - issueMapping.stringFieldBuilder("ruleKey").build(); - issueMapping.stringFieldBuilder("severity").build(); + // TODO rename into projectUuid ? + issueMapping.stringFieldBuilder(FIELD_ISSUE_PROJECT_UUID).enableSorting().build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_REPORTER).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_RESOLUTION).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_RULE_KEY).build(); + issueMapping.stringFieldBuilder(FIELD_ISSUE_SEVERITY).build(); // TODO do we need to sort by severity ? - issueMapping.createByteField("severityValue"); + issueMapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE); // TODO do we really sort by status ? If yes, then we should sort by "int value", but not by string key - issueMapping.stringFieldBuilder("status").enableSorting().build(); - issueMapping.createDateTimeField("updatedAt"); + issueMapping.stringFieldBuilder(FIELD_ISSUE_STATUS).enableSorting().build(); + // TODO is createdAt required ? + issueMapping.createDateTimeField(FIELD_ISSUE_CREATED_AT); + issueMapping.createDateTimeField(FIELD_ISSUE_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java new file mode 100644 index 00000000000..dcd32771f52 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java @@ -0,0 +1,113 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.elasticsearch.action.update.UpdateRequest; +import org.sonar.api.ServerComponent; +import org.sonar.core.persistence.DbSession; +import org.sonar.server.db.DbClient; +import org.sonar.server.issue.index.IssueDoc; + +import java.sql.Connection; +import java.util.Iterator; + +/** + * Not thread-safe + */ +public class IssueIndexer implements ServerComponent { + + private final DbClient dbClient; + private final EsClient esClient; + private long lastUpdatedAt = 0L; + + public IssueIndexer(DbClient dbClient, EsClient esClient) { + this.dbClient = dbClient; + this.esClient = esClient; + } + + public void indexProjectPermissions() { + final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES); + + DbSession dbSession = dbClient.openSession(false); + Connection dbConnection = dbSession.getConnection(); + try { + IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbConnection, getLastUpdatedAt()); + indexIssues(bulk, rowIt); + rowIt.close(); + + } finally { + dbSession.close(); + } + } + + public void indexIssues(boolean large) { + // TODO support timezones + final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES); + bulk.setLarge(large); + + DbSession dbSession = dbClient.openSession(false); + Connection dbConnection = dbSession.getConnection(); + try { + IssueResultSetIterator rowIt = IssueResultSetIterator.create(dbClient, dbConnection, getLastUpdatedAt()); + indexIssues(bulk, rowIt); + rowIt.close(); + + } finally { + dbSession.close(); + } + } + + public void indexIssues(BulkIndexer bulk, Iterator issues) { + bulk.start(); + while (issues.hasNext()) { + IssueDoc issue = issues.next(); + bulk.add(newUpsertRequest(issue)); + + // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance) + long dtoUpdatedAt = issue.updateDate().getTime(); + if (lastUpdatedAt < dtoUpdatedAt) { + lastUpdatedAt = dtoUpdatedAt; + } + } + bulk.stop(); + } + + private long getLastUpdatedAt() { + long result; + if (lastUpdatedAt <= 0L) { + // request ES to get the max(updatedAt) + result = esClient.getLastUpdatedAt(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE); + } else { + // use cache. Will not work with Tomcat cluster. + result = lastUpdatedAt; + } + return result; + } + + private UpdateRequest newUpsertRequest(IssueDoc issue) { + String projectUuid = issue.projectUuid(); + issue.setField("_parent", projectUuid); + return new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE, issue.key()) + .routing(projectUuid) + .parent(projectUuid) + .doc(issue.getFields()) + .upsert(issue.getFields()); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java new file mode 100644 index 00000000000..f71ff238615 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java @@ -0,0 +1,134 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.collect.Maps; +import org.apache.commons.lang.StringUtils; +import org.sonar.server.db.DbClient; +import org.sonar.server.db.ResultSetIterator; +import org.sonar.server.db.migrations.SqlUtil; +import org.sonar.server.issue.index.IssueDoc; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Date; + +/** + * Scroll over table ISSUES and directly read the maps required to + * post index requests + */ +class IssueResultSetIterator extends ResultSetIterator { + + private static final String[] FIELDS = { + // column 1 + "i.kee", + "root.uuid", + "i.updated_at", + "i.created_at", + "i.action_plan_key", + "i.assignee", + "i.effort_to_fix", + "i.issue_attributes", + "i.line", + "i.message", + + // column 11 + "i.resolution", + "i.severity", + "i.status", + "i.technical_debt", + "i.reporter", + "i.author_login", + "i.issue_close_date", + "i.issue_creation_date", + "i.issue_update_date", + "r.plugin_rule_key", + + // column 21 + "r.language", + "p.uuid", + "p.module_uuid", + "p.module_uuid_path", + "p.path" + }; + + private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from issues i " + + "inner join rules r on r.id=i.rule_id " + + "inner join projects p on p.id=i.component_id " + + "inner join projects root on root.id=i.root_component_id"; + + private static final String SQL_AFTER_DATE = SQL_ALL + " where i.updated_at>=?"; + + static IssueResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) { + try { + String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL; + PreparedStatement stmt = dbClient.newScrollingSelectStatement(connection, sql); + if (afterDate > 0L) { + stmt.setTimestamp(0, new Timestamp(afterDate)); + } + return new IssueResultSetIterator(stmt); + } catch (SQLException e) { + throw new IllegalStateException("Fail to prepare SQL request to select all issues", e); + } + } + + private IssueResultSetIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + @Override + protected IssueDoc read(ResultSet rs) throws SQLException { + IssueDoc doc = new IssueDoc(Maps.newHashMapWithExpectedSize(30)); + + String key = rs.getString(1); + String projectUuid = rs.getString(2); + + // all the keys must be present, even if value is null + doc.setKey(key); + doc.setProjectUuid(projectUuid); + doc.setUpdateDate(SqlUtil.getDate(rs, 3)); + doc.setCreationDate(new Date(rs.getTimestamp(4).getTime())); + doc.setActionPlanKey(rs.getString(5)); + doc.setAssignee(rs.getString(6)); + doc.setEffortToFix(SqlUtil.getDouble(rs, 7)); + doc.setAttributes(rs.getString(8)); + doc.setLine(SqlUtil.getInt(rs, 9)); + doc.setMessage(rs.getString(10)); + doc.setResolution(rs.getString(11)); + doc.setSeverity(rs.getString(12)); + doc.setStatus(rs.getString(13)); + doc.setDebt(SqlUtil.getLong(rs, 14)); + doc.setReporter(rs.getString(15)); + doc.setAuthorLogin(rs.getString(16)); + doc.setFuncCloseDate(SqlUtil.getDate(rs, 17)); + doc.setFuncCreationDate(SqlUtil.getDate(rs, 18)); + doc.setFuncUpdateDate(SqlUtil.getDate(rs, 19)); + doc.setRuleKey(rs.getString(20)); + doc.setLanguage(rs.getString(21)); + doc.setComponentUuid(rs.getString(22)); + doc.setModuleUuid(rs.getString(23)); + doc.setModuleUuidPath(rs.getString(24)); + doc.setFilePath(rs.getString(25)); + return doc; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index c236153fcc8..fb2f531099b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -34,11 +34,13 @@ import java.util.TreeMap; public class NewIndex { - public static class NewMapping { + public static class NewIndexType { + private final String name; private final Map attributes = new TreeMap(); private final Map properties = new TreeMap(); - private NewMapping() { + private NewIndexType(String typeName) { + this.name = typeName; // defaults attributes.put("dynamic", false); attributes.put("_all", ImmutableSortedMap.of("enabled", false)); @@ -46,10 +48,14 @@ public class NewIndex { attributes.put("properties", properties); } + public String getName() { + return name; + } + /** * Complete the root json hash of mapping type, for example to set the attribute "_id" */ - public NewMapping setAttribute(String key, Object value) { + public NewIndexType setAttribute(String key, Object value) { attributes.put(key, value); return this; } @@ -57,7 +63,7 @@ public class NewIndex { /** * Complete the json hash named "properties" in mapping type, usually to declare fields */ - public NewMapping setProperty(String key, Object value) { + public NewIndexType setProperty(String key, Object value) { properties.put(key, value); return this; } @@ -66,35 +72,35 @@ public class NewIndex { return new StringFieldBuilder(this, fieldName); } - public NewMapping createBooleanField(String fieldName) { + public NewIndexType createBooleanField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "boolean")); } - public NewMapping createByteField(String fieldName) { + public NewIndexType createByteField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "byte")); } - public NewMapping createDateTimeField(String fieldName) { + public NewIndexType createDateTimeField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time")); } - public NewMapping createDoubleField(String fieldName) { + public NewIndexType createDoubleField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "double")); } - public NewMapping createIntegerField(String fieldName) { + public NewIndexType createIntegerField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "integer")); } - public NewMapping createLongField(String fieldName) { + public NewIndexType createLongField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "long")); } - public NewMapping createShortField(String fieldName) { + public NewIndexType createShortField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "short")); } - public NewMapping createUuidPathField(String fieldName) { + public NewIndexType createUuidPathField(String fieldName) { return setProperty(fieldName, ImmutableSortedMap.of( "type", "string", "index", "analyzed", @@ -119,12 +125,12 @@ public class NewIndex { "type", "string", "index", "not_analyzed"); - private final NewMapping newMapping; + private final NewIndexType indexType; private final String fieldName; private boolean sortable = false, wordSearch = false, gramSearch = false; - private StringFieldBuilder(NewMapping newMapping, String fieldName) { - this.newMapping = newMapping; + private StringFieldBuilder(NewIndexType indexType, String fieldName) { + this.indexType = indexType; this.fieldName = fieldName; } @@ -178,13 +184,13 @@ public class NewIndex { hash.putAll(NOT_ANALYZED); } - newMapping.setProperty(fieldName, hash); + indexType.setProperty(fieldName, hash); } } private final String indexName; - private final ImmutableSettings.Builder settings = DefaultMappingSettings.defaults(); - private final SortedMap mappings = new TreeMap(); + private final ImmutableSettings.Builder settings = DefaultIndexSettings.defaults(); + private final SortedMap types = new TreeMap(); NewIndex(String indexName) { Preconditions.checkArgument(StringUtils.isAllLowerCase(indexName), "Index name must be lower-case: " + indexName); @@ -199,13 +205,13 @@ public class NewIndex { return settings; } - public NewMapping createMapping(String typeName) { - NewMapping type = new NewMapping(); - mappings.put(typeName, type); + public NewIndexType createType(String typeName) { + NewIndexType type = new NewIndexType(typeName); + types.put(typeName, type); return type; } - public SortedMap getMappings() { - return mappings; + public SortedMap getTypes() { + return types; } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java index 5ae2f503320..d4e690fef24 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -22,12 +22,15 @@ package org.sonar.server.issue.index; import org.sonar.api.issue.Issue; import org.sonar.api.issue.IssueComment; import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.Severity; import org.sonar.api.utils.Duration; import org.sonar.api.utils.KeyValueFormat; +import org.sonar.server.es.IssueIndexDefinition; import org.sonar.server.search.BaseDoc; import org.sonar.server.search.IndexUtils; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Date; @@ -42,7 +45,7 @@ public class IssueDoc extends BaseDoc implements Issue { @Override public String key() { - return getField(IssueNormalizer.IssueField.KEY.field()); + return getField(IssueIndexDefinition.FIELD_ISSUE_KEY); } @Override @@ -192,4 +195,105 @@ public class IssueDoc extends BaseDoc implements Issue { public String filePath() { return getNullableField(IssueNormalizer.IssueField.FILE_PATH.field()); } + + public void setKey(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_KEY, s); + } + + public void setComponentUuid(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, s); + } + + public void setModuleUuid(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, s); + } + + public void setProjectUuid(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, s); + } + + public void setRuleKey(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, s); + } + + public void setLanguage(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_LANGUAGE, s); + } + + public void setSeverity(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_SEVERITY, s); + setField(IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE, Severity.ALL.indexOf(s)); + } + + public void setMessage(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_MESSAGE, s); + } + + public void setLine(@Nullable Integer i) { + setField(IssueIndexDefinition.FIELD_ISSUE_LINE, i); + } + + public void setEffortToFix(@Nullable Double d) { + setField(IssueIndexDefinition.FIELD_ISSUE_EFFORT, d); + } + + public void setStatus(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_STATUS, s); + } + + public void setResolution(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION, s); + } + + public void setReporter(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_REPORTER, s); + } + + public void setAssignee(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, s); + } + + public void setCreationDate(Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_CREATED_AT, d); + } + + public void setUpdateDate(Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_UPDATED_AT, d); + } + + public void setFuncCreationDate(Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, d); + } + + public void setFuncUpdateDate(Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT, d); + } + + public void setFuncCloseDate(Date d) { + setField(IssueIndexDefinition.FIELD_ISSUE_FUNC_CLOSED_AT, d); + } + + public void setAttributes(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_ATTRIBUTES, s); + } + + public void setAuthorLogin(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, s); + } + + public void setActionPlanKey(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_ACTION_PLAN, s); + } + + public void setDebt(Long l) { + setField(IssueIndexDefinition.FIELD_ISSUE_DEBT, l); + } + + public void setFilePath(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_FILE_PATH, s); + } + + public void setModuleUuidPath(String s) { + setField(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, s); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index 2e01a36ca47..3bbd0c35dc7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -143,7 +143,9 @@ import org.sonar.server.duplication.ws.DuplicationsParser; import org.sonar.server.duplication.ws.DuplicationsWs; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexCreator; +import org.sonar.server.es.IndexRegistry; import org.sonar.server.es.IssueIndexDefinition; +import org.sonar.server.es.IssueIndexer; import org.sonar.server.issue.ActionService; import org.sonar.server.issue.AssignAction; import org.sonar.server.issue.CommentAction; @@ -479,6 +481,7 @@ class ServerComponents { pico.addSingleton(Periods.class); pico.addSingleton(ServerWs.class); pico.addSingleton(BackendCleanup.class); + pico.addSingleton(IndexRegistry.class); pico.addSingleton(IndexCreator.class); // batch @@ -628,6 +631,7 @@ class ServerComponents { // issues pico.addSingleton(IssueIndexDefinition.class); + pico.addSingleton(IssueIndexer.class); pico.addSingleton(ServerIssueStorage.class); pico.addSingleton(IssueUpdater.class); pico.addSingleton(FunctionExecutor.class); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/BaseDoc.java b/server/sonar-server/src/main/java/org/sonar/server/search/BaseDoc.java index 9b1b8cd6329..194a5eda5ea 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/BaseDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/BaseDoc.java @@ -20,6 +20,7 @@ package org.sonar.server.search; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import java.util.Map; /** @@ -58,4 +59,12 @@ public abstract class BaseDoc { } return value; } + + public void setField(String key, @Nullable Object value) { + fields.put(key, value); + } + + public Map getFields() { + return fields; + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/EmbeddedDatabaseTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/EmbeddedDatabaseTest.java index 548340194d5..055d9223acb 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/db/EmbeddedDatabaseTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/db/EmbeddedDatabaseTest.java @@ -40,7 +40,7 @@ public class EmbeddedDatabaseTest { @Rule public ExpectedException throwable = ExpectedException.none(); - @Test(timeout = 10000) + @Test(timeout = 5000) public void should_start_and_stop() throws IOException { int port = NetworkUtils.freePort(); @@ -58,28 +58,8 @@ public class EmbeddedDatabaseTest { database.stop(); } - @Test(timeout = 10000) - public void should_support_memory_database() throws IOException { - int port = NetworkUtils.freePort(); - - EmbeddedDatabase database = new EmbeddedDatabase(testSettings(port) - .setProperty(DatabaseProperties.PROP_URL, "jdbc:h2:tcp://localhost:" + port + "/mem:sonarIT;USER=sonar;PASSWORD=sonar")); - database.start(); - - try { - String driverUrl = String.format("jdbc:h2:tcp://localhost:%d/mem:sonarIT;USER=sonar;PASSWORD=sonar", port); - DriverManager.registerDriver(new Driver()); - DriverManager.getConnection(driverUrl).close(); - } catch (Exception ex) { - fail("Unable to connect after start"); - } - - database.stop(); - } - @Test public void should_return_embedded_data_directory() throws Exception { - Settings settings = testSettings(0); EmbeddedDatabase database = new EmbeddedDatabase(settings); diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/ResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/ResultSetIteratorTest.java new file mode 100644 index 00000000000..0e1ba81cc9e --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/db/ResultSetIteratorTest.java @@ -0,0 +1,155 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.db; + +import org.apache.commons.dbutils.DbUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.core.persistence.TestDatabase; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.fest.assertions.Assertions.assertThat; +import static org.fest.assertions.Fail.fail; + +public class ResultSetIteratorTest { + + @Rule + public TestDatabase dbTester = new TestDatabase().schema(ResultSetIteratorTest.class, "schema.sql"); + + Connection connection = null; + + @Before + public void setUp() throws Exception { + connection = dbTester.openConnection(); + } + + @After + public void tearDown() throws Exception { + DbUtils.closeQuietly(connection); + } + + @Test + public void iterate_through_statement() throws Exception { + dbTester.prepareDbUnit(getClass(), "feed.xml"); + + PreparedStatement stmt = connection.prepareStatement("select * from fake order by id"); + FirstIntColumnIterator iterator = new FirstIntColumnIterator(stmt); + + assertThat(iterator.hasNext()).isTrue(); + + // calling multiple times hasNext() is ok + assertThat(iterator.hasNext()).isTrue(); + + assertThat(iterator.next()).isEqualTo(10); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo(20); + + // call next() without calling hasNext() + assertThat(iterator.next()).isEqualTo(30); + assertThat(iterator.hasNext()).isFalse(); + + iterator.close(); + // statement is closed by ResultSetIterator + assertThat(stmt.isClosed()).isTrue(); + } + + @Test + public void iterate_through_rs() throws Exception { + dbTester.prepareDbUnit(getClass(), "feed.xml"); + + PreparedStatement stmt = connection.prepareStatement("select * from fake order by id"); + ResultSet rs = stmt.executeQuery(); + FirstIntColumnIterator iterator = new FirstIntColumnIterator(rs); + + assertThat(iterator.next()).isEqualTo(10); + assertThat(iterator.next()).isEqualTo(20); + assertThat(iterator.next()).isEqualTo(30); + + iterator.close(); + assertThat(rs.isClosed()).isTrue(); + stmt.close(); + } + + @Test + public void remove_row_is_not_supported() throws Exception { + PreparedStatement stmt = connection.prepareStatement("select * from fake order by id"); + FirstIntColumnIterator iterator = new FirstIntColumnIterator(stmt); + + try { + iterator.remove(); + fail(); + } catch (UnsupportedOperationException ok) { + } + + iterator.close(); + } + + @Test + public void fail_to_read_row() throws Exception { + dbTester.prepareDbUnit(getClass(), "feed.xml"); + + PreparedStatement stmt = connection.prepareStatement("select * from fake order by id"); + FailIterator iterator = new FailIterator(stmt); + + assertThat(iterator.hasNext()).isTrue(); + try { + iterator.next(); + fail(); + } catch (IllegalStateException e) { + assertThat(e.getCause()).isInstanceOf(SQLException.class); + } + iterator.close(); + } + + private static class FirstIntColumnIterator extends ResultSetIterator { + + public FirstIntColumnIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + public FirstIntColumnIterator(ResultSet rs) { + super(rs); + } + + @Override + protected Integer read(ResultSet rs) throws SQLException { + return rs.getInt(1); + } + } + + private static class FailIterator extends ResultSetIterator { + + public FailIterator(PreparedStatement stmt) throws SQLException { + super(stmt); + } + + @Override + protected Integer read(ResultSet rs) throws SQLException { + // column does not exist + return rs.getInt(1234); + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/SqlUtilTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/SqlUtilTest.java index 6594dc4ff25..b3bcc72ead8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/SqlUtilTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/SqlUtilTest.java @@ -21,7 +21,6 @@ package org.sonar.server.db.migrations; import org.junit.Test; import org.slf4j.Logger; -import org.sonar.server.db.migrations.SqlUtil; import java.sql.SQLException; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexRequestIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexRequestIteratorTest.java deleted file mode 100644 index 73c6de6b385..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexRequestIteratorTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.NoSuchElementException; - -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verifyZeroInteractions; - -public class BulkIndexRequestIteratorTest { - - @Test - public void iterate_over_requests() throws Exception { - List input = Arrays.asList("foo", "bar", "3requests", "baz"); - BulkIndexRequestIterator.InputConverter converter = new BulkIndexRequestIterator.InputConverter() { - @Override - public List convert(String input) { - if ("3requests".equals(input)) { - return Collections.nCopies(3, (ActionRequest) new IndexRequest(input)); - } - return Arrays.asList((ActionRequest) new IndexRequest(input)); - } - }; - - BulkIndexRequestIterator it = new BulkIndexRequestIterator(input, converter); - - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "foo"); - - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "bar"); - - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "3requests"); - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "3requests"); - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "3requests"); - - assertThat(it.hasNext()).isTrue(); - assertIndex(it.next(), "baz"); - - assertThat(it.hasNext()).isFalse(); - try { - it.next(); - fail(); - } catch (NoSuchElementException e) { - // expected - } - } - - @Test - public void empty() throws Exception { - List input = Collections.emptyList(); - BulkIndexRequestIterator.InputConverter converter = mock(BulkIndexRequestIterator.InputConverter.class); - - BulkIndexRequestIterator it = new BulkIndexRequestIterator(input, converter); - - assertThat(it.hasNext()).isFalse(); - verifyZeroInteractions(converter); - } - - @Test - public void removal_is_not_supported() throws Exception { - List input = Arrays.asList("foo"); - BulkIndexRequestIterator.InputConverter converter = mock(BulkIndexRequestIterator.InputConverter.class); - - BulkIndexRequestIterator it = new BulkIndexRequestIterator(input, converter); - - try { - it.remove(); - fail(); - } catch (UnsupportedOperationException e) { - // expected - } - } - - private void assertIndex(ActionRequest req, String indexName) { - assertThat(req).isNotNull(); - assertThat(req).isInstanceOf(IndexRequest.class); - assertThat(((IndexRequest) req).index()).isEqualTo(indexName); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java new file mode 100644 index 00000000000..7dfa0315e66 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -0,0 +1,101 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.junit.Rule; +import org.junit.Test; + +import static org.fest.assertions.Assertions.assertThat; + +public class BulkIndexerTest { + + @Rule + public EsTester esTester = new EsTester().addDefinitions(new FakeIndexDefinition().setReplicas(1)); + + @Test + public void index_nothing() throws Exception { + BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX); + indexer.start(); + indexer.stop(); + + assertThat(count()).isEqualTo(0); + } + + @Test + public void index_documents() throws Exception { + BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX) + .setRefresh(true); + indexer.start(); + indexer.add(newIndexRequest(42)); + indexer.add(newIndexRequest(78)); + + // request is not sent yet + assertThat(count()).isEqualTo(0); + + // send remaining requests + indexer.stop(); + assertThat(count()).isEqualTo(2); + } + + @Test + public void large_indexing() throws Exception { + // index has one replica + assertThat(replicas()).isEqualTo(1); + + BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX) + .setLarge(true) + .setFlushByteSize(new ByteSizeValue(1, ByteSizeUnit.BYTES).bytes()); + indexer.start(); + + // replicas are temporarily disabled + assertThat(replicas()).isEqualTo(0); + + for (int i = 0; i < 10; i++) { + indexer.add(newIndexRequest(i)); + } + indexer.stop(); + + assertThat(count()).isEqualTo(10); + + // replicas are re-enabled + assertThat(replicas()).isEqualTo(1); + } + + private long count() { + return esTester.client().prepareCount(FakeIndexDefinition.INDEX).get().getCount(); + } + + private int replicas() { + GetSettingsResponse settingsResp = esTester.client().nativeClient().admin().indices() + .prepareGetSettings(FakeIndexDefinition.INDEX).get(); + return Integer.parseInt(settingsResp.getSetting(FakeIndexDefinition.INDEX, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + } + + private IndexRequest newIndexRequest(int intField) { + return new IndexRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.TYPE) + .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, intField)); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/DefaultMappingSettingsTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java similarity index 87% rename from server/sonar-server/src/test/java/org/sonar/server/es/DefaultMappingSettingsTest.java rename to server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java index 3e2a1303406..a6f8eb33106 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/DefaultMappingSettingsTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java @@ -25,11 +25,11 @@ import org.sonar.test.TestUtils; import static org.fest.assertions.Assertions.assertThat; -public class DefaultMappingSettingsTest { +public class DefaultIndexSettingsTest { @Test public void defaults() throws Exception { - ImmutableMap map = DefaultMappingSettings.defaults().build().getAsMap(); + ImmutableMap map = DefaultIndexSettings.defaults().build().getAsMap(); assertThat(map).isNotEmpty(); // test some values @@ -40,7 +40,7 @@ public class DefaultMappingSettingsTest { @Test public void only_statics() throws Exception { - TestUtils.hasOnlyPrivateConstructors(DefaultMappingSettings.class); + TestUtils.hasOnlyPrivateConstructors(DefaultIndexSettings.class); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index 82837661d53..d7e199981f0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -19,8 +19,9 @@ */ package org.sonar.server.es; +import com.google.common.collect.Lists; +import org.apache.commons.lang.math.RandomUtils; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.ImmutableSettings; @@ -29,46 +30,72 @@ import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.junit.rules.ExternalResource; import org.sonar.api.config.Settings; +import org.sonar.api.platform.ComponentContainer; import org.sonar.core.profiling.Profiling; -import java.util.Date; +import java.util.Collections; +import java.util.List; import static org.fest.assertions.Assertions.assertThat; public class EsTester extends ExternalResource { - /** - * This instance is shared for performance reasons. Never stopped. - */ - private static Node sharedNode; - private static EsClient client; + private static int PROCESS_ID = RandomUtils.nextInt(Integer.MAX_VALUE); + private Node node; + private EsClient client; + private final List definitions = Lists.newArrayList(); + + public EsTester addDefinitions(IndexDefinition... defs) { + Collections.addAll(definitions, defs); + return this; + } - @Override protected void before() throws Throwable { - if (sharedNode == null) { - String nodeName = EsTester.class.getName(); - sharedNode = NodeBuilder.nodeBuilder().local(true).data(true).settings(ImmutableSettings.builder() - .put(ClusterName.SETTING, nodeName) - .put("node.name", nodeName) - // the two following properties are probably not used because they are - // declared on indices too - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors) - .put("processors", 1) - .put("http.enabled", false) - .put("index.store.type", "ram") - .put("config.ignore_system_properties", true) - .put("gateway.type", "none")) - .build(); - sharedNode.start(); - assertThat(DiscoveryNode.localNode(sharedNode.settings())).isTrue(); - client = new EsClient(new Profiling(new Settings()), sharedNode.client()); + String nodeName = "es-ram-" + PROCESS_ID; + node = NodeBuilder.nodeBuilder().local(true).data(true).settings(ImmutableSettings.builder() + .put("cluster.name", nodeName) + .put("node.name", nodeName) + // the two following properties are probably not used because they are + // declared on indices too + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + // limit the number of threads created (see org.elasticsearch.common.util.concurrent.EsExecutors) + .put("processors", 1) + .put("http.enabled", false) + .put("index.store.type", "memory") + .put("config.ignore_system_properties", true) + // reuse the same directory than other tests for faster initialization + .put("path.home", "target/es-" + PROCESS_ID) + .put("gateway.type", "none")) + .build(); + node.start(); + assertThat(DiscoveryNode.localNode(node.settings())).isTrue(); + + // delete the indices created by previous tests + DeleteIndexResponse response = node.client().admin().indices().prepareDelete("_all").get(); + assertThat(response.isAcknowledged()).isTrue(); + + client = new EsClient(new Profiling(new Settings()), node.client()); + client.start(); - } else { - // delete the indices created by previous tests - DeleteIndexResponse response = sharedNode.client().admin().indices().prepareDelete("_all").get(); - assertThat(response.isAcknowledged()).isTrue(); + if (!definitions.isEmpty()) { + ComponentContainer container = new ComponentContainer(); + container.addSingletons(definitions); + container.addSingleton(client); + container.addSingleton(IndexRegistry.class); + container.addSingleton(IndexCreator.class); + container.startComponents(); + } + } + + @Override + protected void after() { + if (client != null) { + client.stop(); + } + if (node != null) { + node.stop(); + node.close(); } } @@ -87,7 +114,7 @@ public class EsTester extends ExternalResource { } public Node node() { - return sharedNode; + return node; } public EsClient client() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/FakeIndexDefinition.java b/server/sonar-server/src/test/java/org/sonar/server/es/FakeIndexDefinition.java new file mode 100644 index 00000000000..d0e9e56dfd5 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/FakeIndexDefinition.java @@ -0,0 +1,44 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.elasticsearch.cluster.metadata.IndexMetaData; + +public class FakeIndexDefinition implements IndexDefinition { + + public static final String INDEX = "fakes"; + public static final String TYPE = "fake"; + public static final String INT_FIELD = "intField"; + + private int replicas = 0; + + public FakeIndexDefinition setReplicas(int replicas) { + this.replicas = replicas; + return this; + } + + @Override + public void define(IndexDefinitionContext context) { + NewIndex index = context.create(INDEX); + index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + NewIndex.NewIndexType type = index.createType(TYPE); + type.createIntegerField(INT_FIELD); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 25917025cac..8f46e89cc6b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -26,6 +26,7 @@ import org.junit.Rule; import org.junit.Test; import javax.annotation.CheckForNull; + import java.io.IOException; import java.util.Map; @@ -40,7 +41,9 @@ public class IndexCreatorTest { public void create_index() throws Exception { assertThat(mappings()).isEmpty(); - IndexCreator creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinition()}); + IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()}); + registry.start(); + IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); // check that index is created with related mapping @@ -58,24 +61,23 @@ public class IndexCreatorTest { assertThat(mappings()).isNotEmpty(); } - private String setting(String indexName, String settingKey) { - GetSettingsResponse indexSettings = es.client().nativeClient().admin().indices().prepareGetSettings(indexName).get(); - return indexSettings.getSetting(indexName, settingKey); - } - @Test public void recreate_index_on_definition_changes() throws Exception { assertThat(mappings()).isEmpty(); // v1 - IndexCreator creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinition()}); + IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()}); + registry.start(); + IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); creator.stop(); String hashV1 = setting("fakes", "index.sonar_hash"); assertThat(hashV1).isNotEmpty(); // v2 - creator = new IndexCreator(es.client(), new IndexDefinition[]{new FakeIndexDefinitionV2()}); + registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinitionV2()}); + registry.start(); + creator = new IndexCreator(es.client(), registry); creator.start(); ImmutableOpenMap> mappings = mappings(); MappingMetaData mapping = mappings.get("fakes").get("fake"); @@ -87,6 +89,11 @@ public class IndexCreatorTest { creator.stop(); } + private String setting(String indexName, String settingKey) { + GetSettingsResponse indexSettings = es.client().nativeClient().admin().indices().prepareGetSettings(indexName).get(); + return indexSettings.getSetting(indexName, settingKey); + } + private ImmutableOpenMap> mappings() { return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); } @@ -105,7 +112,7 @@ public class IndexCreatorTest { @Override public void define(IndexDefinitionContext context) { NewIndex index = context.create("fakes"); - NewIndex.NewMapping mapping = index.createMapping("fake"); + NewIndex.NewIndexType mapping = index.createType("fake"); mapping.stringFieldBuilder("key").build(); mapping.createDateTimeField("updatedAt"); } @@ -115,7 +122,7 @@ public class IndexCreatorTest { @Override public void define(IndexDefinitionContext context) { NewIndex index = context.create("fakes"); - NewIndex.NewMapping mapping = index.createMapping("fake"); + NewIndex.NewIndexType mapping = index.createType("fake"); mapping.stringFieldBuilder("key").build(); mapping.createDateTimeField("updatedAt"); mapping.createIntegerField("newField"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexHashTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexHashTest.java index 8f537276c57..7856f9852c2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexHashTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexHashTest.java @@ -29,25 +29,25 @@ public class IndexHashTest { @Test public void of() throws Exception { - NewIndex indexV1 = createIndex(); + IndexRegistry.Index indexV1 = new IndexRegistry.Index(createIndex()); String hashV1 = new IndexHash().of(indexV1); assertThat(hashV1).isNotEmpty(); // always the same assertThat(hashV1).isEqualTo(new IndexHash().of(indexV1)); - NewIndex indexV2 = createIndex(); - indexV2.getMappings().get("fake").createIntegerField("max"); - String hashV2 = new IndexHash().of(indexV2); + NewIndex newIndexV2 = createIndex(); + newIndexV2.getTypes().get("fake").createIntegerField("max"); + String hashV2 = new IndexHash().of(new IndexRegistry.Index(newIndexV2)); assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1); } private NewIndex createIndex() { - NewIndex index = new NewIndex("fakes"); - NewIndex.NewMapping mapping = index.createMapping("fake"); + NewIndex newIndex = new NewIndex("fakes"); + NewIndex.NewIndexType mapping = newIndex.createType("fake"); mapping.setAttribute("list_attr", Arrays.asList("foo", "bar")); mapping.stringFieldBuilder("key").build(); mapping.createDateTimeField("updatedAt"); - return index; + return newIndex; } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexDefinitionTest.java index 1a5d442f0ad..893d9f563fe 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexDefinitionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexDefinitionTest.java @@ -37,7 +37,7 @@ public class IssueIndexDefinitionTest { assertThat(context.getIndices()).hasSize(1); NewIndex issuesIndex = context.getIndices().get("issues"); assertThat(issuesIndex).isNotNull(); - assertThat(issuesIndex.getMappings().keySet()).containsOnly("issue", "issueAuthorization"); + assertThat(issuesIndex.getTypes().keySet()).containsOnly("issue", "issueAuthorization"); // no cluster by default assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("1"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java index 80df925250d..eac18f6c939 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/NewIndexTest.java @@ -35,7 +35,7 @@ public class NewIndexTest { public void most_basic_index() throws Exception { NewIndex index = new NewIndex("issues"); assertThat(index.getName()).isEqualTo("issues"); - assertThat(index.getMappings()).isEmpty(); + assertThat(index.getTypes()).isEmpty(); Settings settings = index.getSettings().build(); // test some basic settings assertThat(settings.get("index.number_of_shards")).isNotEmpty(); @@ -54,7 +54,7 @@ public class NewIndexTest { @Test public void define_fields() throws Exception { NewIndex index = new NewIndex("issues"); - NewIndex.NewMapping mapping = index.createMapping("issue"); + NewIndex.NewIndexType mapping = index.createType("issue"); mapping.setAttribute("dynamic", "true"); mapping.setProperty("foo_field", ImmutableMap.of("type", "string")); mapping.createBooleanField("boolean_field"); @@ -66,7 +66,7 @@ public class NewIndexTest { mapping.createShortField("short_field"); mapping.createUuidPathField("uuid_path_field"); - mapping = index.getMappings().get("issue"); + mapping = index.getTypes().get("issue"); assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true"); assertThat(mapping).isNotNull(); assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class); @@ -84,7 +84,7 @@ public class NewIndexTest { @Test public void define_string_field() throws Exception { NewIndex index = new NewIndex("issues"); - NewIndex.NewMapping mapping = index.createMapping("issue"); + NewIndex.NewIndexType mapping = index.createType("issue"); mapping.stringFieldBuilder("basic_field").build(); mapping.stringFieldBuilder("all_capabilities_field") .enableGramSearch() diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java index 74073a45029..f0a2d2c9a57 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.common.settings.ImmutableSettings; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -86,6 +87,13 @@ public class BaseIndexTest { searchClient = new SearchClient(settings); } + @After + public void tearDown() throws Exception { + if (searchClient != null) { + searchClient.stop(); + } + } + @Test public void can_load() { BaseIndex index = getIndex(this.searchClient); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyBulkRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyBulkRequestBuilderTest.java index febf81402ee..b31c40f3fcf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyBulkRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyBulkRequestBuilderTest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -41,6 +42,11 @@ public class ProxyBulkRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void bulk() { BulkRequestBuilder bulkRequestBuilder = searchClient.prepareBulk(); @@ -96,6 +102,7 @@ public class ProxyBulkRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test(expected = UnsupportedOperationException.class) diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterHealthRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterHealthRequestBuilderTest.java index c180ca3ef85..60fb4459b9b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterHealthRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterHealthRequestBuilderTest.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyClusterHealthRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void state() { try { @@ -70,6 +76,7 @@ public class ProxyClusterHealthRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStateRequestBuilderText.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStateRequestBuilderText.java index 1450e3debeb..b6b4b7025b1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStateRequestBuilderText.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStateRequestBuilderText.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyClusterStateRequestBuilderText { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void state() { try { @@ -56,9 +62,9 @@ public class ProxyClusterStateRequestBuilderText { @Test public void with_profiling_basic() { + Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name())); + SearchClient searchClient = new SearchClient(new Settings(), profiling); try { - Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name())); - SearchClient searchClient = new SearchClient(new Settings(), profiling); ClusterStateRequestBuilder requestBuilder = searchClient.prepareState(); requestBuilder.get(); @@ -70,6 +76,7 @@ public class ProxyClusterStateRequestBuilderText { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStatsRequestBuilderText.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStatsRequestBuilderText.java index af0d1737ae3..f130a7fa80d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStatsRequestBuilderText.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyClusterStatsRequestBuilderText.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyClusterStatsRequestBuilderText { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void stats() { try { @@ -70,6 +76,7 @@ public class ProxyClusterStatsRequestBuilderText { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCountRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCountRequestBuilderTest.java index 016e2bd3f86..e927fefe5b2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCountRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCountRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyCountRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void count() { try { @@ -69,6 +75,7 @@ public class ProxyCountRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCreateIndexRequestBuilderText.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCreateIndexRequestBuilderText.java index ea715872792..14260a0a7f6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCreateIndexRequestBuilderText.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyCreateIndexRequestBuilderText.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyCreateIndexRequestBuilderText { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void create_index() { try { @@ -69,6 +75,7 @@ public class ProxyCreateIndexRequestBuilderText { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyDeleteByQueryRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyDeleteByQueryRequestBuilderTest.java index e939c528553..39ed9a9a5f6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyDeleteByQueryRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyDeleteByQueryRequestBuilderTest.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -36,6 +37,11 @@ public class ProxyDeleteByQueryRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void bulk() { try { @@ -69,6 +75,7 @@ public class ProxyDeleteByQueryRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyFlushRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyFlushRequestBuilderTest.java index 5639c9ed0bf..354f333b3cc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyFlushRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyFlushRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyFlushRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void flush() { try { @@ -68,6 +74,7 @@ public class ProxyFlushRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyGetRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyGetRequestBuilderTest.java index 27d371db52f..0ca52fe95f5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyGetRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyGetRequestBuilderTest.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -36,6 +37,11 @@ public class ProxyGetRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void get() { try { @@ -83,6 +89,7 @@ public class ProxyGetRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesExistsRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesExistsRequestBuilderTest.java index c528dd17c1b..881bf7ff65c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesExistsRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesExistsRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyIndicesExistsRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void exists() { try { @@ -62,6 +68,7 @@ public class ProxyIndicesExistsRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesStatsRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesStatsRequestBuilderTest.java index 0886a0fb3d9..55b80f340c5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesStatsRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyIndicesStatsRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyIndicesStatsRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void stats() { try { @@ -68,6 +74,7 @@ public class ProxyIndicesStatsRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyMultiGetRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyMultiGetRequestBuilderTest.java index e53ce97fdfc..eac43b06bb8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyMultiGetRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyMultiGetRequestBuilderTest.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.fetch.source.FetchSourceContext; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -38,6 +39,11 @@ public class ProxyMultiGetRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void multi_get() { try { @@ -82,6 +88,7 @@ public class ProxyMultiGetRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyNodesStatsRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyNodesStatsRequestBuilderTest.java index ab14b04ac84..0536a91cd1e 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyNodesStatsRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyNodesStatsRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxyNodesStatsRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void stats() { try { @@ -70,6 +76,7 @@ public class ProxyNodesStatsRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyPutMappingRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyPutMappingRequestBuilderTest.java index 1093fc56507..c129c2bf663 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyPutMappingRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyPutMappingRequestBuilderTest.java @@ -23,6 +23,7 @@ package org.sonar.server.search.request; import com.google.common.collect.ImmutableMap; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -40,6 +41,11 @@ public class ProxyPutMappingRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void put_mapping() { try { @@ -85,6 +91,7 @@ public class ProxyPutMappingRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyRefreshRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyRefreshRequestBuilderTest.java index 4c786a5cc93..3d3efb956b3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyRefreshRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxyRefreshRequestBuilderTest.java @@ -22,6 +22,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -36,6 +37,11 @@ public class ProxyRefreshRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void refresh() { try { @@ -73,6 +79,7 @@ public class ProxyRefreshRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchRequestBuilderTest.java index e9dd3820ce1..682d6a68f81 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchRequestBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.server.search.request; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -35,6 +36,11 @@ public class ProxySearchRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void search() { try { @@ -72,6 +78,7 @@ public class ProxySearchRequestBuilderTest { } // TODO assert profiling + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchScrollRequestBuilderTest.java index c7dfbe6b786..34f5db920c0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchScrollRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/request/ProxySearchScrollRequestBuilderTest.java @@ -23,6 +23,7 @@ package org.sonar.server.search.request; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; +import org.junit.After; import org.junit.Test; import org.sonar.api.config.Settings; import org.sonar.core.profiling.Profiling; @@ -37,6 +38,11 @@ public class ProxySearchScrollRequestBuilderTest { Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.FULL.name())); SearchClient searchClient = new SearchClient(new Settings(), profiling); + @After + public void tearDown() throws Exception { + searchClient.stop(); + } + @Test public void search_scroll() { try { @@ -56,9 +62,9 @@ public class ProxySearchScrollRequestBuilderTest { @Test public void with_profiling_basic() { + Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name())); + SearchClient searchClient = new SearchClient(new Settings(), profiling); try { - Profiling profiling = new Profiling(new Settings().setProperty(Profiling.CONFIG_PROFILING_LEVEL, Profiling.Level.BASIC.name())); - SearchClient searchClient = new SearchClient(new Settings(), profiling); SearchResponse search = searchClient.prepareSearch(IndexDefinition.RULE.getIndexName()) .setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueSeconds(3L)) @@ -71,6 +77,7 @@ public class ProxySearchScrollRequestBuilderTest { assertThat(e).isInstanceOf(IllegalStateException.class); assertThat(e.getMessage()).contains("Fail to execute ES search request '{}' on indices '[rules]'"); } + searchClient.stop(); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java index dd4b6ce5162..a543d47403c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java @@ -105,10 +105,15 @@ public class ServerTester extends ExternalResource { } } - searchServer.start(); - platform.init(properties); - platform.addComponents(components); - platform.doStart(); + try { + searchServer.start(); + platform.init(properties); + platform.addComponents(components); + platform.doStart(); + } catch (RuntimeException e) { + stop(); + throw e; + } if (!platform.isStarted()) { throw new IllegalStateException("Server not started. You should check that db migrations " + "are correctly declared, for example in schema-h2.sql or DatabaseVersion"); @@ -118,7 +123,7 @@ public class ServerTester extends ExternalResource { private File createTempDir() { try { // Technique to create a temp directory from a temp file - File f = File.createTempFile("SonarQube", ""); + File f = File.createTempFile("tmp-sq", ""); f.delete(); f.mkdir(); return f; @@ -139,8 +144,20 @@ public class ServerTester extends ExternalResource { * This method should not be called by test when ServerTester is annotated with {@link org.junit.Rule} */ public void stop() { - platform.doStop(); - searchServer.stop(); + try { + if (platform != null) { + platform.doStop(); + } + } catch (Exception e) { + e.printStackTrace(); + } + try { + if (searchServer != null) { + searchServer.stop(); + } + } catch (Exception e) { + e.printStackTrace(); + } FileUtils.deleteQuietly(homeDir); } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/feed.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/feed.xml new file mode 100644 index 00000000000..f072b93cb9d --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/feed.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/schema.sql b/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/schema.sql new file mode 100644 index 00000000000..09a292231eb --- /dev/null +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/ResultSetIteratorTest/schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE "FAKE" ( + "ID" INTEGER NOT NULL GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT BY 1), + "KEE" VARCHAR(200) NOT NULL +); -- 2.39.5