From 05266f94881823d0c78a5a4f1a7cb905db8a8550 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Wed, 25 May 2016 13:32:48 +0200 Subject: SONAR-7653 upgrade to Elasticsearch 2.3.3 --- pom.xml | 29 +--- server/sonar-search/pom.xml | 4 + .../src/main/java/org/sonar/search/EsSettings.java | 44 +++--- .../main/java/org/sonar/search/SearchServer.java | 17 +- .../test/java/org/sonar/search/EsSettingsTest.java | 11 +- .../java/org/sonar/search/SearchServerTest.java | 11 +- .../server/benchmark/IssueIndexBenchmarkTest.java | 2 +- server/sonar-server/pom.xml | 11 ++ .../sonar/server/activity/index/ActivityDoc.java | 15 ++ .../sonar/server/activity/index/ActivityIndex.java | 36 ++--- .../activity/index/ActivityIndexDefinition.java | 2 - .../computation/component/ComponentImpl.java | 2 +- .../computation/dbcleaner/IndexPurgeListener.java | 4 +- .../computation/issue/TrackerRawInputFactory.java | 2 +- .../computation/issue/commonrule/CommonRule.java | 4 +- .../server/computation/scm/ReportScmInfo.java | 2 +- .../step/LoadReportAnalysisMetadataHolderStep.java | 2 +- .../src/main/java/org/sonar/server/es/BaseDoc.java | 8 + .../main/java/org/sonar/server/es/BulkIndexer.java | 22 ++- .../org/sonar/server/es/DefaultIndexSettings.java | 6 +- .../main/java/org/sonar/server/es/EsClient.java | 48 +++--- .../src/main/java/org/sonar/server/es/EsUtils.java | 19 ++- .../src/main/java/org/sonar/server/es/Facets.java | 18 +-- .../java/org/sonar/server/es/IndexCreator.java | 10 +- .../main/java/org/sonar/server/es/NewIndex.java | 35 +---- .../org/sonar/server/es/StickyFacetBuilder.java | 31 ++-- .../server/es/request/ProxyBulkRequestBuilder.java | 6 +- .../es/request/ProxyClearCacheRequestBuilder.java | 13 +- .../request/ProxyClusterHealthRequestBuilder.java | 3 +- .../request/ProxyClusterStateRequestBuilder.java | 3 +- .../request/ProxyClusterStatsRequestBuilder.java | 5 +- .../es/request/ProxyCountRequestBuilder.java | 3 +- .../es/request/ProxyCreateIndexRequestBuilder.java | 3 +- .../request/ProxyDeleteByQueryRequestBuilder.java | 102 ------------ .../es/request/ProxyDeleteRequestBuilder.java | 3 +- .../es/request/ProxyFlushRequestBuilder.java | 3 +- .../server/es/request/ProxyGetRequestBuilder.java | 3 +- .../es/request/ProxyIndexRequestBuilder.java | 3 +- .../request/ProxyIndicesExistsRequestBuilder.java | 3 +- .../request/ProxyIndicesStatsRequestBuilder.java | 3 +- .../es/request/ProxyMultiGetRequestBuilder.java | 3 +- .../es/request/ProxyNodesStatsRequestBuilder.java | 3 +- .../es/request/ProxyPutMappingRequestBuilder.java | 3 +- .../es/request/ProxyRefreshRequestBuilder.java | 3 +- .../es/request/ProxySearchRequestBuilder.java | 3 +- .../request/ProxySearchScrollRequestBuilder.java | 3 +- .../server/issue/index/IssueAuthorizationDoc.java | 54 +++++++ .../issue/index/IssueAuthorizationIndexer.java | 1 + .../org/sonar/server/issue/index/IssueDoc.java | 15 ++ .../org/sonar/server/issue/index/IssueIndex.java | 171 ++++++++++----------- .../server/issue/index/IssueIndexDefinition.java | 19 ++- .../org/sonar/server/issue/index/IssueIndexer.java | 23 +-- .../org/sonar/server/platform/BackendCleanup.java | 6 +- .../server/platform/monitoring/EsMonitor.java | 17 +- .../server/qualityprofile/index/ActiveRuleDoc.java | 15 ++ .../qualityprofile/index/ActiveRuleIndex.java | 26 ++-- .../qualityprofile/index/ActiveRuleIndexer.java | 16 +- .../java/org/sonar/server/rule/index/RuleDoc.java | 20 +++ .../org/sonar/server/rule/index/RuleIndex.java | 92 ++++++----- .../server/rule/index/RuleIndexDefinition.java | 38 ++--- .../org/sonar/server/rule/index/RuleIndexer.java | 4 +- .../main/java/org/sonar/server/search/Facets.java | 21 ++- .../java/org/sonar/server/search/IndexUtils.java | 6 +- .../sonar/server/test/index/CoveredFileDoc.java | 15 ++ .../java/org/sonar/server/test/index/TestDoc.java | 16 ++ .../org/sonar/server/test/index/TestIndex.java | 20 +-- .../server/test/index/TestIndexDefinition.java | 12 +- .../org/sonar/server/test/index/TestIndexer.java | 9 +- .../server/test/index/TestResultSetIterator.java | 3 +- .../java/org/sonar/server/user/index/UserDoc.java | 24 ++- .../org/sonar/server/user/index/UserIndex.java | 34 ++-- .../java/org/sonar/server/view/index/ViewDoc.java | 15 ++ .../org/sonar/server/view/index/ViewIndex.java | 19 +-- .../org/sonar/server/view/index/ViewIndexer.java | 4 +- .../dbcleaner/IndexPurgeListenerTest.java | 4 +- .../issue/ScmAccountToUserLoaderTest.java | 6 +- .../java/org/sonar/server/es/BulkIndexerTest.java | 5 +- .../sonar/server/es/DefaultIndexSettingsTest.java | 4 +- .../java/org/sonar/server/es/EsServerHolder.java | 12 +- .../test/java/org/sonar/server/es/EsTester.java | 19 +-- .../src/test/java/org/sonar/server/es/FakeDoc.java | 15 ++ .../test/java/org/sonar/server/es/SortingTest.java | 18 +-- .../request/ProxyClearCacheRequestBuilderTest.java | 4 +- .../ProxyClusterHealthRequestBuilderTest.java | 2 +- .../ProxyDeleteByQueryRequestBuilderTest.java | 93 ----------- .../request/ProxyPutMappingRequestBuilderTest.java | 2 - .../sonar/server/issue/index/IssueIndexTest.java | 1 + .../sonar/server/issue/index/IssueIndexerTest.java | 45 +++--- .../server/platform/BackendCleanupMediumTest.java | 6 +- .../server/platform/monitoring/EsMonitorTest.java | 2 +- .../server/project/ws/BulkDeleteActionTest.java | 4 +- .../sonar/server/project/ws/DeleteActionTest.java | 4 +- .../server/rule/index/RuleIndexDefinitionTest.java | 2 +- .../org/sonar/server/rule/index/RuleIndexTest.java | 7 +- .../java/org/sonar/server/search/BaseDocTest.java | 56 +++++++ .../org/sonar/server/search/FacetsMediumTest.java | 93 +++++++---- .../org/sonar/server/test/index/TestIndexTest.java | 38 ++--- .../sonar/server/test/index/TestIndexerTest.java | 1 + .../test/index/TestResultSetIteratorTest.java | 7 +- .../org/sonar/server/user/index/UserIndexTest.java | 26 ++-- .../org/sonar/server/view/index/ViewIndexTest.java | 10 +- sonar-application/assembly.xml | 1 + sonar-application/pom.xml | 4 +- .../src/main/java/org/sonar/db/purge/PurgeDao.java | 2 +- .../java/org/sonar/db/purge/PurgeListener.java | 4 +- .../test/java/org/sonar/db/purge/PurgeDaoTest.java | 27 ++-- .../should_delete_old_closed_issues-result.xml | 12 +- .../should_delete_old_closed_issues.xml | 12 +- 108 files changed, 954 insertions(+), 878 deletions(-) delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilder.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationDoc.java delete mode 100644 server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilderTest.java diff --git a/pom.xml b/pom.xml index 7a2e450a1ed..ae356ad3e6a 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ 1.1.3 1.7.12 8.0.30 - 1.7.5 + 2.3.3 3.11 2.6.0 @@ -554,32 +554,15 @@ ${sonarCSharp.version} sonar-plugin + + net.java.dev.jna + jna + 4.1.0 + org.elasticsearch elasticsearch ${elasticsearch.version} - - - org.apache.lucene - lucene-spatial - - - org.ow2.asm - asm - - - org.ow2.asm - asm-commons - - - org.antlr - antlr-runtime - - - org.yaml - snakeyaml - - com.github.tlrx diff --git a/server/sonar-search/pom.xml b/server/sonar-search/pom.xml index c006f179e25..d155517a242 100644 --- a/server/sonar-search/pom.xml +++ b/server/sonar-search/pom.xml @@ -23,6 +23,10 @@ org.elasticsearch elasticsearch + + net.java.dev.jna + jna + com.google.code.findbugs jsr305 diff --git a/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java b/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java index 50802429918..950afcfbc76 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java +++ b/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java @@ -20,13 +20,14 @@ package org.sonar.search; import java.io.File; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.Set; import java.util.TreeSet; import org.apache.commons.lang.StringUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +73,7 @@ public class EsSettings implements EsSettingsMBean { } Settings build() { - ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder(); + Settings.Builder builder = Settings.builder(); configureFileSystem(builder); configureIndexDefaults(builder); configureNetwork(builder); @@ -81,10 +82,9 @@ public class EsSettings implements EsSettingsMBean { return builder.build(); } - private void configureFileSystem(ImmutableSettings.Builder builder) { + private void configureFileSystem(Settings.Builder builder) { File homeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME); File dataDir; - File workDir; File logDir; // data dir @@ -96,15 +96,8 @@ public class EsSettings implements EsSettingsMBean { } builder.put("path.data", dataDir.getAbsolutePath()); - // working dir - String workPath = props.value(ProcessProperties.PATH_TEMP); - if (StringUtils.isNotEmpty(workPath)) { - workDir = new File(workPath); - } else { - workDir = new File(homeDir, "temp"); - } - builder.put("path.work", workDir.getAbsolutePath()); - builder.put("path.plugins", workDir.getAbsolutePath()); + String tempPath = props.value(ProcessProperties.PATH_TEMP); + builder.put("path.home", new File(tempPath, "es")); // log dir String logPath = props.value(ProcessProperties.PATH_LOGS); @@ -116,16 +109,16 @@ public class EsSettings implements EsSettingsMBean { builder.put("path.logs", logDir.getAbsolutePath()); } - private void configureNetwork(ImmutableSettings.Builder builder) { - // the following properties can't be null as default values are defined by app process - String host = props.nonNullValue(ProcessProperties.SEARCH_HOST); + private void configureNetwork(Settings.Builder builder) { + InetAddress host = readHost(); int port = Integer.parseInt(props.nonNullValue(ProcessProperties.SEARCH_PORT)); LOGGER.info("Elasticsearch listening on {}:{}", host, port); // disable multicast builder.put("discovery.zen.ping.multicast.enabled", "false"); builder.put("transport.tcp.port", port); - builder.put("transport.host", host); + builder.put("transport.host", host.getHostAddress()); + builder.put("network.host", host.getHostAddress()); // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ? builder.put("network.tcp.reuse_address", true); @@ -140,12 +133,21 @@ public class EsSettings implements EsSettingsMBean { // see https://github.com/lmenezes/elasticsearch-kopf/issues/195 builder.put("http.cors.enabled", true); builder.put("http.enabled", true); - builder.put("http.host", host); + builder.put("http.host", host.getHostAddress()); builder.put("http.port", httpPort); } } - private static void configureIndexDefaults(ImmutableSettings.Builder builder) { + private InetAddress readHost() { + String hostProperty = props.nonNullValue(ProcessProperties.SEARCH_HOST); + try { + return InetAddress.getByName(hostProperty); + } catch (UnknownHostException e) { + throw new IllegalStateException("Can not resolve host [" + hostProperty + "]. Please check network settings and property " + ProcessProperties.SEARCH_HOST, e); + } + } + + private static void configureIndexDefaults(Settings.Builder builder) { builder .put("index.number_of_shards", "1") .put("index.refresh_interval", "30s") @@ -153,7 +155,7 @@ public class EsSettings implements EsSettingsMBean { .put("index.mapper.dynamic", false); } - private void configureCluster(ImmutableSettings.Builder builder) { + private void configureCluster(Settings.Builder builder) { int replicationFactor = 0; if (inCluster()) { replicationFactor = 1; @@ -178,7 +180,7 @@ public class EsSettings implements EsSettingsMBean { builder.put("node.name", nodeName); } - private void configureMarvel(ImmutableSettings.Builder builder) { + private void configureMarvel(Settings.Builder builder) { Set marvels = new TreeSet<>(); marvels.addAll(Arrays.asList(StringUtils.split(props.value(PROP_MARVEL_HOSTS, ""), ","))); diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java index 899ab6891b8..f21987fb994 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java +++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java @@ -19,9 +19,11 @@ */ package org.sonar.search; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.apache.lucene.util.StringHelper; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.internal.InternalNode; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; import org.sonar.process.Jmx; import org.sonar.process.MinimumViableSystem; import org.sonar.process.Monitored; @@ -31,7 +33,7 @@ import org.sonar.process.Props; public class SearchServer implements Monitored { private final EsSettings settings; - private InternalNode node; + private Node node; public SearchServer(Props props) { this.settings = new EsSettings(props); @@ -42,10 +44,17 @@ public class SearchServer implements Monitored { @Override public void start() { Jmx.register(EsSettingsMBean.OBJECT_NAME, settings); - node = new InternalNode(settings.build(), false); + initBootstrap(); + node = NodeBuilder.nodeBuilder().settings(settings.build()).build(); node.start(); } + // copied from https://github.com/elastic/elasticsearch/blob/v2.3.3/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java + private void initBootstrap() { + // init lucene random seed. it will use /dev/urandom where available: + StringHelper.randomId(); + } + @Override public boolean isUp() { return node != null && node.client().admin().cluster().prepareHealth() diff --git a/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java b/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java index f8f12f25272..35fd0c236ef 100644 --- a/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java +++ b/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java @@ -19,6 +19,9 @@ */ package org.sonar.search; +import java.io.File; +import java.io.IOException; +import java.util.Properties; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; import org.junit.Test; @@ -27,10 +30,6 @@ import org.sonar.process.MessageException; import org.sonar.process.ProcessProperties; import org.sonar.process.Props; -import java.io.File; -import java.io.IOException; -import java.util.Properties; - import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -60,7 +59,7 @@ public class EsSettingsTest { assertThat(generated.get("path.data")).isNotNull(); assertThat(generated.get("path.logs")).isNotNull(); - assertThat(generated.get("path.work")).isNotNull(); + assertThat(generated.get("path.home")).isNotNull(); // http is disabled for security reasons assertThat(generated.get("http.enabled")).isEqualTo("false"); @@ -98,7 +97,7 @@ public class EsSettingsTest { assertThat(settings.get("path.data")).isEqualTo(new File(dataDir, "es").getAbsolutePath()); assertThat(settings.get("path.logs")).isEqualTo(logDir.getAbsolutePath()); - assertThat(settings.get("path.work")).isEqualTo(tempDir.getAbsolutePath()); + assertThat(settings.get("path.home")).isEqualTo(new File(tempDir, "es").getAbsolutePath()); } @Test diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java index ebf7e106f4b..8b63fb52eb4 100644 --- a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java +++ b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java @@ -21,11 +21,10 @@ package org.sonar.search; import java.net.InetAddress; import java.util.Properties; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.junit.After; @@ -72,8 +71,8 @@ public class SearchServerTest { public void start_stop_server() throws Exception { Props props = new Props(new Properties()); // the following properties have always default values (see ProcessProperties) - String host = InetAddress.getLocalHost().getHostAddress(); - props.set(ProcessProperties.SEARCH_HOST, host); + InetAddress host = InetAddress.getLocalHost(); + props.set(ProcessProperties.SEARCH_HOST, host.getHostAddress()); props.set(ProcessProperties.SEARCH_PORT, String.valueOf(port)); props.set(ProcessProperties.CLUSTER_NAME, CLUSTER_NAME); props.set(ProcessProperties.CLUSTER_NODE_NAME, "test"); @@ -84,8 +83,8 @@ public class SearchServerTest { searchServer.start(); assertThat(searchServer.isUp()).isTrue(); - Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", CLUSTER_NAME).build(); - client = new TransportClient(settings) + Settings settings = Settings.builder().put("cluster.name", CLUSTER_NAME).build(); + client = TransportClient.builder().settings(settings).build() .addTransportAddress(new InetSocketTransportAddress(host, port)); assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN); diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java index 6131f87787b..526c3701302 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java @@ -116,7 +116,7 @@ public class IssueIndexBenchmarkTest { benchmark.expectAround("Throughput to index issues", throughputPerSecond, 6500, Benchmark.DEFAULT_ERROR_MARGIN_PERCENTS); // be sure that physical files do not evolve during estimation of size - tester.get(EsClient.class).prepareOptimize("issues").get(); + tester.get(EsClient.class).prepareForceMerge("issues").get(); long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()); LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize))); benchmark.expectBetween("ES dir size (b)", dirSize, 200L * FileUtils.ONE_MB, 420L * FileUtils.ONE_MB); diff --git a/server/sonar-server/pom.xml b/server/sonar-server/pom.xml index 4b4895dfdc3..61954cd6395 100644 --- a/server/sonar-server/pom.xml +++ b/server/sonar-server/pom.xml @@ -159,6 +159,17 @@ org.elasticsearch elasticsearch + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test + test-jar + + + net.java.dev.jna + jna + com.google.protobuf protobuf-java diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java index 02e17fd4beb..5267da37dd4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java @@ -39,6 +39,21 @@ public class ActivityDoc extends BaseDoc { super(new HashMap()); } + @Override + public String getId() { + return getKey(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + public void setCreatedAt(Date date) { setField(ActivityIndexDefinition.FIELD_CREATED_AT, date); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java index 4a1ff8ffe62..3e8042ce616 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java @@ -20,11 +20,12 @@ package org.sonar.server.activity.index; import com.google.common.base.Function; +import java.util.Date; +import java.util.Map; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.AndFilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.OrFilterBuilder; +import org.elasticsearch.index.query.AndQueryBuilder; +import org.elasticsearch.index.query.OrQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import org.sonar.core.util.NonNullInputFunction; @@ -33,9 +34,6 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.SearchResult; -import java.util.Date; -import java.util.Map; - public class ActivityIndex extends BaseIndex { /** @@ -67,35 +65,33 @@ public class ActivityIndex extends BaseIndex { requestBuilder.setSize(options.getLimit()); requestBuilder.addSort(ActivityIndexDefinition.FIELD_CREATED_AT, SortOrder.DESC); - AndFilterBuilder filter = FilterBuilders.andFilter(); + AndQueryBuilder filter = QueryBuilders.andQuery(); if (!query.getTypes().isEmpty()) { - OrFilterBuilder typeFilter = FilterBuilders.orFilter(); + OrQueryBuilder typeQuery = QueryBuilders.orQuery(); for (String type : query.getTypes()) { - typeFilter.add(FilterBuilders.termFilter(ActivityIndexDefinition.FIELD_TYPE, type)); + typeQuery.add(QueryBuilders.termQuery(ActivityIndexDefinition.FIELD_TYPE, type)); } - filter.add(typeFilter); + filter.add(typeQuery); } if (!query.getDataOrFilters().isEmpty()) { for (Map.Entry entry : query.getDataOrFilters().entrySet()) { - OrFilterBuilder orFilter = FilterBuilders.orFilter(); - orFilter.add(FilterBuilders.nestedFilter(ActivityIndexDefinition.FIELD_DETAILS, - FilterBuilders.termFilter(ActivityIndexDefinition.FIELD_DETAILS + "." + entry.getKey(), entry.getValue()))); - filter.add(orFilter); + OrQueryBuilder orQuery = QueryBuilders.orQuery(); + orQuery.add(QueryBuilders.nestedQuery(ActivityIndexDefinition.FIELD_DETAILS, + QueryBuilders.termQuery(ActivityIndexDefinition.FIELD_DETAILS + "." + entry.getKey(), entry.getValue()))); + filter.add(orQuery); } } Date since = query.getSince(); if (since != null) { - filter.add(FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT) - .gt(since) - .cache(false)); + filter.add(QueryBuilders.rangeQuery(ActivityIndexDefinition.FIELD_CREATED_AT) + .gt(since)); } Date to = query.getTo(); if (to != null) { - filter.add(FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT) - .lt(to) - .cache(false)); + filter.add(QueryBuilders.rangeQuery(ActivityIndexDefinition.FIELD_CREATED_AT) + .lt(to)); } requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)); diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java index 0ded443ffde..180ef978dfe 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java @@ -19,7 +19,6 @@ */ package org.sonar.server.activity.index; -import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -54,7 +53,6 @@ public class ActivityIndexDefinition implements IndexDefinition { // type "activity" NewIndex.NewIndexType mapping = index.createType(TYPE); - mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_KEY)); mapping.stringFieldBuilder(FIELD_KEY).disableNorms().build(); mapping.stringFieldBuilder(FIELD_TYPE).disableNorms().build(); mapping.stringFieldBuilder(FIELD_ACTION).disableNorms().build(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/component/ComponentImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/component/ComponentImpl.java index 32479e48960..6d26474b0cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/component/ComponentImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/component/ComponentImpl.java @@ -33,7 +33,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static java.util.Arrays.asList; -import static org.elasticsearch.common.lang3.StringUtils.trimToNull; +import static org.apache.commons.lang.StringUtils.trimToNull; @Immutable public class ComponentImpl implements Component { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java index 76f1f45aa0a..75b70050590 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java @@ -41,8 +41,8 @@ public class IndexPurgeListener implements PurgeListener { } @Override - public void onIssuesRemoval(List issueKeys) { - issueIndexer.deleteByKeys(issueKeys); + public void onIssuesRemoval(String projectUuid, List issueKeys) { + issueIndexer.deleteByKeys(projectUuid, issueKeys); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/issue/TrackerRawInputFactory.java b/server/sonar-server/src/main/java/org/sonar/server/computation/issue/TrackerRawInputFactory.java index d692dfd0cd3..bb81467ab44 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/issue/TrackerRawInputFactory.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/issue/TrackerRawInputFactory.java @@ -43,7 +43,7 @@ import org.sonar.server.computation.source.SourceLinesRepository; import org.sonar.server.rule.CommonRuleKeys; import static com.google.common.collect.Lists.newArrayList; -import static org.elasticsearch.common.lang3.StringUtils.isNotEmpty; +import static org.apache.commons.lang.StringUtils.isNotEmpty; public class TrackerRawInputFactory { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/issue/commonrule/CommonRule.java b/server/sonar-server/src/main/java/org/sonar/server/computation/issue/commonrule/CommonRule.java index 6ad789b6162..aac4481a0d7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/issue/commonrule/CommonRule.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/issue/commonrule/CommonRule.java @@ -21,13 +21,13 @@ package org.sonar.server.computation.issue.commonrule; import com.google.common.base.Optional; import javax.annotation.CheckForNull; -import org.elasticsearch.common.lang3.StringUtils; import org.sonar.api.rule.RuleKey; import org.sonar.core.issue.DefaultIssue; import org.sonar.server.computation.component.Component; import org.sonar.server.computation.qualityprofile.ActiveRule; import org.sonar.server.computation.qualityprofile.ActiveRulesHolder; +import static org.apache.commons.lang.StringUtils.isNotBlank; import static org.sonar.server.rule.CommonRuleKeys.commonRepositoryForLang; public abstract class CommonRule { @@ -80,7 +80,7 @@ public abstract class CommonRule { protected static double getMinDensityParam(ActiveRule activeRule, String paramKey) { String s = activeRule.getParams().get(paramKey); - if (StringUtils.isNoneBlank(s)) { + if (isNotBlank(s)) { double d = Double.parseDouble(s); if (d < 0.0 || d > 100.0) { throw new IllegalStateException(String.format("Minimum density of rule [%s] is incorrect. Got [%s] but must be between 0 and 100.", activeRule.getRuleKey(), s)); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/scm/ReportScmInfo.java b/server/sonar-server/src/main/java/org/sonar/server/computation/scm/ReportScmInfo.java index e9a5cf0c3dd..5863a7e5159 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/scm/ReportScmInfo.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/scm/ReportScmInfo.java @@ -32,7 +32,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.FluentIterable.from; import static java.util.Objects.requireNonNull; -import static org.elasticsearch.common.lang3.StringUtils.isNotEmpty; +import static org.apache.commons.lang.StringUtils.isNotEmpty; /** * ScmInfo implementation based on the changeset information from the Report diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadReportAnalysisMetadataHolderStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadReportAnalysisMetadataHolderStep.java index 0bffb7e5f2d..ec74ba5ff2b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadReportAnalysisMetadataHolderStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/LoadReportAnalysisMetadataHolderStep.java @@ -31,7 +31,7 @@ import org.sonar.server.computation.qualityprofile.QualityProfile; import static com.google.common.collect.Maps.transformValues; import static java.lang.String.format; -import static org.elasticsearch.common.lang3.StringUtils.isNotEmpty; +import static org.apache.commons.lang.StringUtils.isNotEmpty; /** * Feed analysis metadata holder with metadata from the analysis report. diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java index 4a1226f70a3..c4654208c2d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java @@ -46,6 +46,14 @@ public abstract class BaseDoc { return (String) fields.get("key"); } + public abstract String getId(); + + @CheckForNull + public abstract String getRouting(); + + @CheckForNull + public abstract String getParent(); + /** * Use this method when field value can be null. See warning in {@link #getField(String)} */ diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 6c404a5f1b2..86209815835 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -32,14 +32,16 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.sort.SortOrder; import org.picocontainer.Startable; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -144,8 +146,8 @@ public class BulkIndexer implements Startable { public void addDeletion(SearchRequestBuilder searchRequest) { searchRequest + .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(5)) - .setSearchType(SearchType.SCAN) .setSize(100) // load only doc ids, not _source fields .setFetchSource(false); @@ -157,14 +159,20 @@ public class BulkIndexer implements Startable { SearchResponse searchResponse = searchRequest.get(); while (true) { - searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) - .setScroll(TimeValue.timeValueMinutes(5)) - .get(); SearchHit[] hits = searchResponse.getHits().getHits(); for (SearchHit hit : hits) { - add(client.prepareDelete(hit.index(), hit.type(), hit.getId()).request()); + DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.index(), hit.type(), hit.getId()); + SearchHitField routing = hit.field("_routing"); + if (routing != null) { + deleteRequestBuilder.setRouting((String) routing.getValue()); + } + add(deleteRequestBuilder.request()); } + + String scrollId = searchResponse.getScrollId(); + searchResponse = client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get(); if (hits.length == 0) { + client.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break; } } @@ -204,7 +212,7 @@ public class BulkIndexer implements Startable { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ - client.prepareOptimize(indexName).get(); + client.prepareForceMerge(indexName).get(); updateSettings(largeInitialSettings); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java index 1432b5c5b6e..7e76057dc71 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java @@ -20,7 +20,7 @@ package org.sonar.server.es; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; class DefaultIndexSettings { @@ -28,8 +28,8 @@ class DefaultIndexSettings { // only static stuff } - static ImmutableSettings.Builder defaults() { - return ImmutableSettings.builder() + static Settings.Builder defaults() { + return Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.refresh_interval", "30s") diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 333427452da..e6c80612c57 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -19,9 +19,10 @@ */ package org.sonar.server.es; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; @@ -29,14 +30,13 @@ import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheReque import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.delete.DeleteRequestBuilder; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -44,10 +44,10 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -56,7 +56,6 @@ import org.picocontainer.Startable; import org.sonar.api.config.Settings; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.process.LoopbackAddress; import org.sonar.process.ProcessProperties; import org.sonar.server.es.request.ProxyBulkRequestBuilder; import org.sonar.server.es.request.ProxyClearCacheRequestBuilder; @@ -65,7 +64,6 @@ import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; import org.sonar.server.es.request.ProxyCountRequestBuilder; import org.sonar.server.es.request.ProxyCreateIndexRequestBuilder; -import org.sonar.server.es.request.ProxyDeleteByQueryRequestBuilder; import org.sonar.server.es.request.ProxyDeleteRequestBuilder; import org.sonar.server.es.request.ProxyFlushRequestBuilder; import org.sonar.server.es.request.ProxyGetRequestBuilder; @@ -79,6 +77,8 @@ import org.sonar.server.es.request.ProxyRefreshRequestBuilder; import org.sonar.server.es.request.ProxySearchRequestBuilder; import org.sonar.server.es.request.ProxySearchScrollRequestBuilder; +import static org.apache.commons.lang.StringUtils.defaultIfEmpty; + /** * Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions * with context) and profiling of requests. @@ -162,6 +162,10 @@ public class EsClient implements Startable { return new ProxyMultiGetRequestBuilder(nativeClient()); } + /** + * @deprecated use {@link #prepareSearch(String...)} with size 0 + */ + @Deprecated public CountRequestBuilder prepareCount(String... indices) { return new ProxyCountRequestBuilder(nativeClient()).setIndices(indices); } @@ -174,22 +178,13 @@ public class EsClient implements Startable { return new ProxyDeleteRequestBuilder(nativeClient(), index).setType(type).setId(id); } - /** - * @deprecated delete-by-query is dropped from ES 2.0 and should not be used. See - * https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-delete-by-query.html - */ - @Deprecated - public DeleteByQueryRequestBuilder prepareDeleteByQuery(String... indices) { - return new ProxyDeleteByQueryRequestBuilder(nativeClient()).setIndices(indices); - } - public IndexRequestBuilder prepareIndex(String index, String type) { return new ProxyIndexRequestBuilder(nativeClient()).setIndex(index).setType(type); } - public OptimizeRequestBuilder prepareOptimize(String indexName) { + public ForceMergeRequestBuilder prepareForceMerge(String indexName) { // TODO add proxy for profiling - return nativeClient().admin().indices().prepareOptimize(indexName) + return nativeClient().admin().indices().prepareForceMerge(indexName) .setMaxNumSegments(1); } @@ -212,17 +207,18 @@ public class EsClient implements Startable { public void start() { if (nativeClient == null) { ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); - org.elasticsearch.common.settings.Settings esSettings = ImmutableSettings.settingsBuilder() - .put("node.name", StringUtils.defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "sq_local_client")) - .put("network.bind_host", StringUtils.defaultIfEmpty(settings.getString(ProcessProperties.SEARCH_HOST), "localhost")) - .put("node.rack_id", StringUtils.defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "unknown")) + org.elasticsearch.common.settings.Settings esSettings = org.elasticsearch.common.settings.Settings.builder() + .put("node.name", defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "sq_local_client")) + .put("node.rack_id", defaultIfEmpty(settings.getString(ProcessProperties.CLUSTER_NODE_NAME), "unknown")) .put("cluster.name", StringUtils.defaultIfBlank(settings.getString(ProcessProperties.CLUSTER_NAME), "sonarqube")) .build(); - nativeClient = new TransportClient(esSettings); - ((TransportClient) nativeClient).addTransportAddress(new InetSocketTransportAddress(StringUtils.defaultIfEmpty(settings.getString(ProcessProperties.SEARCH_HOST), - LoopbackAddress.get() - .getHostAddress()), - settings.getInt(ProcessProperties.SEARCH_PORT))); + nativeClient = TransportClient.builder().settings(esSettings).build(); + String host = settings.getString(ProcessProperties.SEARCH_HOST); + try { + ((TransportClient) nativeClient).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), settings.getInt(ProcessProperties.SEARCH_PORT))); + } catch (UnknownHostException e) { + throw new IllegalStateException("Can not resolve host [" + host + "]", e); + } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java index 9c74c0e3a4a..25770e0d97e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java @@ -33,12 +33,16 @@ import java.util.NoSuchElementException; import java.util.Queue; import javax.annotation.CheckForNull; import javax.annotation.Nullable; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.joda.time.format.ISODateTimeFormat; + +import static java.lang.String.format; public class EsUtils { @@ -60,7 +64,7 @@ public class EsUtils { LinkedHashMap map = new LinkedHashMap<>(); List buckets = terms.getBuckets(); for (Terms.Bucket bucket : buckets) { - map.put(bucket.getKey(), bucket.getDocCount()); + map.put(bucket.getKeyAsString(), bucket.getDocCount()); } return map; } @@ -69,7 +73,7 @@ public class EsUtils { return Lists.transform(terms.getBuckets(), new Function() { @Override public String apply(Terms.Bucket bucket) { - return bucket.getKey(); + return bucket.getKeyAsString(); } }); } @@ -90,6 +94,15 @@ public class EsUtils { return null; } + public static BulkResponse executeBulkRequest(BulkRequestBuilder builder, String errorMessage, Object... errorMessageArgs) { + BulkResponse bulkResponse = builder.get(); + if (bulkResponse.hasFailures()) { + // do not use Preconditions as the message is expensive to generate (see buildFailureMessage()) + throw new IllegalStateException(format(errorMessage, errorMessageArgs) + ": " + bulkResponse.buildFailureMessage()); + } + return bulkResponse; + } + public static Iterator scroll(EsClient esClient, String scrollId, Function, D> docConverter) { return new DocScrollIterator<>(esClient, scrollId, docConverter); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java b/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java index c96248e65a5..3a87bd5f387 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java @@ -30,7 +30,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.HasAggregations; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.sum.Sum; @@ -64,8 +64,8 @@ public class Facets { processTermsAggregation((Terms) aggregation); } else if (HasAggregations.class.isAssignableFrom(aggregation.getClass())) { processSubAggregations((HasAggregations) aggregation); - } else if (DateHistogram.class.isAssignableFrom(aggregation.getClass())) { - processDateHistogram((DateHistogram) aggregation); + } else if (Histogram.class.isAssignableFrom(aggregation.getClass())) { + processDateHistogram((Histogram) aggregation); } else if (Sum.class.isAssignableFrom(aggregation.getClass())) { processSum((Sum) aggregation); } else { @@ -95,9 +95,9 @@ public class Facets { LinkedHashMap facet = getOrCreateFacet(facetName); for (Terms.Bucket value : aggregation.getBuckets()) { if (value.getAggregations().getAsMap().containsKey(FACET_MODE_EFFORT)) { - facet.put(value.getKey(), Math.round(((Sum) value.getAggregations().get(FACET_MODE_EFFORT)).getValue())); + facet.put(value.getKeyAsString(), Math.round(((Sum) value.getAggregations().get(FACET_MODE_EFFORT)).getValue())); } else { - facet.put(value.getKey(), value.getDocCount()); + facet.put(value.getKeyAsString(), value.getDocCount()); } } } @@ -108,13 +108,13 @@ public class Facets { } } - private void processDateHistogram(DateHistogram aggregation) { + private void processDateHistogram(Histogram aggregation) { LinkedHashMap facet = getOrCreateFacet(aggregation.getName()); - for (DateHistogram.Bucket value : aggregation.getBuckets()) { + for (Histogram.Bucket value : aggregation.getBuckets()) { if (value.getAggregations().getAsMap().containsKey(FACET_MODE_EFFORT)) { - facet.put(value.getKey(), Math.round(((Sum) value.getAggregations().get(FACET_MODE_EFFORT)).getValue())); + facet.put(value.getKeyAsString(), Math.round(((Sum) value.getAggregations().get(FACET_MODE_EFFORT)).getValue())); } else { - facet.put(value.getKey(), value.getDocCount()); + facet.put(value.getKeyAsString(), value.getDocCount()); } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java index 71a96209c30..e08494d9264 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java @@ -19,18 +19,17 @@ */ package org.sonar.server.es; +import java.util.Map; import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; import org.picocontainer.Startable; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import java.util.Map; - /** * Creates/deletes all indices in Elasticsearch during server startup. */ @@ -76,7 +75,7 @@ public class IndexCreator implements Startable { private void createIndex(IndexDefinitions.Index index) { LOGGER.info(String.format("Create index %s", index.getName())); - ImmutableSettings.Builder settings = ImmutableSettings.builder(); + Settings.Builder settings = Settings.builder(); settings.put(index.getSettings()); settings.put(SETTING_HASH, new IndexDefinitionHash().of(index)); CreateIndexResponse indexResponse = client @@ -93,7 +92,6 @@ public class IndexCreator implements Startable { LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey())); PutMappingResponse mappingResponse = client.preparePutMapping(index.getName()) .setType(entry.getKey()) - .setIgnoreConflicts(false) .setSource(entry.getValue().getAttributes()) .get(); if (!mappingResponse.isAcknowledged()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index e8a6260e69c..099098a4253 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -23,14 +23,14 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Maps; +import java.util.LinkedHashMap; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import javax.annotation.CheckForNull; import org.apache.commons.lang.StringUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.sonar.api.config.Settings; +import org.elasticsearch.common.settings.Settings; import org.sonar.process.ProcessProperties; import static java.lang.String.format; @@ -89,10 +89,6 @@ public class NewIndex { return new StringFieldBuilder(this, fieldName); } - public NestedObjectBuilder nestedObjectBuilder(String fieldName, NewIndexType nestedMapping) { - return new NestedObjectBuilder(this, nestedMapping, fieldName); - } - public NewIndexType createBooleanField(String fieldName) { return setProperty(fieldName, ImmutableMap.of("type", "boolean")); } @@ -142,23 +138,6 @@ public class NewIndex { } } - public static class NestedObjectBuilder { - private final NewIndexType indexType; - private final NewIndexType nestedType; - private final String fieldName; - - public NestedObjectBuilder(NewIndexType indexType, NewIndexType nestedType, String fieldName) { - this.indexType = indexType; - this.nestedType = nestedType; - this.fieldName = fieldName; - } - - public void build() { - nestedType.setAttribute("type", "nested"); - indexType.setProperty(fieldName, nestedType.attributes); - } - } - /** * Helper to define a string field in mapping of index type */ @@ -290,8 +269,8 @@ public class NewIndex { } private final String indexName; - private final ImmutableSettings.Builder settings = DefaultIndexSettings.defaults(); - private final SortedMap types = new TreeMap<>(); + private final Settings.Builder settings = DefaultIndexSettings.defaults(); + private final Map types = new LinkedHashMap<>(); NewIndex(String indexName) { Preconditions.checkArgument(StringUtils.isAllLowerCase(indexName), "Index name must be lower-case: " + indexName); @@ -302,7 +281,7 @@ public class NewIndex { return indexName; } - public ImmutableSettings.Builder getSettings() { + public Settings.Builder getSettings() { return settings; } @@ -312,11 +291,11 @@ public class NewIndex { return type; } - public SortedMap getTypes() { + public Map getTypes() { return types; } - public void configureShards(Settings settings) { + public void configureShards(org.sonar.api.config.Settings settings) { boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE); int shards = settings.getInt(format("sonar.search.%s.shards", indexName)); if (shards == 0) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java index b4d210936a0..4c0070d08ac 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/StickyFacetBuilder.java @@ -23,9 +23,7 @@ import com.google.common.base.Joiner; import java.util.Map; import javax.annotation.Nullable; import org.apache.commons.lang.ArrayUtils; -import org.elasticsearch.index.query.BoolFilterBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -35,22 +33,25 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; + public class StickyFacetBuilder { private static final int FACET_DEFAULT_MIN_DOC_COUNT = 1; private static final int FACET_DEFAULT_SIZE = 10; private static final Order FACET_DEFAULT_ORDER = Terms.Order.count(false); + private static final Joiner PIPE_JOINER = Joiner.on('|'); private final QueryBuilder query; - private final Map filters; + private final Map filters; private final AbstractAggregationBuilder subAggregation; private final Order order; - public StickyFacetBuilder(QueryBuilder query, Map filters) { + public StickyFacetBuilder(QueryBuilder query, Map filters) { this(query, filters, null, FACET_DEFAULT_ORDER); } - public StickyFacetBuilder(QueryBuilder query, Map filters, @Nullable AbstractAggregationBuilder subAggregation, @Nullable Order order) { + public StickyFacetBuilder(QueryBuilder query, Map filters, @Nullable AbstractAggregationBuilder subAggregation, @Nullable Order order) { this.query = query; this.filters = filters; this.subAggregation = subAggregation; @@ -61,7 +62,7 @@ public class StickyFacetBuilder { return query; } - public Map filters() { + public Map filters() { return filters; } @@ -70,7 +71,7 @@ public class StickyFacetBuilder { } public AggregationBuilder buildStickyFacet(String fieldName, String facetName, int size, Object... selected) { - BoolFilterBuilder facetFilter = getStickyFacetFilter(fieldName); + BoolQueryBuilder facetFilter = getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = buildTopFacetAggregation(fieldName, facetName, facetFilter, size); facetTopAggregation = addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, selected); @@ -79,9 +80,9 @@ public class StickyFacetBuilder { .subAggregation(facetTopAggregation); } - public BoolFilterBuilder getStickyFacetFilter(String... fieldNames) { - BoolFilterBuilder facetFilter = FilterBuilders.boolFilter().must(FilterBuilders.queryFilter(query)); - for (Map.Entry filter : filters.entrySet()) { + public BoolQueryBuilder getStickyFacetFilter(String... fieldNames) { + BoolQueryBuilder facetFilter = boolQuery().must(query); + for (Map.Entry filter : filters.entrySet()) { if (filter.getValue() != null && !ArrayUtils.contains(fieldNames, filter.getKey())) { facetFilter.must(filter.getValue()); } @@ -89,11 +90,10 @@ public class StickyFacetBuilder { return facetFilter; } - public FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolFilterBuilder facetFilter, int size) { + public FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolQueryBuilder facetFilter, int size) { TermsBuilder termsAggregation = AggregationBuilders.terms(facetName) .field(fieldName) .order(order) - // .order(Terms.Order.aggregation("debt", false)) .size(size) .minDocCount(FACET_DEFAULT_MIN_DOC_COUNT); if (subAggregation != null) { @@ -109,12 +109,11 @@ public class StickyFacetBuilder { if (selected.length > 0) { TermsBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") .field(fieldName) - .include(Joiner.on('|').join(selected)); + .include(PIPE_JOINER.join(selected)); if (subAggregation != null) { selectedTerms = selectedTerms.subAggregation(subAggregation); } - facetTopAggregation.subAggregation( - selectedTerms); + facetTopAggregation.subAggregation(selectedTerms); } return facetTopAggregation; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyBulkRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyBulkRequestBuilder.java index 108a3c2bf06..debb008dd67 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyBulkRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyBulkRequestBuilder.java @@ -21,8 +21,10 @@ package org.sonar.server.es.request; import com.google.common.collect.LinkedHashMultiset; import com.google.common.collect.Multiset; +import java.util.Set; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -33,12 +35,10 @@ import org.elasticsearch.common.unit.TimeValue; import org.sonar.api.utils.log.Profiler; import org.sonar.server.es.EsClient; -import java.util.Set; - public class ProxyBulkRequestBuilder extends BulkRequestBuilder { public ProxyBulkRequestBuilder(Client client) { - super(client); + super(client, BulkAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java index ab499353924..cbe465bc82a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyClearCacheRequestBuilder extends ClearIndicesCacheRequestBuilder { public ProxyClearCacheRequestBuilder(Client client) { - super(client.admin().indices()); + super(client.admin().indices(), ClearIndicesCacheAction.INSTANCE); } @Override @@ -74,18 +75,14 @@ public class ProxyClearCacheRequestBuilder extends ClearIndicesCacheRequestBuild if (fields != null && fields.length > 0) { message.append(String.format(" on fields '%s'", StringUtils.join(fields, ","))); } - String[] filterKeys = request.filterKeys(); - if (filterKeys != null && filterKeys.length > 0) { - message.append(String.format(" on filterKeys '%s'", StringUtils.join(filterKeys, ","))); - } - if (request.filterCache()) { + if (request.queryCache()) { message.append(" with filter cache"); } if (request.fieldDataCache()) { message.append(" with field data cache"); } - if (request.idCache()) { - message.append(" with id cache"); + if (request.requestCache()) { + message.append(" with request cache"); } return message.toString(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java index 9b73bdadfde..fc2b8586067 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyClusterHealthRequestBuilder extends ClusterHealthRequestBuilder { public ProxyClusterHealthRequestBuilder(Client client) { - super(client.admin().cluster()); + super(client.admin().cluster(), ClusterHealthAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java index 3ad580f7aca..3a785862a5a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyClusterStateRequestBuilder extends ClusterStateRequestBuilder { public ProxyClusterStateRequestBuilder(Client client) { - super(client.admin().cluster()); + super(client.admin().cluster(), ClusterStateAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java index a0ebe39b480..13416461edc 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyClusterStatsRequestBuilder extends ClusterStatsRequestBuilder { public ProxyClusterStatsRequestBuilder(Client client) { - super(client.admin().cluster()); + super(client.admin().cluster(), ClusterStatsAction.INSTANCE); } @Override @@ -67,7 +68,7 @@ public class ProxyClusterStatsRequestBuilder extends ClusterStatsRequestBuilder public String toString() { StringBuilder message = new StringBuilder(); message.append("ES cluster stats request"); - if (request.nodesIds().length > 0) { + if (request.nodesIds() != null) { message.append(String.format(" on nodes '%s'", StringUtils.join(request.nodesIds(), ","))); } return message.toString(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCountRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCountRequestBuilder.java index 627f395650d..9c36df38502 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCountRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCountRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.count.CountAction; import org.elasticsearch.action.count.CountRequestBuilder; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyCountRequestBuilder extends CountRequestBuilder { public ProxyCountRequestBuilder(Client client) { - super(client); + super(client, CountAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java index 2c7092d035c..8b1ca4d25cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java @@ -20,6 +20,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.client.Client; @@ -32,7 +33,7 @@ public class ProxyCreateIndexRequestBuilder extends CreateIndexRequestBuilder { private final String index; public ProxyCreateIndexRequestBuilder(Client client, String index) { - super(client.admin().indices(), index); + super(client.admin().indices(), CreateIndexAction.INSTANCE, index); this.index = index; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilder.java deleted file mode 100644 index d0b3b0c921c..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilder.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; -import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilder; -import org.sonar.api.utils.log.Profiler; -import org.sonar.server.es.EsClient; - -import java.io.IOException; - -public class ProxyDeleteByQueryRequestBuilder extends DeleteByQueryRequestBuilder { - - private QueryBuilder internalBuilder; - - public ProxyDeleteByQueryRequestBuilder(Client client) { - super(client); - } - - @Override - public DeleteByQueryResponse get() { - Profiler profiler = Profiler.createIfTrace(EsClient.LOGGER).start(); - try { - return super.execute().actionGet(); - } catch (Exception e) { - throw new IllegalStateException(String.format("Fail to execute %s", toString()), e); - } finally { - if (profiler.isTraceEnabled()) { - profiler.stopTrace(toString()); - } - } - } - - @Override - public DeleteByQueryResponse get(TimeValue timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public DeleteByQueryResponse get(String timeout) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - @Override - public ListenableActionFuture execute() { - throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); - } - - @Override - public DeleteByQueryRequestBuilder setQuery(QueryBuilder queryBuilder) { - this.internalBuilder = queryBuilder; - return super.setQuery(queryBuilder); - } - - @Override - public String toString() { - StringBuilder message = new StringBuilder(); - message.append(String.format("ES delete by query request '%s'", xContentToString(internalBuilder))); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); - } - return message.toString(); - } - - private String xContentToString(ToXContent toXContent) { - if (internalBuilder == null) { - return ""; - } - try { - XContentBuilder builder = XContentFactory.jsonBuilder(); - toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS); - return builder.string(); - } catch (IOException e) { - throw new IllegalStateException("Fail to convert request to string", e); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java index a7c2823e929..b27d7ccd62c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java @@ -20,6 +20,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.client.Client; @@ -30,7 +31,7 @@ import org.sonar.server.es.EsClient; public class ProxyDeleteRequestBuilder extends DeleteRequestBuilder { public ProxyDeleteRequestBuilder(Client client, String index) { - super(client, index); + super(client, DeleteAction.INSTANCE, index); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyFlushRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyFlushRequestBuilder.java index 83dc6f4f862..581740f66b3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyFlushRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyFlushRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.flush.FlushAction; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyFlushRequestBuilder extends FlushRequestBuilder { public ProxyFlushRequestBuilder(Client client) { - super(client.admin().indices()); + super(client.admin().indices(), FlushAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java index ddc7f302243..f0ca32b9dd2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java @@ -20,6 +20,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; @@ -30,7 +31,7 @@ import org.sonar.server.es.EsClient; public class ProxyGetRequestBuilder extends GetRequestBuilder { public ProxyGetRequestBuilder(Client client) { - super(client); + super(client, GetAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java index 8679031514b..9f952f873ed 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java @@ -20,6 +20,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; @@ -30,7 +31,7 @@ import org.sonar.server.es.EsClient; public class ProxyIndexRequestBuilder extends IndexRequestBuilder { public ProxyIndexRequestBuilder(Client client) { - super(client); + super(client, IndexAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java index b969f159b5c..6c4bec2e26f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyIndicesExistsRequestBuilder extends IndicesExistsRequestBuilder { public ProxyIndicesExistsRequestBuilder(Client client, String... indices) { - super(client.admin().indices(), indices); + super(client.admin().indices(), IndicesExistsAction.INSTANCE, indices); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java index 60928b1b7c2..993640422f2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyIndicesStatsRequestBuilder extends IndicesStatsRequestBuilder { public ProxyIndicesStatsRequestBuilder(Client client) { - super(client.admin().indices()); + super(client.admin().indices(), IndicesStatsAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilder.java index b0ca86f9f77..01b9bafe773 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyMultiGetRequestBuilder.java @@ -20,6 +20,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.get.MultiGetAction; import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyMultiGetRequestBuilder extends MultiGetRequestBuilder { public ProxyMultiGetRequestBuilder(Client client) { - super(client); + super(client, MultiGetAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java index 130b2f13502..9a65924cca2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyNodesStatsRequestBuilder extends NodesStatsRequestBuilder { public ProxyNodesStatsRequestBuilder(Client client) { - super(client.admin().cluster()); + super(client.admin().cluster(), NodesStatsAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java index 0861d115929..d05f1dd4f67 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyPutMappingRequestBuilder extends PutMappingRequestBuilder { public ProxyPutMappingRequestBuilder(Client client) { - super(client.admin().indices()); + super(client.admin().indices(), PutMappingAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java index 7f1d2d46049..abd5924333c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.client.Client; @@ -31,7 +32,7 @@ import org.sonar.server.es.EsClient; public class ProxyRefreshRequestBuilder extends RefreshRequestBuilder { public ProxyRefreshRequestBuilder(Client client) { - super(client.admin().indices()); + super(client.admin().indices(), RefreshAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java index 9ddd7fb3e3b..0129b5c4800 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java @@ -22,6 +22,7 @@ package org.sonar.server.es.request; import java.io.IOException; import java.util.Arrays; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -35,7 +36,7 @@ import org.sonar.server.es.EsClient; public class ProxySearchRequestBuilder extends SearchRequestBuilder { public ProxySearchRequestBuilder(Client client) { - super(client); + super(client, SearchAction.INSTANCE); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java index ca961b07e9b..f157bc040a0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java @@ -21,6 +21,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; @@ -30,7 +31,7 @@ import org.sonar.server.es.EsClient; public class ProxySearchScrollRequestBuilder extends SearchScrollRequestBuilder { public ProxySearchScrollRequestBuilder(String scrollId, Client client) { - super(client, scrollId); + super(client, SearchScrollAction.INSTANCE, scrollId); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationDoc.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationDoc.java new file mode 100644 index 00000000000..0ec8da3788a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationDoc.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import com.google.common.collect.Maps; +import org.sonar.server.es.BaseDoc; + +public class IssueAuthorizationDoc extends BaseDoc { + + public IssueAuthorizationDoc() { + super(Maps.newHashMap()); + } + + @Override + public String getId() { + return projectUuid(); + } + + @Override + public String getRouting() { + return projectUuid(); + } + + @Override + public String getParent() { + return null; + } + + public String projectUuid() { + return getField(IssueIndexDefinition.FIELD_AUTHORIZATION_PROJECT_UUID); + } + + public IssueAuthorizationDoc setProjectUuid(String s) { + setField(IssueIndexDefinition.FIELD_AUTHORIZATION_PROJECT_UUID, s); + return this; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java index dc13c48a25c..676942313f3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java @@ -87,6 +87,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer { esClient .prepareDelete(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, uuid) .setRefresh(refresh) + .setRouting(uuid) .get(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java index e43853c9134..8984c4e86cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -47,6 +47,21 @@ public class IssueDoc extends BaseDoc implements Issue { super(Maps.newHashMap()); } + @Override + public String getId() { + return key(); + } + + @Override + public String getRouting() { + return projectUuid(); + } + + @Override + public String getParent() { + return projectUuid(); + } + @Override public String key() { return getField(IssueIndexDefinition.FIELD_ISSUE_KEY); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 6b4f76a6064..6f16aab83b5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -45,10 +45,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolFilterBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.OrFilterBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -56,7 +53,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.global.GlobalBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; @@ -83,6 +80,11 @@ import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; import static com.google.common.collect.Lists.newArrayList; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonarqube.ws.client.issue.IssueFilterParameters.ASSIGNEES; import static org.sonarqube.ws.client.issue.IssueFilterParameters.AUTHORS; import static org.sonarqube.ws.client.issue.IssueFilterParameters.CREATED_AT; @@ -215,16 +217,16 @@ public class IssueIndex extends BaseIndex { configureSorting(query, requestBuilder); configurePagination(options, requestBuilder); - QueryBuilder esQuery = QueryBuilders.matchAllQuery(); - BoolFilterBuilder esFilter = FilterBuilders.boolFilter(); - Map filters = createFilters(query); - for (FilterBuilder filter : filters.values()) { + QueryBuilder esQuery = matchAllQuery(); + BoolQueryBuilder esFilter = boolQuery(); + Map filters = createFilters(query); + for (QueryBuilder filter : filters.values()) { if (filter != null) { esFilter.must(filter); } } if (esFilter.hasClauses()) { - requestBuilder.setQuery(QueryBuilders.filteredQuery(esQuery, esFilter)); + requestBuilder.setQuery(boolQuery().must(esQuery).filter(esFilter)); } else { requestBuilder.setQuery(esQuery); } @@ -243,27 +245,27 @@ public class IssueIndex extends BaseIndex { } } - protected void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { + private void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); } - private Map createFilters(IssueQuery query) { - Map filters = new HashMap<>(); + private Map createFilters(IssueQuery query) { + Map filters = new HashMap<>(); filters.put("__authorization", createAuthorizationFilter(query.checkAuthorization(), query.userLogin(), query.userGroups())); // Issue is assigned Filter if (BooleanUtils.isTrue(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, FilterBuilders.existsFilter(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE)); + filters.put(IS_ASSIGNED_FILTER, existsQuery(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE)); } else if (BooleanUtils.isFalse(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, FilterBuilders.missingFilter(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE)); + filters.put(IS_ASSIGNED_FILTER, boolQuery().mustNot(existsQuery(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE))); } // Issue is Resolved Filter String isResolved = "__isResolved"; if (BooleanUtils.isTrue(query.resolved())) { - filters.put(isResolved, FilterBuilders.existsFilter(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION)); + filters.put(isResolved, existsQuery(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION)); } else if (BooleanUtils.isFalse(query.resolved())) { - filters.put(isResolved, FilterBuilders.missingFilter(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION)); + filters.put(isResolved, boolQuery().mustNot(existsQuery(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION))); } // Field Filters @@ -286,14 +288,14 @@ public class IssueIndex extends BaseIndex { return filters; } - private void addComponentRelatedFilters(IssueQuery query, Map filters) { - FilterBuilder viewFilter = createViewFilter(query.viewUuids()); - FilterBuilder componentFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.componentUuids()); - FilterBuilder projectFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, query.projectUuids()); - FilterBuilder moduleRootFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids()); - FilterBuilder moduleFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, query.moduleUuids()); - FilterBuilder directoryFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH, query.directories()); - FilterBuilder fileFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.fileUuids()); + private void addComponentRelatedFilters(IssueQuery query, Map filters) { + QueryBuilder viewFilter = createViewFilter(query.viewUuids()); + QueryBuilder componentFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.componentUuids()); + QueryBuilder projectFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, query.projectUuids()); + QueryBuilder moduleRootFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids()); + QueryBuilder moduleFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, query.moduleUuids()); + QueryBuilder directoryFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH, query.directories()); + QueryBuilder fileFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.fileUuids()); if (BooleanUtils.isTrue(query.onComponentOnly())) { filters.put(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, componentFilter); @@ -312,68 +314,56 @@ public class IssueIndex extends BaseIndex { } @CheckForNull - private FilterBuilder createViewFilter(Collection viewUuids) { + private QueryBuilder createViewFilter(Collection viewUuids) { if (viewUuids.isEmpty()) { return null; } - OrFilterBuilder viewsFilter = FilterBuilders.orFilter(); + BoolQueryBuilder viewsFilter = boolQuery(); for (String viewUuid : viewUuids) { - viewsFilter.add(FilterBuilders.termsLookupFilter(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID) + viewsFilter.should(QueryBuilders.termsLookupQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID) .lookupIndex(ViewIndexDefinition.INDEX) .lookupType(ViewIndexDefinition.TYPE_VIEW) .lookupId(viewUuid) - .lookupPath(ViewIndexDefinition.FIELD_PROJECTS)) - .cacheKey(viewsLookupCacheKey(viewUuid)); + .lookupPath(ViewIndexDefinition.FIELD_PROJECTS)); } return viewsFilter; } - public static String viewsLookupCacheKey(String viewUuid) { - return String.format("%s%s%s", IssueIndexDefinition.TYPE_ISSUE, viewUuid, ViewIndexDefinition.TYPE_VIEW); - } - - private static FilterBuilder createAuthorizationFilter(boolean checkAuthorization, @Nullable String userLogin, Set userGroups) { + private static QueryBuilder createAuthorizationFilter(boolean checkAuthorization, @Nullable String userLogin, Set userGroups) { if (checkAuthorization) { - OrFilterBuilder groupsAndUser = FilterBuilders.orFilter(); + BoolQueryBuilder groupsAndUser = boolQuery(); if (userLogin != null) { - groupsAndUser.add(FilterBuilders.termFilter(IssueIndexDefinition.FIELD_AUTHORIZATION_USERS, userLogin)); + groupsAndUser.should(termQuery(IssueIndexDefinition.FIELD_AUTHORIZATION_USERS, userLogin)); } for (String group : userGroups) { - groupsAndUser.add(FilterBuilders.termFilter(IssueIndexDefinition.FIELD_AUTHORIZATION_GROUPS, group)); + groupsAndUser.should(termQuery(IssueIndexDefinition.FIELD_AUTHORIZATION_GROUPS, group)); } - return FilterBuilders.hasParentFilter(IssueIndexDefinition.TYPE_AUTHORIZATION, - QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.boolFilter() - .must(groupsAndUser) - .cache(true))); - } else { - return FilterBuilders.matchAllFilter(); + return QueryBuilders.hasParentQuery(IssueIndexDefinition.TYPE_AUTHORIZATION, + QueryBuilders.boolQuery().must(matchAllQuery()).filter(groupsAndUser)); } + return matchAllQuery(); } - private void addDatesFilter(Map filters, IssueQuery query) { + private void addDatesFilter(Map filters, IssueQuery query) { Date createdAfter = query.createdAfter(); Date createdBefore = query.createdBefore(); validateCreationDateBounds(createdBefore, createdAfter); if (createdAfter != null) { - filters.put("__createdAfter", FilterBuilders - .rangeFilter(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) - .gte(createdAfter) - .cache(false)); + filters.put("__createdAfter", QueryBuilders + .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) + .gte(createdAfter)); } if (createdBefore != null) { - filters.put("__createdBefore", FilterBuilders - .rangeFilter(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) - .lt(createdBefore) - .cache(false)); + filters.put("__createdBefore", QueryBuilders + .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) + .lt(createdBefore)); } Date createdAt = query.createdAt(); if (createdAt != null) { - filters.put("__createdAt", FilterBuilders.termFilter(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, createdAt).cache(false)); + filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, createdAt)); } } @@ -384,7 +374,7 @@ public class IssueIndex extends BaseIndex { "Start bound cannot be larger than end bound"); } - private void configureStickyFacets(IssueQuery query, SearchOptions options, Map filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) { + private void configureStickyFacets(IssueQuery query, SearchOptions options, Map filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) { if (!options.getFacets().isEmpty()) { StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, esQuery); // Execute Term aggregations @@ -431,7 +421,7 @@ public class IssueIndex extends BaseIndex { } } - private static StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map filters, QueryBuilder esQuery) { + private static StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map filters, QueryBuilder esQuery) { if (hasQueryEffortFacet(query)) { return new StickyFacetBuilder(esQuery, filters, EFFORT_AGGREGATION, EFFORT_AGGREGATION_ORDER); } @@ -456,10 +446,10 @@ public class IssueIndex extends BaseIndex { return FACET_MODE_EFFORT.equals(query.facetMode()) || DEPRECATED_FACET_MODE_DEBT.equals(query.facetMode()); } - private AggregationBuilder getCreatedAtFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { + private AggregationBuilder getCreatedAtFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { long now = system.now(); - DateHistogram.Interval bucketSize = DateHistogram.Interval.YEAR; + DateHistogramInterval bucketSize = DateHistogramInterval.YEAR; Date createdAfter = query.createdAfter(); long startTime = createdAfter == null ? getMinCreatedAt(filters, esQuery) : createdAfter.getTime(); Date createdBefore = query.createdBefore(); @@ -468,11 +458,11 @@ public class IssueIndex extends BaseIndex { Duration timeSpan = new Duration(startTime, endTime); if (timeSpan.isShorterThan(TWENTY_DAYS)) { - bucketSize = DateHistogram.Interval.DAY; + bucketSize = DateHistogramInterval.DAY; } else if (timeSpan.isShorterThan(TWENTY_WEEKS)) { - bucketSize = DateHistogram.Interval.WEEK; + bucketSize = DateHistogramInterval.WEEK; } else if (timeSpan.isShorterThan(TWENTY_MONTHS)) { - bucketSize = DateHistogram.Interval.MONTH; + bucketSize = DateHistogramInterval.MONTH; } // from GMT to server TZ @@ -491,14 +481,14 @@ public class IssueIndex extends BaseIndex { return dateHistogram; } - private long getMinCreatedAt(Map filters, QueryBuilder esQuery) { + private long getMinCreatedAt(Map filters, QueryBuilder esQuery) { String facetNameAndField = IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT; SearchRequestBuilder esRequest = getClient() .prepareSearch(IssueIndexDefinition.INDEX) .setTypes(IssueIndexDefinition.TYPE_ISSUE) .setSize(0); - BoolFilterBuilder esFilter = FilterBuilders.boolFilter(); - for (FilterBuilder filter : filters.values()) { + BoolQueryBuilder esFilter = boolQuery(); + for (QueryBuilder filter : filters.values()) { if (filter != null) { esFilter.must(filter); } @@ -518,16 +508,16 @@ public class IssueIndex extends BaseIndex { } } - private AggregationBuilder createAssigneesFacet(IssueQuery query, Map filters, QueryBuilder queryBuilder) { + private AggregationBuilder createAssigneesFacet(IssueQuery query, Map filters, QueryBuilder queryBuilder) { String fieldName = IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE; String facetName = ASSIGNEES; // Same as in super.stickyFacetBuilder - Map assigneeFilters = Maps.newHashMap(filters); + Map assigneeFilters = Maps.newHashMap(filters); assigneeFilters.remove(IS_ASSIGNED_FILTER); assigneeFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder); - BoolFilterBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); + BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE); Collection assigneesEscaped = escapeValuesForFacetInclusion(query.assignees()); @@ -555,7 +545,7 @@ public class IssueIndex extends BaseIndex { }); } - private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map filters, QueryBuilder queryBuilder) { + private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map filters, QueryBuilder queryBuilder) { String login = userSession.getLogin(); if (!options.getFacets().contains(FACET_ASSIGNED_TO_ME) || StringUtils.isEmpty(login)) { @@ -567,7 +557,7 @@ public class IssueIndex extends BaseIndex { // Same as in super.stickyFacetBuilder StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); - BoolFilterBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); + BoolQueryBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); FilterAggregationBuilder facetTopAggregation = AggregationBuilders .filter(facetName + "__filter") @@ -579,16 +569,16 @@ public class IssueIndex extends BaseIndex { .subAggregation(facetTopAggregation)); } - private static AggregationBuilder createResolutionFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { + private static AggregationBuilder createResolutionFacet(IssueQuery query, Map filters, QueryBuilder esQuery) { String fieldName = IssueIndexDefinition.FIELD_ISSUE_RESOLUTION; String facetName = RESOLUTIONS; // Same as in super.stickyFacetBuilder - Map resolutionFilters = Maps.newHashMap(filters); + Map resolutionFilters = Maps.newHashMap(filters); resolutionFilters.remove("__isResolved"); resolutionFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery); - BoolFilterBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); + BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE); facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation); @@ -604,9 +594,9 @@ public class IssueIndex extends BaseIndex { } @CheckForNull - private FilterBuilder createTermsFilter(String field, Collection values) { + private QueryBuilder createTermsFilter(String field, Collection values) { if (!values.isEmpty()) { - return FilterBuilders.termsFilter(field, values); + return termsQuery(field, values); } else { return null; } @@ -617,8 +607,7 @@ public class IssueIndex extends BaseIndex { .prepareSearch(IssueIndexDefinition.INDEX, RuleIndexDefinition.INDEX) .setTypes(IssueIndexDefinition.TYPE_ISSUE, RuleIndexDefinition.TYPE_RULE); - requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), - createBoolFilter(query))); + requestBuilder.setQuery(boolQuery().must(matchAllQuery()).filter(createBoolFilter(query))); GlobalBuilder topAggreg = AggregationBuilders.global("tags"); String tagsOnIssuesSubAggregation = "tags__issues"; @@ -669,8 +658,7 @@ public class IssueIndex extends BaseIndex { .setSize(0) .setTypes(IssueIndexDefinition.TYPE_ISSUE); - requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), - createBoolFilter(query))); + requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); TermsBuilder aggreg = AggregationBuilders.terms("_ref") .field(fieldName) @@ -685,15 +673,14 @@ public class IssueIndex extends BaseIndex { return searchResponse.getAggregations().get("_ref"); } - private BoolFilterBuilder createBoolFilter(IssueQuery query) { - BoolFilterBuilder boolFilter = FilterBuilders.boolFilter(); - for (FilterBuilder filter : createFilters(query).values()) { - // TODO Can it be null ? + private BoolQueryBuilder createBoolFilter(IssueQuery query) { + BoolQueryBuilder boolQuery = boolQuery(); + for (QueryBuilder filter : createFilters(query).values()) { if (filter != null) { - boolFilter.must(filter); + boolQuery.must(filter); } } - return boolFilter; + return boolQuery; } /** @@ -701,16 +688,16 @@ public class IssueIndex extends BaseIndex { * Only fields needed for the batch are returned. */ public Iterator selectIssuesForBatch(ComponentDto component) { - BoolFilterBuilder filter = FilterBuilders.boolFilter() + BoolQueryBuilder filter = boolQuery() .must(createAuthorizationFilter(true, userSession.getLogin(), userSession.getUserGroups())) - .mustNot(FilterBuilders.termsFilter(IssueIndexDefinition.FIELD_ISSUE_STATUS, Issue.STATUS_CLOSED)); + .mustNot(termsQuery(IssueIndexDefinition.FIELD_ISSUE_STATUS, Issue.STATUS_CLOSED)); switch (component.scope()) { case Scopes.PROJECT: - filter.must(FilterBuilders.termsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, component.uuid())); + filter.must(termsQuery(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, component.uuid())); break; case Scopes.FILE: - filter.must(FilterBuilders.termsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, component.uuid())); + filter.must(termsQuery(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, component.uuid())); break; default: throw new IllegalStateException(String.format("Component of scope '%s' is not allowed", component.scope())); @@ -721,7 +708,7 @@ public class IssueIndex extends BaseIndex { .setTypes(IssueIndexDefinition.TYPE_ISSUE) .setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueMinutes(EsUtils.SCROLL_TIME_IN_MINUTES)) - .setSize(10000) + .setSize(10_000) .setFetchSource( new String[] {IssueIndexDefinition.FIELD_ISSUE_KEY, IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, IssueIndexDefinition.FIELD_ISSUE_FILE_PATH, IssueIndexDefinition.FIELD_ISSUE_SEVERITY, IssueIndexDefinition.FIELD_ISSUE_MANUAL_SEVERITY, @@ -729,7 +716,7 @@ public class IssueIndex extends BaseIndex { IssueIndexDefinition.FIELD_ISSUE_LINE, IssueIndexDefinition.FIELD_ISSUE_MESSAGE, IssueIndexDefinition.FIELD_ISSUE_CHECKSUM, IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT}, null) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)); + .setQuery(boolQuery().must(matchAllQuery()).filter(filter)); SearchResponse response = requestBuilder.get(); return EsUtils.scroll(getClient(), response.getScrollId(), DOC_CONVERTER); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index d0f4ccc40b9..f26a51b6431 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -93,19 +93,10 @@ public class IssueIndexDefinition implements IndexDefinition { index.refreshHandledByIndexer(); index.configureShards(settings); - // type "authorization" - NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION); - authorizationMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_AUTHORIZATION_PROJECT_UUID)); - authorizationMapping.createDateTimeField(FIELD_AUTHORIZATION_UPDATED_AT); - authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_PROJECT_UUID).disableNorms().build(); - authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_GROUPS).disableNorms().build(); - authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_USERS).disableNorms().build(); - // type "issue" NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE); - issueMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_ISSUE_KEY)); issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION)); - issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_ISSUE_PROJECT_UUID)); + issueMapping.setAttribute("_routing", ImmutableMap.of("required", "true")); issueMapping.stringFieldBuilder(FIELD_ISSUE_ASSIGNEE).enableSorting().build(); issueMapping.stringFieldBuilder(FIELD_ISSUE_ATTRIBUTES).docValues().disableNorms().disableSearch().build(); issueMapping.stringFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().docValues().build(); @@ -132,5 +123,13 @@ public class IssueIndexDefinition implements IndexDefinition { issueMapping.stringFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build(); issueMapping.createDateTimeField(FIELD_ISSUE_TECHNICAL_UPDATED_AT); issueMapping.stringFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build(); + + // type "authorization" + NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION); + authorizationMapping.setAttribute("_routing", ImmutableMap.of("required", "true")); + authorizationMapping.createDateTimeField(FIELD_AUTHORIZATION_UPDATED_AT); + authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_PROJECT_UUID).disableNorms().build(); + authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_GROUPS).disableNorms().build(); + authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_USERS).disableNorms().build(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 97e747ec8a7..d0e821c4f11 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -25,21 +25,24 @@ import javax.annotation.Nullable; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.QueryBuilders; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.server.es.BaseIndexer; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; +import org.sonar.server.es.EsUtils; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT; import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_AUTHORIZATION; import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; public class IssueIndexer extends BaseIndexer { + private static final String DELETE_ERROR_MESSAGE = "Fail to delete some issues of project [%s]"; private static final int MAX_BATCH_SIZE = 1000; private final DbClient dbClient; @@ -106,16 +109,14 @@ public class IssueIndexer extends BaseIndexer { bulk.setDisableRefresh(false); bulk.start(); SearchRequestBuilder search = esClient.prepareSearch(INDEX) + .setTypes(TYPE_ISSUE, TYPE_AUTHORIZATION) .setRouting(uuid) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(FIELD_ISSUE_PROJECT_UUID, uuid)) - )); + .setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, uuid))); bulk.addDeletion(search); bulk.stop(); } - public void deleteByKeys(List issueKeys){ + public void deleteByKeys(String projectUuid, List issueKeys) { if (issueKeys.isEmpty()) { return; } @@ -123,15 +124,17 @@ public class IssueIndexer extends BaseIndexer { int count = 0; BulkRequestBuilder builder = esClient.prepareBulk(); for (String issueKey : issueKeys) { - builder.add(esClient.prepareDelete(INDEX, TYPE_ISSUE, issueKey)); + builder.add(esClient.prepareDelete(INDEX, TYPE_ISSUE, issueKey) + .setRefresh(false) + .setRouting(projectUuid)); count++; if (count >= MAX_BATCH_SIZE) { - builder.get(); + EsUtils.executeBulkRequest(builder, DELETE_ERROR_MESSAGE, projectUuid); builder = esClient.prepareBulk(); count = 0; } } - builder.get(); + EsUtils.executeBulkRequest(builder, DELETE_ERROR_MESSAGE, projectUuid); esClient.prepareRefresh(INDEX).get(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java index d6b55b2d573..7850e8e0ee9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java @@ -24,7 +24,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; import org.apache.commons.dbutils.DbUtils; -import org.elasticsearch.action.support.IndicesOptions; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Loggers; import org.sonar.db.DbSession; @@ -163,10 +162,7 @@ public class BackendCleanup { * Completely remove a index with all types */ public void clearIndex(String indexName) { - String[] indicesToClear = esClient.prepareState().get().getState().getMetaData().concreteIndices(IndicesOptions.strictExpand(), indexName); - for (String index : indicesToClear) { - BulkIndexer.delete(esClient, index, esClient.prepareSearch(index).setQuery(matchAllQuery())); - } + BulkIndexer.delete(esClient, indexName, esClient.prepareSearch(indexName).setQuery(matchAllQuery())); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java b/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java index 7f37d2b0f31..69f757c0f23 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/monitoring/EsMonitor.java @@ -21,12 +21,12 @@ package org.sonar.server.platform.monitoring; import java.util.LinkedHashMap; import java.util.Map; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.breaker.CircuitBreaker; import org.sonar.server.es.EsClient; @@ -105,19 +105,18 @@ public class EsMonitor extends BaseMonitorMBean implements EsMonitorMBean { nodeAttributes.put("Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().bytes())); nodeAttributes.put("Store Size", byteCountToDisplaySize(stats.getIndices().getStore().getSizeInBytes())); nodeAttributes.put("Open Files", stats.getProcess().getOpenFileDescriptors()); - nodeAttributes.put("JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPrecent())); + nodeAttributes.put("JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPercent())); nodeAttributes.put("JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().bytes())); nodeAttributes.put("JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().bytes())); nodeAttributes.put("JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().bytes())); - nodeAttributes.put("JVM Threads", stats.getJvm().getThreads().count()); + nodeAttributes.put("JVM Threads", stats.getJvm().getThreads().getCount()); nodeAttributes.put("Field Data Memory", byteCountToDisplaySize(stats.getIndices().getFieldData().getMemorySizeInBytes())); - nodeAttributes.put("Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getLimit())); - nodeAttributes.put("Field Data Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated())); - nodeAttributes.put("Request Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.Name.REQUEST).getLimit())); - nodeAttributes.put("Request Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.Name.REQUEST).getEstimated())); - nodeAttributes.put("Filter Cache Memory", byteCountToDisplaySize(stats.getIndices().getFilterCache().getMemorySizeInBytes())); + nodeAttributes.put("Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit())); + nodeAttributes.put("Field Data Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated())); + nodeAttributes.put("Request Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit())); + nodeAttributes.put("Request Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getEstimated())); nodeAttributes.put("Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes())); - nodeAttributes.put("ID Cache Memory", byteCountToDisplaySize(stats.getIndices().getIdCache().getMemorySizeInBytes())); + nodeAttributes.put("Request Cache Memory", byteCountToDisplaySize(stats.getIndices().getRequestCache().getMemorySizeInBytes())); nodeAttributes.put("Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes())); } return nodes; diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java index bfebf07fd31..11b65c2c9cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java @@ -50,6 +50,21 @@ public class ActiveRuleDoc extends BaseDoc { setField(FIELD_ACTIVE_RULE_REPOSITORY, key.ruleKey().repository()); } + @Override + public String getId() { + return key().toString(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return key.ruleKey().toString(); + } + public ActiveRuleKey key() { return key; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index 1e7c9c16f56..9a8b2a84372 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -27,8 +27,7 @@ import java.util.Map; import javax.annotation.Nullable; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -64,12 +63,12 @@ public class ActiveRuleIndex extends BaseIndex { public Map countAllByQualityProfileKey() { return countByField(FIELD_ACTIVE_RULE_PROFILE_KEY, - FilterBuilders.hasParentFilter(TYPE_RULE, - FilterBuilders.notFilter( - FilterBuilders.termFilter(FIELD_RULE_STATUS, "REMOVED")))); + QueryBuilders.hasParentQuery(TYPE_RULE, + QueryBuilders.boolQuery().mustNot( + QueryBuilders.termQuery(FIELD_RULE_STATUS, "REMOVED")))); } - private Map countByField(String indexField, FilterBuilder filter) { + private Map countByField(String indexField, QueryBuilder filter) { Map counts = new HashMap<>(); SearchRequestBuilder request = getClient().prepareSearch(INDEX) @@ -90,18 +89,17 @@ public class ActiveRuleIndex extends BaseIndex { Terms values = response.getAggregations().get(indexField); for (Terms.Bucket value : values.getBuckets()) { - counts.put(value.getKey(), value.getDocCount()); + counts.put(value.getKeyAsString(), value.getDocCount()); } return counts; } public Map> getStatsByProfileKeys(List keys) { SearchRequestBuilder request = getClient().prepareSearch(INDEX) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.termsQuery(FIELD_ACTIVE_RULE_PROFILE_KEY, keys), - FilterBuilders.boolFilter() - .mustNot(FilterBuilders.hasParentFilter(TYPE_RULE, - FilterBuilders.termFilter(FIELD_RULE_STATUS, RuleStatus.REMOVED.name()))))) + .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(FIELD_ACTIVE_RULE_PROFILE_KEY, keys)).filter( + QueryBuilders.boolQuery() + .mustNot(QueryBuilders.hasParentQuery(TYPE_RULE, + QueryBuilders.termQuery(FIELD_RULE_STATUS, RuleStatus.REMOVED.name()))))) .addAggregation(AggregationBuilders.terms(FIELD_ACTIVE_RULE_PROFILE_KEY) .field(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).size(0) .subAggregation(AggregationBuilders.terms(FIELD_ACTIVE_RULE_INHERITANCE) @@ -115,7 +113,7 @@ public class ActiveRuleIndex extends BaseIndex { Map> stats = new HashMap<>(); Aggregation aggregation = response.getAggregations().get(FIELD_ACTIVE_RULE_PROFILE_KEY); for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { - stats.put(value.getKey(), processAggregations(value.getAggregations())); + stats.put(value.getKeyAsString(), processAggregations(value.getAggregations())); } return stats; @@ -127,7 +125,7 @@ public class ActiveRuleIndex extends BaseIndex { for (Aggregation aggregation : aggregations.asList()) { if (aggregation instanceof StringTerms) { for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { - FacetValue facetValue = new FacetValue(value.getKey(), value.getDocCount()); + FacetValue facetValue = new FacetValue(value.getKeyAsString(), value.getDocCount()); stats.put(aggregation.getName(), facetValue); } } else if (aggregation instanceof InternalValueCount) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 4337e9eaad2..29d1f9a5901 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -27,7 +27,6 @@ import java.util.List; import javax.annotation.Nonnull; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -36,8 +35,10 @@ import org.sonar.server.es.BaseIndexer; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; import org.sonar.server.qualityprofile.ActiveRuleChange; -import org.sonar.server.rule.index.RuleIndexDefinition; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_UPDATED_AT; import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX; import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; @@ -100,10 +101,7 @@ public class ActiveRuleIndexer extends BaseIndexer { bulk.start(); SearchRequestBuilder search = esClient.prepareSearch(INDEX) .setTypes(TYPE_ACTIVE_RULE) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY, qualityProfileKey)) - )); + .setQuery(QueryBuilders.boolQuery().must(termsQuery(FIELD_ACTIVE_RULE_PROFILE_KEY, qualityProfileKey))); bulk.addDeletion(search); bulk.stop(); } @@ -113,10 +111,7 @@ public class ActiveRuleIndexer extends BaseIndexer { bulk.start(); SearchRequestBuilder search = esClient.prepareSearch(INDEX) .setTypes(TYPE_ACTIVE_RULE) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY, keys)) - )); + .setQuery(QueryBuilders.boolQuery().must(termsQuery(FIELD_ACTIVE_RULE_KEY, keys))); bulk.addDeletion(search); bulk.stop(); } @@ -130,7 +125,6 @@ public class ActiveRuleIndexer extends BaseIndexer { private static IndexRequest newIndexRequest(ActiveRuleDoc doc) { return new IndexRequest(INDEX, TYPE_ACTIVE_RULE, doc.key().toString()) .parent(doc.key().ruleKey().toString()) - .routing(doc.key().ruleKey().repository()) .source(doc.getFields()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java index 0f20bb301bb..c29f1d92ed3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java @@ -19,6 +19,7 @@ */ package org.sonar.server.rule.index; +import com.google.common.collect.Maps; import java.util.Collection; import java.util.Map; import javax.annotation.CheckForNull; @@ -38,6 +39,25 @@ public class RuleDoc extends BaseDoc { super(fields); } + public RuleDoc() { + super(Maps.newHashMap()); + } + + @Override + public String getId() { + return key().toString(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + public RuleKey key() { return RuleKey.parse(this.getField(RuleIndexDefinition.FIELD_RULE_KEY)); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index ae5c60bd87d..d7fe053024f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -40,11 +40,8 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolFilterBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.HasParentFilterBuilder; +import org.elasticsearch.index.query.HasParentQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -66,6 +63,7 @@ import org.sonar.server.es.SearchIdResult; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.StickyFacetBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES; @@ -120,7 +118,7 @@ public class RuleIndex extends BaseIndex { .setTypes(TYPE_RULE); QueryBuilder qb = buildQuery(query); - Map filters = buildFilters(query); + Map filters = buildFilters(query); if (!options.getFacets().isEmpty()) { for (AggregationBuilder aggregation : getFacets(query, options, qb, filters).values()) { @@ -131,12 +129,12 @@ public class RuleIndex extends BaseIndex { setSorting(query, esSearch); setPagination(options, esSearch); - BoolFilterBuilder fb = FilterBuilders.boolFilter(); - for (FilterBuilder filterBuilder : filters.values()) { + BoolQueryBuilder fb = boolQuery(); + for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } - esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); + esSearch.setQuery(boolQuery().must(qb).filter(fb)); return new SearchIdResult<>(esSearch.get(), ToRuleKey.INSTANCE); } @@ -151,15 +149,15 @@ public class RuleIndex extends BaseIndex { .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); QueryBuilder qb = buildQuery(query); - Map filters = buildFilters(query); + Map filters = buildFilters(query); setSorting(query, esSearch); - BoolFilterBuilder fb = FilterBuilders.boolFilter(); - for (FilterBuilder filterBuilder : filters.values()) { + BoolQueryBuilder fb = boolQuery(); + for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } - esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); + esSearch.setQuery(boolQuery().must(qb).filter(fb)); SearchResponse response = esSearch.get(); return scrollIds(getClient(), response.getScrollId(), ToRuleKey.INSTANCE); } @@ -174,7 +172,7 @@ public class RuleIndex extends BaseIndex { } // Build RuleBased contextual query - BoolQueryBuilder qb = QueryBuilders.boolQuery(); + BoolQueryBuilder qb = boolQuery(); String queryString = query.getQueryText(); // Human readable type of querying @@ -210,58 +208,58 @@ public class RuleIndex extends BaseIndex { } /* Build main filter (match based) */ - private static Map buildFilters(RuleQuery query) { + private static Map buildFilters(RuleQuery query) { - Map filters = new HashMap<>(); + Map filters = new HashMap<>(); /* Add enforced filter on rules that are REMOVED */ filters.put(FIELD_RULE_STATUS, - FilterBuilders.boolFilter().mustNot( - FilterBuilders.termFilter(FIELD_RULE_STATUS, + boolQuery().mustNot( + QueryBuilders.termQuery(FIELD_RULE_STATUS, RuleStatus.REMOVED.toString()))); if (StringUtils.isNotEmpty(query.getInternalKey())) { filters.put(FIELD_RULE_INTERNAL_KEY, - FilterBuilders.termFilter(FIELD_RULE_INTERNAL_KEY, query.getInternalKey())); + QueryBuilders.termQuery(FIELD_RULE_INTERNAL_KEY, query.getInternalKey())); } if (StringUtils.isNotEmpty(query.getRuleKey())) { filters.put(FIELD_RULE_RULE_KEY, - FilterBuilders.termFilter(FIELD_RULE_RULE_KEY, query.getRuleKey())); + QueryBuilders.termQuery(FIELD_RULE_RULE_KEY, query.getRuleKey())); } if (isNotEmpty(query.getLanguages())) { filters.put(FIELD_RULE_LANGUAGE, - FilterBuilders.termsFilter(FIELD_RULE_LANGUAGE, query.getLanguages())); + QueryBuilders.termsQuery(FIELD_RULE_LANGUAGE, query.getLanguages())); } if (isNotEmpty(query.getRepositories())) { filters.put(FIELD_RULE_REPOSITORY, - FilterBuilders.termsFilter(FIELD_RULE_REPOSITORY, query.getRepositories())); + QueryBuilders.termsQuery(FIELD_RULE_REPOSITORY, query.getRepositories())); } if (isNotEmpty(query.getSeverities())) { filters.put(FIELD_RULE_SEVERITY, - FilterBuilders.termsFilter(FIELD_RULE_SEVERITY, query.getSeverities())); + QueryBuilders.termsQuery(FIELD_RULE_SEVERITY, query.getSeverities())); } if (StringUtils.isNotEmpty(query.getKey())) { filters.put(FIELD_RULE_KEY, - FilterBuilders.termFilter(FIELD_RULE_KEY, query.getKey())); + QueryBuilders.termQuery(FIELD_RULE_KEY, query.getKey())); } if (isNotEmpty(query.getTags())) { filters.put(FIELD_RULE_ALL_TAGS, - FilterBuilders.termsFilter(FIELD_RULE_ALL_TAGS, query.getTags())); + QueryBuilders.termsQuery(FIELD_RULE_ALL_TAGS, query.getTags())); } if (isNotEmpty(query.getTypes())) { filters.put(FIELD_RULE_TYPE, - FilterBuilders.termsFilter(FIELD_RULE_TYPE, query.getTypes())); + QueryBuilders.termsQuery(FIELD_RULE_TYPE, query.getTypes())); } if (query.getAvailableSinceLong() != null) { - filters.put("availableSince", FilterBuilders.rangeFilter(FIELD_RULE_CREATED_AT) + filters.put("availableSince", QueryBuilders.rangeQuery(FIELD_RULE_CREATED_AT) .gte(query.getAvailableSinceLong())); } @@ -271,55 +269,55 @@ public class RuleIndex extends BaseIndex { stringStatus.add(status.name()); } filters.put(FIELD_RULE_STATUS, - FilterBuilders.termsFilter(FIELD_RULE_STATUS, stringStatus)); + QueryBuilders.termsQuery(FIELD_RULE_STATUS, stringStatus)); } Boolean isTemplate = query.isTemplate(); if (isTemplate != null) { filters.put(FIELD_RULE_IS_TEMPLATE, - FilterBuilders.termFilter(FIELD_RULE_IS_TEMPLATE, Boolean.toString(isTemplate))); + QueryBuilders.termQuery(FIELD_RULE_IS_TEMPLATE, Boolean.toString(isTemplate))); } String template = query.templateKey(); if (template != null) { filters.put(FIELD_RULE_TEMPLATE_KEY, - FilterBuilders.termFilter(FIELD_RULE_TEMPLATE_KEY, template)); + QueryBuilders.termQuery(FIELD_RULE_TEMPLATE_KEY, template)); } // ActiveRule Filter (profile and inheritance) - BoolFilterBuilder childrenFilter = FilterBuilders.boolFilter(); + BoolQueryBuilder childrenFilter = boolQuery(); addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_PROFILE_KEY, query.getQProfileKey()); addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance()); addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_SEVERITY, query.getActiveSeverities()); // ChildQuery - FilterBuilder childQuery; + QueryBuilder childQuery; if (childrenFilter.hasClauses()) { childQuery = childrenFilter; } else { - childQuery = FilterBuilders.matchAllFilter(); + childQuery = QueryBuilders.matchAllQuery(); } /** Implementation of activation query */ if (Boolean.TRUE.equals(query.getActivation())) { filters.put("activation", - FilterBuilders.hasChildFilter(TYPE_ACTIVE_RULE, + QueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE, childQuery)); } else if (Boolean.FALSE.equals(query.getActivation())) { filters.put("activation", - FilterBuilders.boolFilter().mustNot( - FilterBuilders.hasChildFilter(TYPE_ACTIVE_RULE, + boolQuery().mustNot( + QueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE, childQuery))); } return filters; } - private static BoolFilterBuilder addTermFilter(BoolFilterBuilder filter, String field, @Nullable Collection values) { + private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable Collection values) { if (isNotEmpty(values)) { - BoolFilterBuilder valuesFilter = FilterBuilders.boolFilter(); + BoolQueryBuilder valuesFilter = boolQuery(); for (String value : values) { - FilterBuilder valueFilter = FilterBuilders.termFilter(field, value); + QueryBuilder valueFilter = QueryBuilders.termQuery(field, value); valuesFilter.should(valueFilter); } filter.must(valuesFilter); @@ -327,14 +325,14 @@ public class RuleIndex extends BaseIndex { return filter; } - private static BoolFilterBuilder addTermFilter(BoolFilterBuilder filter, String field, @Nullable String value) { + private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable String value) { if (StringUtils.isNotEmpty(value)) { - filter.must(FilterBuilders.termFilter(field, value)); + filter.must(QueryBuilders.termQuery(field, value)); } return filter; } - private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { + private static Map getFacets(RuleQuery query, SearchOptions options, QueryBuilder queryBuilder, Map filters) { Map aggregations = new HashMap<>(); StickyFacetBuilder stickyFacetBuilder = stickyFacetBuilder(queryBuilder, filters); @@ -380,7 +378,7 @@ public class RuleIndex extends BaseIndex { private static void addStatusFacetIfNeeded(SearchOptions options, Map aggregations, StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_STATUSES)) { - BoolFilterBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS); + BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS); AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter") .filter(facetFilter) .subAggregation( @@ -400,15 +398,15 @@ public class RuleIndex extends BaseIndex { // We are building a children aggregation on active rules // so the rule filter has to be used as parent filter for active rules // from which we remove filters that concern active rules ("activation") - HasParentFilterBuilder ruleFilter = FilterBuilders.hasParentFilter( + HasParentQueryBuilder ruleFilter = QueryBuilders.hasParentQuery( TYPE_RULE, stickyFacetBuilder.getStickyFacetFilter("activation")); // Rebuilding the active rule filter without severities - BoolFilterBuilder childrenFilter = FilterBuilders.boolFilter(); + BoolQueryBuilder childrenFilter = boolQuery(); addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_PROFILE_KEY, query.getQProfileKey()); RuleIndex.addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance()); - FilterBuilder activeRuleFilter; + QueryBuilder activeRuleFilter; if (childrenFilter.hasClauses()) { activeRuleFilter = childrenFilter.must(ruleFilter); } else { @@ -430,7 +428,7 @@ public class RuleIndex extends BaseIndex { } } - private static StickyFacetBuilder stickyFacetBuilder(QueryBuilder query, Map filters) { + private static StickyFacetBuilder stickyFacetBuilder(QueryBuilder query, Map filters) { return new StickyFacetBuilder(query, filters); } @@ -492,7 +490,7 @@ public class RuleIndex extends BaseIndex { if (aggregation != null) { for (Terms.Bucket value : aggregation.getBuckets()) { - tags.add(value.getKey()); + tags.add(value.getKeyAsString()); } } return tags; diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java index 19be74edf10..ce9d2ac8d5d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -83,15 +83,28 @@ public class RuleIndexDefinition implements IndexDefinition { index.refreshHandledByIndexer(); index.configureShards(settings); + // Active rule type + NewIndex.NewIndexType activeRuleMapping = index.createType(RuleIndexDefinition.TYPE_ACTIVE_RULE); + activeRuleMapping.setEnableSource(false); + activeRuleMapping.setAttribute("_parent", ImmutableMap.of("type", RuleIndexDefinition.TYPE_RULE)); + + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY).enableSorting().build(); + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY).enableSorting().build(); + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY).build(); + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).disableNorms().docValues().build(); + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().docValues().build(); + activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).disableNorms().docValues().build(); + + activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_CREATED_AT); + activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_UPDATED_AT); + // Rule type NewIndex.NewIndexType ruleMapping = index.createType(TYPE_RULE); - ruleMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_RULE_KEY)); - ruleMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", RuleIndexDefinition.FIELD_RULE_REPOSITORY)); ruleMapping.setEnableSource(false); ruleMapping.stringFieldBuilder(FIELD_RULE_KEY).enableSorting().build(); ruleMapping.stringFieldBuilder(FIELD_RULE_RULE_KEY).enableSorting().build(); - ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).disableNorms().docValues().build(); + ruleMapping.stringFieldBuilder(FIELD_RULE_REPOSITORY).build(); ruleMapping.stringFieldBuilder(FIELD_RULE_INTERNAL_KEY).disableNorms().disableSearch().docValues().build(); ruleMapping.stringFieldBuilder(FIELD_RULE_NAME).enableSorting().enableWordSearch().build(); @@ -99,7 +112,7 @@ public class RuleIndexDefinition implements IndexDefinition { "type", "string", "index", "analyzed", "doc_values", "false", - "index_analyzer", "html_analyzer", + "analyzer", "html_analyzer", "search_analyzer", "html_analyzer" )); ruleMapping.stringFieldBuilder(FIELD_RULE_SEVERITY).disableNorms().docValues().build(); @@ -114,22 +127,5 @@ public class RuleIndexDefinition implements IndexDefinition { ruleMapping.createLongField(FIELD_RULE_CREATED_AT); ruleMapping.createLongField(FIELD_RULE_UPDATED_AT); - - // Active rule type - NewIndex.NewIndexType activeRuleMapping = index.createType(RuleIndexDefinition.TYPE_ACTIVE_RULE); - activeRuleMapping.setEnableSource(false); - activeRuleMapping.setAttribute("_id", ImmutableMap.of("path", RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY)); - activeRuleMapping.setAttribute("_parent", ImmutableMap.of("type", RuleIndexDefinition.TYPE_RULE)); - activeRuleMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY)); - - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_KEY).enableSorting().build(); - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY).disableNorms().disableSearch().docValues().build(); - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_REPOSITORY).disableNorms().disableSearch().docValues().build(); - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_KEY).disableNorms().docValues().build(); - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().docValues().build(); - activeRuleMapping.stringFieldBuilder(RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY).disableNorms().docValues().build(); - - activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_CREATED_AT); - activeRuleMapping.createLongField(RuleIndexDefinition.FIELD_ACTIVE_RULE_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index 35da71cb5af..e8689201e07 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -83,8 +83,6 @@ public class RuleIndexer extends BaseIndexer { } private static IndexRequest newIndexRequest(RuleDoc rule) { - return new IndexRequest(INDEX, TYPE_RULE, rule.key().toString()) - .routing(rule.repository()) - .source(rule.getFields()); + return new IndexRequest(INDEX, TYPE_RULE, rule.key().toString()).source(rule.getFields()); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/Facets.java b/server/sonar-server/src/main/java/org/sonar/server/search/Facets.java index 6b65220e09c..25ed16776fd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/Facets.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/Facets.java @@ -21,20 +21,19 @@ package org.sonar.server.search; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.HasAggregations; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - public class Facets { private static final Logger LOGGER = Loggers.get(Facets.class); @@ -58,7 +57,7 @@ public class Facets { processTermsAggregation(aggregation); } else if (HasAggregations.class.isAssignableFrom(aggregation.getClass())) { processSubAggregations(aggregation); - } else if (DateHistogram.class.isAssignableFrom(aggregation.getClass())) { + } else if (Histogram.class.isAssignableFrom(aggregation.getClass())) { processDateHistogram(aggregation); } else { LOGGER.warn("Cannot process {} type of aggregation", aggregation.getClass()); @@ -85,7 +84,7 @@ public class Facets { facetName = facetName.substring(0, facetName.indexOf("__")); } facetName = facetName.replace("_selected", ""); - this.facetValues.put(facetName, new FacetValue(value.getKey(), value.getDocCount())); + this.facetValues.put(facetName, new FacetValue(value.getKeyAsString(), value.getDocCount())); } } @@ -97,9 +96,9 @@ public class Facets { } private void processDateHistogram(Aggregation aggregation) { - DateHistogram dateHistogram = (DateHistogram) aggregation; - for (DateHistogram.Bucket value : dateHistogram.getBuckets()) { - this.facetValues.put(dateHistogram.getName(), new FacetValue(value.getKeyAsText().toString(), value.getDocCount())); + Histogram dateHistogram = (Histogram) aggregation; + for (Histogram.Bucket value : dateHistogram.getBuckets()) { + this.facetValues.put(dateHistogram.getName(), new FacetValue(value.getKeyAsString(), value.getDocCount())); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexUtils.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexUtils.java index af35f16d7dd..6536be850a1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexUtils.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexUtils.java @@ -19,12 +19,10 @@ */ package org.sonar.server.search; -import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; - +import java.util.Date; import javax.annotation.CheckForNull; import javax.annotation.Nullable; - -import java.util.Date; +import org.joda.time.format.ISODateTimeFormat; public class IndexUtils { diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/CoveredFileDoc.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/CoveredFileDoc.java index 9668bf24b7f..c23d5645f60 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/CoveredFileDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/CoveredFileDoc.java @@ -33,6 +33,21 @@ public class CoveredFileDoc extends BaseDoc { super(fields); } + @Override + public String getId() { + throw new UnsupportedOperationException(); + } + + @Override + public String getRouting() { + throw new UnsupportedOperationException(); + } + + @Override + public String getParent() { + throw new UnsupportedOperationException(); + } + @VisibleForTesting public CoveredFileDoc() { super(Maps.newHashMapWithExpectedSize(2)); diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java index 7a3d9b10d4a..9ada72ab9a9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java @@ -38,6 +38,7 @@ import static org.sonar.server.test.index.TestIndexDefinition.FIELD_STATUS; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_TEST_UUID; public class TestDoc extends BaseDoc { + public TestDoc(Map fields) { super(fields); } @@ -47,6 +48,21 @@ public class TestDoc extends BaseDoc { super(Maps.newHashMapWithExpectedSize(10)); } + @Override + public String getId() { + return testUuid(); + } + + @Override + public String getRouting() { + return projectUuid(); + } + + @Override + public String getParent() { + return null; + } + public String projectUuid() { return getField(FIELD_PROJECT_UUID); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java index efeef12ebd2..53df2eeabba 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndex.java @@ -25,8 +25,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.sonar.core.util.NonNullInputFunction; import org.sonar.server.es.BaseIndex; @@ -34,6 +32,10 @@ import org.sonar.server.es.EsClient; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.SearchResult; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_COVERED_FILES; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_COVERED_FILE_LINES; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_COVERED_FILE_UUID; @@ -58,7 +60,7 @@ public class TestIndex extends BaseIndex { for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX) .setTypes(TestIndexDefinition.TYPE) .setSize(1) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_TEST_UUID, testUuid))) + .setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_TEST_UUID, testUuid))) .get().getHits().getHits()) { coveredFiles.addAll(new TestDoc(hit.sourceAsMap()).coveredFiles()); } @@ -71,7 +73,7 @@ public class TestIndex extends BaseIndex { .setTypes(TestIndexDefinition.TYPE) .setSize(searchOptions.getLimit()) .setFrom(searchOptions.getOffset()) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_FILE_UUID, testFileUuid))); + .setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_FILE_UUID, testFileUuid))); return new SearchResult<>(searchRequest.get(), CONVERTER); } @@ -81,9 +83,9 @@ public class TestIndex extends BaseIndex { .setTypes(TestIndexDefinition.TYPE) .setSize(searchOptions.getLimit()) .setFrom(searchOptions.getOffset()) - .setQuery(QueryBuilders.nestedQuery(FIELD_COVERED_FILES, FilterBuilders.boolFilter() - .must(FilterBuilders.termFilter(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_UUID, sourceFileUuid).cache(false)) - .must(FilterBuilders.termFilter(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_LINES, lineNumber).cache(false)))); + .setQuery(nestedQuery(FIELD_COVERED_FILES, boolQuery() + .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_UUID, sourceFileUuid)) + .must(termQuery(FIELD_COVERED_FILES + "." + FIELD_COVERED_FILE_LINES, lineNumber)))); return new SearchResult<>(searchRequest.get(), CONVERTER); } @@ -101,7 +103,7 @@ public class TestIndex extends BaseIndex { for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX) .setTypes(TestIndexDefinition.TYPE) .setSize(1) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_TEST_UUID, testUuid))) + .setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_TEST_UUID, testUuid))) .get().getHits().getHits()) { return Optional.of(new TestDoc(hit.sourceAsMap())); } @@ -114,7 +116,7 @@ public class TestIndex extends BaseIndex { .setTypes(TestIndexDefinition.TYPE) .setSize(searchOptions.getLimit()) .setFrom(searchOptions.getOffset()) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_TEST_UUID, testUuid))); + .setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_TEST_UUID, testUuid))); return new SearchResult<>(searchRequest.get(), CONVERTER); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java index 6b4c532c4f8..f5c8569d1ea 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java @@ -53,13 +53,8 @@ public class TestIndexDefinition implements IndexDefinition { index.refreshHandledByIndexer(); index.configureShards(settings); - NewIndex.NewIndexType nestedMapping = index.createType(TYPE); - nestedMapping.stringFieldBuilder(FIELD_COVERED_FILE_UUID).build(); - nestedMapping.createIntegerField(FIELD_COVERED_FILE_LINES); - NewIndex.NewIndexType mapping = index.createType(TYPE); - mapping.setAttribute("_id", ImmutableMap.of("path", FIELD_TEST_UUID)); - mapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_PROJECT_UUID)); + mapping.setAttribute("_routing", ImmutableMap.of("required", true)); mapping.stringFieldBuilder(FIELD_PROJECT_UUID).disableNorms().build(); mapping.stringFieldBuilder(FIELD_FILE_UUID).disableNorms().build(); mapping.stringFieldBuilder(FIELD_TEST_UUID).disableNorms().build(); @@ -68,7 +63,10 @@ public class TestIndexDefinition implements IndexDefinition { mapping.createLongField(FIELD_DURATION_IN_MS); mapping.stringFieldBuilder(FIELD_MESSAGE).disableNorms().disableSearch().build(); mapping.stringFieldBuilder(FIELD_STACKTRACE).disableNorms().disableSearch().build(); - mapping.nestedObjectBuilder(FIELD_COVERED_FILES, nestedMapping).build(); + mapping.setProperty(FIELD_COVERED_FILES, ImmutableMap.of("type", "nested", "properties", ImmutableMap.of( + FIELD_COVERED_FILE_UUID, ImmutableMap.of("type", "string", "index", "not_analyzed"), + FIELD_COVERED_FILE_LINES, ImmutableMap.of("type", "integer") + ))); mapping.createDateTimeField(FIELD_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java index c6c11cdcd97..ad112c2f1b9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java @@ -23,7 +23,6 @@ import java.util.Iterator; import javax.annotation.Nullable; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -103,9 +102,7 @@ public class TestIndexer extends BaseIndexer { public void deleteByFile(String fileUuid) { SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX) .setTypes(TYPE) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.termFilter(FIELD_FILE_UUID, fileUuid).cache(false))); + .setQuery(QueryBuilders.termsQuery(FIELD_FILE_UUID, fileUuid)); BulkIndexer.delete(esClient, INDEX, searchRequest); } @@ -113,9 +110,7 @@ public class TestIndexer extends BaseIndexer { SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX) .setRouting(projectUuid) .setTypes(TYPE) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.termFilter(TestIndexDefinition.FIELD_PROJECT_UUID, projectUuid).cache(false))); + .setQuery(QueryBuilders.termQuery(TestIndexDefinition.FIELD_PROJECT_UUID, projectUuid)); BulkIndexer.delete(esClient, INDEX, searchRequest); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java index 995bfa7ffd5..e5268e60468 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestResultSetIterator.java @@ -35,6 +35,7 @@ import org.sonar.db.DbSession; import org.sonar.db.ResultSetIterator; import org.sonar.db.protobuf.DbFileSources; import org.sonar.db.source.FileSourceDto; +import org.sonar.server.es.EsUtils; import org.sonar.server.source.index.FileSourcesUpdaterHelper; import org.sonar.server.source.index.FileSourcesUpdaterHelper.Row; @@ -99,7 +100,7 @@ public class TestResultSetIterator extends ResultSetIterator { writer.prop(FIELD_DURATION_IN_MS, test.hasExecutionTimeMs() ? test.getExecutionTimeMs() : null); writer.prop(FIELD_MESSAGE, test.hasMsg() ? test.getMsg() : null); writer.prop(FIELD_STACKTRACE, test.hasStacktrace() ? test.getStacktrace() : null); - writer.prop(FIELD_UPDATED_AT, updatedAt.getTime()); + writer.prop(FIELD_UPDATED_AT, EsUtils.formatDateTime(updatedAt)); writer.name(FIELD_COVERED_FILES); writer.beginArray(); for (DbFileSources.Test.CoveredFile coveredFile : test.getCoveredFileList()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java index 65c6fe72518..ad0d9fbbdd4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java @@ -20,6 +20,7 @@ package org.sonar.server.user.index; import com.google.common.collect.Maps; +import java.util.Date; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -36,6 +37,21 @@ public class UserDoc extends BaseDoc implements User { this(Maps.newHashMap()); } + @Override + public String getId() { + return login(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + @Override public String login() { return getField(UserIndexDefinition.FIELD_LOGIN); @@ -62,11 +78,11 @@ public class UserDoc extends BaseDoc implements User { } public long createdAt() { - return getField(UserIndexDefinition.FIELD_CREATED_AT); + return getFieldAsDate(UserIndexDefinition.FIELD_CREATED_AT).getTime(); } public long updatedAt() { - return getField(UserIndexDefinition.FIELD_UPDATED_AT); + return getFieldAsDate(UserIndexDefinition.FIELD_UPDATED_AT).getTime(); } public UserDoc setLogin(@Nullable String s) { @@ -95,12 +111,12 @@ public class UserDoc extends BaseDoc implements User { } public UserDoc setCreatedAt(long l) { - setField(UserIndexDefinition.FIELD_CREATED_AT, l); + setField(UserIndexDefinition.FIELD_CREATED_AT, new Date(l)); return this; } public UserDoc setUpdatedAt(long l) { - setField(UserIndexDefinition.FIELD_UPDATED_AT, l); + setField(UserIndexDefinition.FIELD_UPDATED_AT, new Date(l)); return this; } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java index 3a4453c7d63..a65bebaa122 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -33,8 +33,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.BoolFilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -49,6 +48,11 @@ import org.sonar.server.es.EsUtils; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.SearchResult; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; + @ServerSide @ComputeEngineSide public class UserIndex { @@ -80,12 +84,12 @@ public class UserIndex { if (!StringUtils.isEmpty(scmAccount)) { SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX) .setTypes(UserIndexDefinition.TYPE_USER) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), - FilterBuilders.boolFilter() - .must(FilterBuilders.termFilter(UserIndexDefinition.FIELD_ACTIVE, true)) - .should(FilterBuilders.termFilter(UserIndexDefinition.FIELD_LOGIN, scmAccount)) - .should(FilterBuilders.termFilter(UserIndexDefinition.FIELD_EMAIL, scmAccount)) - .should(FilterBuilders.termFilter(UserIndexDefinition.FIELD_SCM_ACCOUNTS, scmAccount)))) + .setQuery(boolQuery().must(matchAllQuery()).filter( + boolQuery() + .must(termQuery(UserIndexDefinition.FIELD_ACTIVE, true)) + .should(termQuery(UserIndexDefinition.FIELD_LOGIN, scmAccount)) + .should(termQuery(UserIndexDefinition.FIELD_EMAIL, scmAccount)) + .should(termQuery(UserIndexDefinition.FIELD_SCM_ACCOUNTS, scmAccount)))) .setSize(3); for (SearchHit hit : request.get().getHits().getHits()) { result.add(DOC_CONVERTER.apply(hit.sourceAsMap())); @@ -95,8 +99,8 @@ public class UserIndex { } public Iterator selectUsersForBatch(List logins) { - BoolFilterBuilder filter = FilterBuilders.boolFilter() - .must(FilterBuilders.termsFilter(UserIndexDefinition.FIELD_LOGIN, logins)); + BoolQueryBuilder filter = boolQuery() + .must(termsQuery(UserIndexDefinition.FIELD_LOGIN, logins)); SearchRequestBuilder requestBuilder = esClient .prepareSearch(UserIndexDefinition.INDEX) @@ -108,7 +112,7 @@ public class UserIndex { .setFetchSource( new String[] {UserIndexDefinition.FIELD_LOGIN, UserIndexDefinition.FIELD_NAME}, null) - .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter)); + .setQuery(QueryBuilders.filteredQuery(matchAllQuery(), filter)); SearchResponse response = requestBuilder.get(); return EsUtils.scroll(esClient, response.getScrollId(), DOC_CONVERTER); @@ -121,12 +125,12 @@ public class UserIndex { .setFrom(options.getOffset()) .addSort(UserIndexDefinition.FIELD_NAME, SortOrder.ASC); - BoolFilterBuilder userFilter = FilterBuilders.boolFilter() - .must(FilterBuilders.termFilter(UserIndexDefinition.FIELD_ACTIVE, true)); + BoolQueryBuilder userQuery = boolQuery() + .must(termQuery(UserIndexDefinition.FIELD_ACTIVE, true)); QueryBuilder query; if (StringUtils.isEmpty(searchText)) { - query = QueryBuilders.matchAllQuery(); + query = matchAllQuery(); } else { query = QueryBuilders.multiMatchQuery(searchText, UserIndexDefinition.FIELD_LOGIN, @@ -137,7 +141,7 @@ public class UserIndex { } request.setQuery(QueryBuilders.filteredQuery(query, - userFilter)); + userQuery)); return new SearchResult<>(request.get(), DOC_CONVERTER); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java index 910055ab243..a0d7d487769 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewDoc.java @@ -34,6 +34,21 @@ public class ViewDoc extends BaseDoc { super(Maps.newHashMap()); } + @Override + public String getId() { + return uuid(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + public String uuid() { return getField(ViewIndexDefinition.FIELD_UUID); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java index e8c4f6e8ed8..3872c7f831d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java @@ -23,18 +23,18 @@ import java.util.Collection; import java.util.List; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.ce.ComputeEngineSide; import org.sonar.api.server.ServerSide; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; import static com.google.common.collect.Lists.newArrayList; -import static org.elasticsearch.index.query.FilterBuilders.termsFilter; -import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; @ServerSide @ComputeEngineSide @@ -51,9 +51,9 @@ public class ViewIndex { public List findAllViewUuids() { SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX) .setTypes(ViewIndexDefinition.TYPE_VIEW) - .setSearchType(SearchType.SCAN) + .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) - .setFetchSource(ViewIndexDefinition.FIELD_UUID, null) + .setFetchSource(false) .setSize(100) .setQuery(matchAllQuery()); @@ -62,13 +62,15 @@ public class ViewIndex { while (true) { List hits = newArrayList(response.getHits()); for (SearchHit hit : hits) { - result.add((String) hit.getSource().get(ViewIndexDefinition.FIELD_UUID)); + result.add(hit.getId()); } - response = esClient.prepareSearchScroll(response.getScrollId()) + String scrollId = response.getScrollId(); + response = esClient.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .get(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { + esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break; } } @@ -78,8 +80,7 @@ public class ViewIndex { public void delete(Collection viewUuids) { SearchRequestBuilder searchRequest = esClient.prepareSearch(ViewIndexDefinition.INDEX) .setTypes(ViewIndexDefinition.TYPE_VIEW) - .setQuery(filteredQuery(matchAllQuery(), termsFilter(ViewIndexDefinition.FIELD_UUID, viewUuids) - )); + .setQuery(boolQuery().must(matchAllQuery()).filter(termsQuery(ViewIndexDefinition.FIELD_UUID, viewUuids))); BulkIndexer.delete(esClient, ViewIndexDefinition.INDEX, searchRequest); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 296a9cccaf8..aaf546137e2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -29,7 +29,6 @@ import org.sonar.db.component.UuidWithProjectUuidDto; import org.sonar.server.es.BaseIndexer; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; -import org.sonar.server.issue.index.IssueIndex; import static com.google.common.collect.Maps.newHashMap; @@ -127,8 +126,7 @@ public class ViewIndexer extends BaseIndexer { private void clearLookupCache(String viewUuid) { try { esClient.prepareClearCache() - .setFilterCache(true) - .setFilterKeys(IssueIndex.viewsLookupCacheKey(viewUuid)) + .setQueryCache(true) .get(); } catch (Exception e) { throw new IllegalStateException(String.format("Unable to clear lookup cache of view '%s'", viewUuid), e); diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java index 3a5c58ea757..fd4743abe7b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java @@ -43,9 +43,9 @@ public class IndexPurgeListenerTest { @Test public void test_onIssuesRemoval() { - underTest.onIssuesRemoval(asList("ISSUE1", "ISSUE2")); + underTest.onIssuesRemoval("P1", asList("ISSUE1", "ISSUE2")); - verify(issueIndexer).deleteByKeys(asList("ISSUE1", "ISSUE2")); + verify(issueIndexer).deleteByKeys("P1", asList("ISSUE1", "ISSUE2")); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/issue/ScmAccountToUserLoaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/issue/ScmAccountToUserLoaderTest.java index 4127332a2e7..030674412f0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/issue/ScmAccountToUserLoaderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/issue/ScmAccountToUserLoaderTest.java @@ -49,7 +49,7 @@ public class ScmAccountToUserLoaderTest { .setEmail("charlie@hebdo.com") .setActive(true) .setScmAccounts(asList("charlie", "jesuis@charlie.com")); - esTester.index(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, "charlie", user.getFields()); + esTester.putDocuments(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, user); UserIndex index = new UserIndex(esTester.client()); ScmAccountToUserLoader underTest = new ScmAccountToUserLoader(index); @@ -66,14 +66,14 @@ public class ScmAccountToUserLoaderTest { .setEmail("charlie@hebdo.com") .setActive(true) .setScmAccounts(asList("charlie", "jesuis@charlie.com")); - esTester.index(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, user1.login(), user1.getFields()); + esTester.putDocuments(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, user1); UserDoc user2 = new UserDoc() .setLogin("another.charlie") .setName("Another Charlie") .setActive(true) .setScmAccounts(asList("charlie")); - esTester.index(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, user2.login(), user2.getFields()); + esTester.putDocuments(UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, user2); UserIndex index = new UserIndex(esTester.client()); ScmAccountToUserLoader underTest = new ScmAccountToUserLoader(index); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java index 0f7f087fe5a..ff89f4a01b9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Rule; import org.junit.Test; @@ -97,9 +96,7 @@ public class BulkIndexerTest { SearchRequestBuilder req = esTester.client().prepareSearch(FakeIndexDefinition.INDEX) .setTypes(FakeIndexDefinition.TYPE) - .setQuery(QueryBuilders.filteredQuery( - QueryBuilders.matchAllQuery(), - FilterBuilders.rangeFilter(FakeIndexDefinition.INT_FIELD).gte(removeFrom))); + .setQuery(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)); BulkIndexer.delete(esTester.client(), FakeIndexDefinition.INDEX, req); assertThat(count()).isEqualTo(removeFrom); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java index 18c73bcc285..666b9276f3b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java @@ -19,7 +19,7 @@ */ package org.sonar.server.es; -import org.elasticsearch.common.collect.ImmutableMap; +import java.util.Map; import org.junit.Test; import org.sonar.test.TestUtils; @@ -29,7 +29,7 @@ public class DefaultIndexSettingsTest { @Test public void defaults() { - ImmutableMap map = DefaultIndexSettings.defaults().build().getAsMap(); + Map map = DefaultIndexSettings.defaults().build().getAsMap(); assertThat(map).isNotEmpty(); // test some values diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java index be6fb6a1259..e1634e7a292 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsServerHolder.java @@ -23,7 +23,7 @@ import java.io.File; import java.util.Properties; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.sonar.process.LoopbackAddress; import org.sonar.process.NetworkUtils; @@ -63,9 +63,9 @@ public class EsServerHolder { public int getPort() { return port; } - + public String getHostName() { - return hostName; + return hostName; } public SearchServer getServer() { @@ -77,12 +77,12 @@ public class EsServerHolder { } private void reset() { - TransportClient client = new TransportClient(ImmutableSettings.settingsBuilder() + TransportClient client = TransportClient.builder().settings(Settings.builder() .put("node.name", nodeName) .put("network.bind_host", "localhost") .put("cluster.name", clusterName) - .build()); - client.addTransportAddress(new InetSocketTransportAddress(LoopbackAddress.get().getHostAddress(), port)); + .build()).build(); + client.addTransportAddress(new InetSocketTransportAddress(LoopbackAddress.get(), port)); // wait for node to be ready client.admin().cluster().prepareHealth() diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index 83285e45cea..c839686a3cc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -28,20 +28,17 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.lang.reflect.ConstructorUtils; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; @@ -54,7 +51,6 @@ import org.sonar.core.platform.ComponentContainer; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayList; import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; public class EsTester extends ExternalResource { @@ -99,15 +95,12 @@ public class EsTester extends ExternalResource { public void putDocuments(String index, String type, BaseDoc... docs) throws Exception { BulkRequestBuilder bulk = client.prepareBulk().setRefresh(true); for (BaseDoc doc : docs) { - bulk.add(new IndexRequest(index, type).source(doc.getFields())); + bulk.add(new IndexRequest(index, type, doc.getId()) + .parent(doc.getParent()) + .routing(doc.getRouting()) + .source(doc.getFields())); } - BulkResponse response = bulk.get(); - assertThat(response.hasFailures()).as(response.buildFailureMessage()).isFalse(); - } - - public void index(String indexName, String typeName, String id, Map source) { - client.prepareIndex(indexName, typeName).setId(id).setSource(source).get(); - client.prepareRefresh(indexName).get(); + EsUtils.executeBulkRequest(bulk, ""); } public long countDocuments(String indexName, String typeName) { @@ -194,7 +187,7 @@ public class EsTester extends ExternalResource { Path tmpDir = Files.createTempDirectory("tmp-es"); tmpDir.toFile().deleteOnExit(); - node = NodeBuilder.nodeBuilder().local(true).data(true).settings(ImmutableSettings.builder() + node = NodeBuilder.nodeBuilder().local(true).data(true).settings(org.elasticsearch.common.settings.Settings.builder() .put("cluster.name", nodeName) .put("node.name", nodeName) // the two following properties are probably not used because they are diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/FakeDoc.java b/server/sonar-server/src/test/java/org/sonar/server/es/FakeDoc.java index 734c9972b85..39229cb20de 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/FakeDoc.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/FakeDoc.java @@ -29,6 +29,21 @@ public class FakeDoc extends BaseDoc { super(fields); } + @Override + public String getId() { + return null; + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + public FakeDoc() { super(Maps.newHashMap()); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java index d68436df099..ea824d0e176 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/SortingTest.java @@ -19,7 +19,9 @@ */ package org.sonar.server.es; +import java.util.List; import org.apache.commons.lang.reflect.FieldUtils; +import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -28,8 +30,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.junit.Test; import org.sonar.server.exceptions.BadRequestException; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -58,7 +58,7 @@ public class SortingTest { Sorting sorting = new Sorting(); sorting.add("updatedAt"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fill(request, "updatedAt", true); List fields = fields(request); assertThat(fields).hasSize(1); @@ -70,7 +70,7 @@ public class SortingTest { Sorting sorting = new Sorting(); sorting.add("updatedAt"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fill(request, "updatedAt", false); List fields = fields(request); assertThat(fields).hasSize(1); @@ -82,7 +82,7 @@ public class SortingTest { Sorting sorting = new Sorting(); sorting.add("updatedAt").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fill(request, "updatedAt", true); List fields = fields(request); assertThat(fields).hasSize(1); @@ -94,7 +94,7 @@ public class SortingTest { Sorting sorting = new Sorting(); sorting.add("updatedAt").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fill(request, "updatedAt", false); List fields = fields(request); assertThat(fields).hasSize(1); @@ -110,7 +110,7 @@ public class SortingTest { sorting.add("fileLine", "severity").reverse(); sorting.add("fileLine", "key").missingLast(); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fill(request, "fileLine", true); List fields = fields(request); assertThat(fields).hasSize(4); @@ -126,7 +126,7 @@ public class SortingTest { sorting.add("file"); try { - sorting.fill(new SearchRequestBuilder(mock(Client.class)), "unknown", true); + sorting.fill(new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE), "unknown", true); fail(); } catch (BadRequestException e) { assertThat(e.getMessage()).isEqualTo("Bad sort field: unknown"); @@ -138,7 +138,7 @@ public class SortingTest { Sorting sorting = new Sorting(); sorting.addDefault("file"); - SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class)); + SearchRequestBuilder request = new SearchRequestBuilder(mock(Client.class), SearchAction.INSTANCE); sorting.fillDefault(request); List fields = fields(request); assertThat(fields).hasSize(1); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java index 249909f3837..2e39087b8b2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java @@ -50,10 +50,8 @@ public class ProxyClearCacheRequestBuilderTest { assertThat(esTester.client().prepareClearCache().toString()).isEqualTo("ES clear cache request"); assertThat(esTester.client().prepareClearCache("rules").toString()).isEqualTo("ES clear cache request on indices 'rules'"); assertThat(esTester.client().prepareClearCache().setFields("key").toString()).isEqualTo("ES clear cache request on fields 'key'"); - assertThat(esTester.client().prepareClearCache().setFilterKeys("rule1").toString()).isEqualTo("ES clear cache request on filterKeys 'rule1'"); - assertThat(esTester.client().prepareClearCache().setFilterCache(true).toString()).isEqualTo("ES clear cache request with filter cache"); assertThat(esTester.client().prepareClearCache().setFieldDataCache(true).toString()).isEqualTo("ES clear cache request with field data cache"); - assertThat(esTester.client().prepareClearCache().setIdCache(true).toString()).isEqualTo("ES clear cache request with id cache"); + assertThat(esTester.client().prepareClearCache().setRequestCache(true).toString()).isEqualTo("ES clear cache request with request cache"); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilderTest.java index 342ee36988d..cbb9bcfca36 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilderTest.java @@ -21,7 +21,7 @@ package org.sonar.server.es.request; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.unit.TimeValue; import org.junit.ClassRule; import org.junit.Rule; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilderTest.java deleted file mode 100644 index a3f3a75f49d..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyDeleteByQueryRequestBuilderTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2016 SonarSource SA - * mailto:contact AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.request; - -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; - -public class ProxyDeleteByQueryRequestBuilderTest { - - @Rule - public EsTester esTester = new EsTester(new FakeIndexDefinition()); - - @Rule - public LogTester logTester = new LogTester(); - - @Test - public void delete_by_query() { - esTester.client().prepareDeleteByQuery(FakeIndexDefinition.INDEX).setQuery(QueryBuilders.matchAllQuery()).get(); - } - - @Test - public void to_string() { - assertThat(esTester.client().prepareDeleteByQuery(FakeIndexDefinition.INDEX).setQuery(QueryBuilders.matchAllQuery()).toString()).isEqualTo( - "ES delete by query request '{\"match_all\":{}}' on indices 'fakes'"); - assertThat(esTester.client().prepareDeleteByQuery().toString()).isEqualTo("ES delete by query request ''"); - } - - @Test - public void trace_logs() { - logTester.setLevel(LoggerLevel.TRACE); - - esTester.client().prepareDeleteByQuery(FakeIndexDefinition.INDEX).setQuery(QueryBuilders.matchAllQuery()).get(); - - assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); - } - - @Test - public void get_with_string_timeout_is_not_yet_implemented() { - try { - esTester.client().prepareDeleteByQuery().get("1"); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void get_with_time_value_timeout_is_not_yet_implemented() { - try { - esTester.client().prepareDeleteByQuery().get(TimeValue.timeValueMinutes(1)); - fail(); - } catch (UnsupportedOperationException e) { - assertThat(e).hasMessage("Not yet implemented"); - } - } - - @Test - public void execute_should_throw_an_unsupported_operation_exception() { - try { - esTester.client().prepareDeleteByQuery().execute(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); - } - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java index 4f686ccab30..1f8e0e14a73 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java @@ -46,7 +46,6 @@ public class ProxyPutMappingRequestBuilderTest { public void put_mapping() { PutMappingRequestBuilder requestBuilder = esTester.client().preparePutMapping(FakeIndexDefinition.INDEX) .setType(FakeIndexDefinition.TYPE) - .setIgnoreConflicts(true) .setSource(mapDomain()); requestBuilder.get(); } @@ -65,7 +64,6 @@ public class ProxyPutMappingRequestBuilderTest { PutMappingRequestBuilder requestBuilder = esTester.client().preparePutMapping(FakeIndexDefinition.INDEX) .setType(FakeIndexDefinition.TYPE) - .setIgnoreConflicts(true) .setSource(mapDomain()); requestBuilder.get(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index f395b8513ea..a715a73e761 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.TimeZone; import javax.annotation.Nullable; import org.assertj.core.api.Fail; +import org.elasticsearch.test.ESTestCase; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index e656d554a67..995909dcb3f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import javax.annotation.Nonnull; +import org.elasticsearch.search.SearchHit; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.Settings; @@ -39,9 +40,10 @@ import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.server.issue.IssueTesting.newDoc; - public class IssueIndexerTest { + private static final String A_PROJECT_UUID = "P1"; + @Rule public EsTester esTester = new EsTester(new IssueIndexDefinition(new Settings())); @@ -91,12 +93,13 @@ public class IssueIndexerTest { } @Test - public void delete_project_remove_issue() { + public void deleteProject_deletes_issues() { dbTester.prepareDbUnit(getClass(), "index.xml"); IssueIndexer indexer = createIndexer(); indexer.index(); + List docs = esTester.getDocuments("issues", "issue"); assertThat(esTester.countDocuments("issues", "issue")).isEqualTo(1); indexer.deleteProject("THE_PROJECT"); @@ -116,46 +119,47 @@ public class IssueIndexerTest { @Test public void delete_issues_by_keys() throws Exception { - addIssue("Issue1"); - addIssue("Issue2"); - addIssue("Issue3"); + addIssue("P1", "Issue1"); + addIssue("P1", "Issue2"); + addIssue("P1", "Issue3"); + addIssue("P2", "Issue4"); IssueIndexer indexer = createIndexer(); - verifyIssueKeys("Issue1", "Issue2", "Issue3"); + verifyIssueKeys("Issue1", "Issue2", "Issue3", "Issue4"); - indexer.deleteByKeys(asList("Issue1", "Issue2")); + indexer.deleteByKeys("P1", asList("Issue1", "Issue2")); - verifyIssueKeys("Issue3"); + verifyIssueKeys("Issue3", "Issue4"); } @Test public void delete_more_than_one_thousand_issues_by_keys() throws Exception { int numberOfIssues = 1010; List keys = new ArrayList<>(numberOfIssues); - List issueDocs = new ArrayList<>(numberOfIssues); - for (int i=0; iemptyList()); + indexer.deleteByKeys("P1", Collections.emptyList()); verifyIssueKeys("Issue1", "Issue2", "Issue3"); } @@ -166,11 +170,12 @@ public class IssueIndexerTest { return indexer; } - private void addIssue(String issueKey) throws Exception { - esTester.putDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, newDoc().setKey(issueKey)); + private void addIssue(String projectUuid, String issueKey) throws Exception { + esTester.putDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, + newDoc().setKey(issueKey).setProjectUuid(projectUuid)); } - private void verifyIssueKeys(String... expectedKeys){ + private void verifyIssueKeys(String... expectedKeys) { List keys = FluentIterable.from(esTester.getDocuments("issues", "issue", IssueDoc.class)) .transform(DocToKey.INSTANCE) .toList(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java index 3f3c79940ed..e610810347a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupMediumTest.java @@ -19,7 +19,6 @@ */ package org.sonar.server.platform; -import com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.Settings; @@ -111,9 +110,6 @@ public class BackendCleanupMediumTest { } private static RuleDoc newRuleDoc() { - return new RuleDoc(ImmutableMap.of( - RuleIndexDefinition.FIELD_RULE_KEY, RuleTesting.XOO_X1, - RuleIndexDefinition.FIELD_RULE_REPOSITORY, RuleTesting.XOO_X1.repository() - )); + return new RuleDoc().setKey(RuleTesting.XOO_X1.toString()).setRepository(RuleTesting.XOO_X1.repository()); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/EsMonitorTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/EsMonitorTest.java index e02092137f6..911aed32594 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/EsMonitorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/EsMonitorTest.java @@ -20,7 +20,7 @@ package org.sonar.server.platform.monitoring; import java.util.Map; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.Settings; diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java index d8b55783f85..ed00c19b01f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/ws/BulkDeleteActionTest.java @@ -19,7 +19,6 @@ */ package org.sonar.server.project.ws; -import com.google.common.collect.ImmutableMap; import java.util.Arrays; import org.junit.Before; import org.junit.Rule; @@ -48,6 +47,7 @@ import org.sonar.server.component.ComponentFinder; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.issue.IssueTesting; +import org.sonar.server.issue.index.IssueAuthorizationDoc; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; @@ -222,7 +222,7 @@ public class BulkDeleteActionTest { dbSession.commit(); es.putDocuments(IssueIndexDefinition.INDEX, TYPE_ISSUE, IssueTesting.newDoc("issue-key-" + suffix, project)); - es.index(IssueIndexDefinition.INDEX, TYPE_AUTHORIZATION, project.uuid(), ImmutableMap.of(FIELD_AUTHORIZATION_PROJECT_UUID, project.uuid())); + es.putDocuments(IssueIndexDefinition.INDEX, TYPE_AUTHORIZATION, new IssueAuthorizationDoc().setProjectUuid(project.uuid())); TestDoc testDoc = new TestDoc().setUuid("test-uuid-" + suffix).setProjectUuid(project.uuid()).setFileUuid(project.uuid()); es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, testDoc); diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java index 8a073530bdd..e7708782b39 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/project/ws/DeleteActionTest.java @@ -19,7 +19,6 @@ */ package org.sonar.server.project.ws; -import com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -46,6 +45,7 @@ import org.sonar.server.component.ComponentFinder; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.issue.IssueTesting; +import org.sonar.server.issue.index.IssueAuthorizationDoc; import org.sonar.server.issue.index.IssueAuthorizationIndexer; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; @@ -241,7 +241,7 @@ public class DeleteActionTest { dbSession.commit(); es.putDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, IssueTesting.newDoc("issue-key-" + suffix, project)); - es.index(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, project.uuid(), ImmutableMap.of(IssueIndexDefinition.FIELD_AUTHORIZATION_PROJECT_UUID, project.uuid())); + es.putDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, new IssueAuthorizationDoc().setProjectUuid(project.uuid())); TestDoc testDoc = new TestDoc().setUuid("test-uuid-" + suffix).setProjectUuid(project.uuid()).setFileUuid(project.uuid()); es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, testDoc); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java index e1ca4f00888..bdfbd5edc82 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java @@ -77,7 +77,7 @@ public class RuleIndexDefinitionTest { tester.putDocuments(INDEX, RuleIndexDefinition.TYPE_RULE, new RuleDoc(ImmutableMap.of( FIELD_RULE_HTML_DESCRIPTION, longText, FIELD_RULE_REPOSITORY, "squid", - FIELD_RULE_KEY, "S001"))); + FIELD_RULE_KEY, "squid:S001"))); assertThat(tester.countDocuments(INDEX, RuleIndexDefinition.TYPE_RULE)).isEqualTo(1); List tokens = analyzeIndexedTokens(longText); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java index a60151de0fa..ac9cfd29fc8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import java.util.Collections; import java.util.List; import java.util.Map; +import org.elasticsearch.search.SearchHit; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -189,6 +190,10 @@ public class RuleIndexTest { newDoc(RuleKey.of("findbugs", "S001")), newDoc(RuleKey.of("pmd", "S002"))); + List docs = tester.getDocuments("rules", "rule"); + for (SearchHit doc : docs) { + System.out.println(doc.getSourceAsString()); + } RuleQuery query = new RuleQuery().setRepositories(asList("checkstyle", "pmd")); SearchIdResult results = index.search(query, new SearchOptions()); assertThat(results.getIds()).containsOnly(RuleKey.of("pmd", "S002")); @@ -611,7 +616,7 @@ public class RuleIndexTest { SearchIdResult result = index.search(new RuleQuery(), new SearchOptions().addFacets(asList(FACET_LANGUAGES, FACET_REPOSITORIES, FACET_TAGS, FACET_TYPES))); assertThat(result.getFacets().getAll()).hasSize(4); assertThat(result.getFacets().getAll().get(FACET_LANGUAGES).keySet()).containsOnly("cpp", "java", "cobol"); - assertThat(result.getFacets().getAll().get(FACET_REPOSITORIES).keySet()).containsOnly("xoo", "foo"); + assertThat(result.getFacets().getAll().get(FACET_REPOSITORIES).keySet()).containsExactly("xoo", "foo"); assertThat(result.getFacets().getAll().get(FACET_TAGS).keySet()).containsOnly("T1", "T2", "T3", "T4"); assertThat(result.getFacets().getAll().get(FACET_TYPES).keySet()).containsOnly("BUG", "CODE_SMELL", "VULNERABILITY"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java index d694b16ff18..e8891abba1c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java @@ -38,6 +38,20 @@ public class BaseDocTest { fields.put("a_int", 42); fields.put("a_null", null); BaseDoc doc = new BaseDoc(fields) { + @Override + public String getId() { + return null; + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } }; assertThat((String) doc.getNullableField("a_string")).isEqualTo("foo"); @@ -49,6 +63,20 @@ public class BaseDocTest { public void getField_fails_if_missing_field() { Map fields = Collections.emptyMap(); BaseDoc doc = new BaseDoc(fields) { + @Override + public String getId() { + return null; + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } }; try { @@ -62,6 +90,20 @@ public class BaseDocTest { @Test public void getFieldAsDate() { BaseDoc doc = new BaseDoc(Maps.newHashMap()) { + @Override + public String getId() { + return null; + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } }; long now = System.currentTimeMillis(); doc.setField("javaDate", new Date(now)); @@ -74,6 +116,20 @@ public class BaseDocTest { @Test public void getNullableFieldAsDate() { BaseDoc doc = new BaseDoc(Maps.newHashMap()) { + @Override + public String getId() { + return null; + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } }; long now = System.currentTimeMillis(); doc.setField("javaDate", new Date(now)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/FacetsMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/FacetsMediumTest.java index a4131becc04..e58c217455c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/FacetsMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/FacetsMediumTest.java @@ -19,17 +19,18 @@ */ package org.sonar.server.search; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Date; -import java.util.Map; +import java.util.List; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram.Interval; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.DateUtils; +import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsTester; import org.sonar.server.es.NewIndex.NewIndexType; @@ -57,11 +58,11 @@ public class FacetsMediumTest { @Test public void should_ignore_unknown_aggregation_type() throws Exception { - esTester.index(INDEX, TYPE, "noTags", newTagsDocument("noTags")); - esTester.index(INDEX, TYPE, "oneTag", newTagsDocument("oneTag", "tag1")); - esTester.index(INDEX, TYPE, "twoTags", newTagsDocument("twoTags", "tag1", "tag2")); - esTester.index(INDEX, TYPE, "threeTags", newTagsDocument("threeTags", "tag1", "tag2", "tag3")); - esTester.index(INDEX, TYPE, "fourTags", newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("noTags")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("oneTag", "tag1")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("twoTags", "tag1", "tag2")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("threeTags", "tag1", "tag2", "tag3")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); SearchRequestBuilder search = esTester.client().prepareSearch(INDEX).setTypes(TYPE) .addAggregation(AggregationBuilders.cardinality(FIELD_TAGS).field(FIELD_TAGS)); @@ -72,9 +73,9 @@ public class FacetsMediumTest { @Test public void should_process_result_with_nested_missing_and_terms_aggregations() throws Exception { - esTester.index(INDEX, TYPE, "noTags", newTagsDocument("noTags")); - esTester.index(INDEX, TYPE, "oneTag", newTagsDocument("oneTag", "tag1")); - esTester.index(INDEX, TYPE, "fourTags", newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("noTags")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("oneTag", "tag1")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); SearchRequestBuilder search = esTester.client().prepareSearch(INDEX).setTypes(TYPE) .addAggregation(AggregationBuilders.global("tags__global") @@ -98,10 +99,10 @@ public class FacetsMediumTest { @Test public void should_ignore_empty_missing_aggregation() throws Exception { - esTester.index(INDEX, TYPE, "oneTag", newTagsDocument("oneTag", "tag1")); - esTester.index(INDEX, TYPE, "twoTags", newTagsDocument("twoTags", "tag1", "tag2")); - esTester.index(INDEX, TYPE, "threeTags", newTagsDocument("threeTags", "tag1", "tag2", "tag3")); - esTester.index(INDEX, TYPE, "fourTags", newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("oneTag", "tag1")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("twoTags", "tag1", "tag2")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("threeTags", "tag1", "tag2", "tag3")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("fourTags", "tag1", "tag2", "tag3", "tag4")); SearchRequestBuilder search = esTester.client().prepareSearch(INDEX).setTypes(TYPE) .addAggregation(AggregationBuilders.global("tags__global") @@ -118,16 +119,16 @@ public class FacetsMediumTest { @Test public void should_process_result_with_date_histogram() throws Exception { - esTester.index(INDEX, TYPE, "first", newTagsDocument("first")); - esTester.index(INDEX, TYPE, "second", newTagsDocument("second")); - esTester.index(INDEX, TYPE, "third", newTagsDocument("third")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("first")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("second")); + esTester.putDocuments(INDEX, TYPE, newTagsDocument("third")); SearchRequestBuilder search = esTester.client().prepareSearch(INDEX).setTypes(TYPE) .addAggregation( AggregationBuilders.dateHistogram(FIELD_CREATED_AT) .minDocCount(0L) .field(FIELD_CREATED_AT) - .interval(Interval.MINUTE) + .interval(DateHistogramInterval.MINUTE) .format(DateUtils.DATETIME_FORMAT)); Facets facets = new Facets(search.get()); @@ -139,15 +140,11 @@ public class FacetsMediumTest { assertThat(value.getValue()).isEqualTo(3L); } - private static Map newTagsDocument(String key, String... tags) { - ImmutableMap doc = ImmutableMap.of( - FIELD_KEY, key, - FIELD_TAGS, Arrays.asList(tags), - FIELD_CREATED_AT, new Date()); - return doc; + private static TagsDoc newTagsDocument(String key, String... tags) { + return new TagsDoc().setKey(key).setTags(Arrays.asList(tags)).setCreatedAt(new Date()); } - static class FacetsTestDefinition implements org.sonar.server.es.IndexDefinition { + private static class FacetsTestDefinition implements org.sonar.server.es.IndexDefinition { @Override public void define(IndexDefinitionContext context) { @@ -157,4 +154,48 @@ public class FacetsMediumTest { newType.createDateTimeField(FIELD_CREATED_AT); } } + + private static class TagsDoc extends BaseDoc { + public TagsDoc() { + super(Maps.newHashMap()); + } + + @Override + public String getId() { + return getKey(); + } + + @Override + public String getRouting() { + return null; + } + + @Override + public String getParent() { + return null; + } + + public String getKey() { + return getField(FIELD_KEY); + } + + public List getTags() { + return (List) getField(FIELD_TAGS); + } + + public TagsDoc setKey(String s) { + setField(FIELD_KEY, s); + return this; + } + + public TagsDoc setTags(List tags) { + setField(FIELD_TAGS, tags); + return this; + } + + public TagsDoc setCreatedAt(Date date) { + setField(FIELD_CREATED_AT, date); + return this; + } + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java index 7d91e5e71c7..885beb133bf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexTest.java @@ -20,7 +20,6 @@ package org.sonar.server.test.index; import com.google.common.base.Optional; -import java.util.Arrays; import java.util.List; import org.junit.Rule; import org.junit.Test; @@ -28,6 +27,7 @@ import org.sonar.api.config.Settings; import org.sonar.server.es.EsTester; import org.sonar.server.es.SearchOptions; +import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.guava.api.Assertions.assertThat; @@ -40,8 +40,8 @@ public class TestIndexTest { @Test public void coveredFiles() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); List result = underTest.coveredFiles("1"); @@ -53,9 +53,9 @@ public class TestIndexTest { @Test public void searchByTestFileUuid() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7")), - newTestDoc("3", "TESTFILE2", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7")), + newTestDoc("3", "TESTFILE2", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); List result = underTest.searchByTestFileUuid("TESTFILE1", searchOptions()).getDocs(); @@ -66,9 +66,9 @@ public class TestIndexTest { @Test public void searchBySourceFileUuidAndLineNumber() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("10"), newCoverageBlock("11"), newCoverageBlock("12")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("3", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("10"), newCoveredFileDoc("11"), newCoveredFileDoc("12")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("3", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); List result = underTest.searchBySourceFileUuidAndLineNumber("main-uuid-5", 82, searchOptions()).getDocs(); @@ -79,8 +79,8 @@ public class TestIndexTest { @Test public void searchByTestUuid() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); TestDoc test = underTest.getByTestUuid("1"); @@ -97,8 +97,8 @@ public class TestIndexTest { @Test public void getNullableByTestUuid() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); Optional result = underTest.getNullableByTestUuid("1"); @@ -124,8 +124,8 @@ public class TestIndexTest { @Test public void searchByTestUuid_with_SearchOptions() throws Exception { es.putDocuments(TestIndexDefinition.INDEX, TestIndexDefinition.TYPE, - newTestDoc("1", "TESTFILE1", newCoverageBlock("3"), newCoverageBlock("4"), newCoverageBlock("5")), - newTestDoc("2", "TESTFILE1", newCoverageBlock("5"), newCoverageBlock("6"), newCoverageBlock("7"))); + newTestDoc("1", "TESTFILE1", newCoveredFileDoc("3"), newCoveredFileDoc("4"), newCoveredFileDoc("5")), + newTestDoc("2", "TESTFILE1", newCoveredFileDoc("5"), newCoveredFileDoc("6"), newCoveredFileDoc("7"))); List result = underTest.searchByTestUuid("1", searchOptions()).getDocs(); @@ -133,23 +133,23 @@ public class TestIndexTest { assertThat(result.get(0).testUuid()).isEqualTo("1"); } - private CoveredFileDoc newCoverageBlock(String id) { + private CoveredFileDoc newCoveredFileDoc(String id) { return new CoveredFileDoc() .setFileUuid("main-uuid-" + id) - .setCoveredLines(Arrays.asList(25, 33, 82)); + .setCoveredLines(asList(25, 33, 82)); } private TestDoc newTestDoc(String testUuid, String fileUuid, CoveredFileDoc... coveredFiles) { return new TestDoc() .setUuid(testUuid) - .setProjectUuid("project-" + fileUuid) + .setProjectUuid("P1") .setName("name-" + testUuid) .setMessage("message-" + testUuid) .setStackTrace("stacktrace-" + testUuid) .setStatus("status-" + testUuid) .setDurationInMs(Long.valueOf(testUuid)) .setFileUuid(fileUuid) - .setCoveredFiles(Arrays.asList(coveredFiles)); + .setCoveredFiles(asList(coveredFiles)); } private SearchOptions searchOptions() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java index ee3d15f35c4..1cef3e8bd18 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java @@ -175,6 +175,7 @@ public class TestIndexerTest { private void indexTest(String projectUuid, String fileUuid, String testName, String uuid) throws IOException { es.client().prepareIndex(INDEX, TYPE) .setId(uuid) + .setRouting(projectUuid) .setSource(FileUtils.readFileToString(TestUtils.getResource(this.getClass(), projectUuid + "_" + fileUuid + "_" + testName + ".json"))) .setRefresh(true) .get(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java index 7485db97ce0..6e6b5cff38f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestResultSetIteratorTest.java @@ -45,7 +45,6 @@ import static org.sonar.server.test.index.TestIndexDefinition.FIELD_PROJECT_UUID import static org.sonar.server.test.index.TestIndexDefinition.FIELD_STACKTRACE; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_STATUS; import static org.sonar.server.test.index.TestIndexDefinition.FIELD_TEST_UUID; -import static org.sonar.server.test.index.TestIndexDefinition.FIELD_UPDATED_AT; public class TestResultSetIteratorTest { @@ -105,8 +104,7 @@ public class TestResultSetIteratorTest { entry(FIELD_MESSAGE, "MESSAGE_1"), entry(FIELD_DURATION_IN_MS, 1), entry(FIELD_STACKTRACE, "STACKTRACE_1"), - entry(FIELD_NAME, "NAME_1"), - entry(FIELD_UPDATED_AT, 1416239042000L)); + entry(FIELD_NAME, "NAME_1")); } /** @@ -135,8 +133,7 @@ public class TestResultSetIteratorTest { entry(FIELD_PROJECT_UUID, "P1"), entry(FIELD_FILE_UUID, "F1"), entry(FIELD_TEST_UUID, "U1"), - entry(FIELD_NAME, "N1"), - entry(FIELD_UPDATED_AT, 1416239042000L)); + entry(FIELD_NAME, "N1")); ; // null values assertThat(doc).containsKeys( diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java index e6f708e3f55..6d542314170 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java @@ -55,8 +55,8 @@ public class UserIndexTest { @Test public void get_nullable_by_login() throws Exception { UserDoc user1 = newUser(USER1_LOGIN, asList("scmA", "scmB")); - esTester.index(INDEX, TYPE_USER, USER1_LOGIN, user1.getFields()); - esTester.index(INDEX, TYPE_USER, USER2_LOGIN, newUser(USER2_LOGIN, Collections.emptyList()).getFields()); + esTester.putDocuments(INDEX, TYPE_USER, user1); + esTester.putDocuments(INDEX, TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList())); UserDoc userDoc = index.getNullableByLogin(USER1_LOGIN); assertThat(userDoc).isNotNull(); @@ -75,7 +75,7 @@ public class UserIndexTest { @Test public void get_nullable_by_login_should_be_case_sensitive() throws Exception { UserDoc user1 = newUser(USER1_LOGIN, asList("scmA", "scmB")); - esTester.index(INDEX, TYPE_USER, USER1_LOGIN, user1.getFields()); + esTester.putDocuments(INDEX, TYPE_USER, user1); assertThat(index.getNullableByLogin(USER1_LOGIN)).isNotNull(); assertThat(index.getNullableByLogin("UsEr1")).isNull(); @@ -86,9 +86,9 @@ public class UserIndexTest { UserDoc user1 = newUser("user1", asList("user_1", "u1")); UserDoc user2 = newUser("user_with_same_email_as_user1", asList("user_2")).setEmail(user1.email()); UserDoc user3 = newUser("inactive_user_with_same_scm_as_user1", user1.scmAccounts()).setActive(false); - esTester.index(INDEX, TYPE_USER, user1.login(), user1.getFields()); - esTester.index(INDEX, TYPE_USER, user2.login(), user2.getFields()); - esTester.index(INDEX, TYPE_USER, user3.login(), user3.getFields()); + esTester.putDocuments(INDEX, TYPE_USER, user1); + esTester.putDocuments(INDEX, TYPE_USER, user2); + esTester.putDocuments(INDEX, TYPE_USER, user3); assertThat(index.getAtMostThreeActiveUsersForScmAccount(user1.scmAccounts().get(0))).extractingResultOf("login").containsOnly(user1.login()); assertThat(index.getAtMostThreeActiveUsersForScmAccount(user1.login())).extractingResultOf("login").containsOnly(user1.login()); @@ -104,7 +104,7 @@ public class UserIndexTest { public void getAtMostThreeActiveUsersForScmAccount_ignore_inactive_user() throws Exception { String scmAccount = "scmA"; UserDoc user = newUser(USER1_LOGIN, asList(scmAccount)).setActive(false); - esTester.index(INDEX, TYPE_USER, user.login(), user.getFields()); + esTester.putDocuments(INDEX, TYPE_USER, user); assertThat(index.getAtMostThreeActiveUsersForScmAccount(user.login())).isEmpty(); assertThat(index.getAtMostThreeActiveUsersForScmAccount(scmAccount)).isEmpty(); @@ -117,10 +117,10 @@ public class UserIndexTest { UserDoc user2 = newUser("user2", Collections.emptyList()).setEmail(email); UserDoc user3 = newUser("user3", Collections.emptyList()).setEmail(email); UserDoc user4 = newUser("user4", Collections.emptyList()).setEmail(email); - esTester.index(INDEX, TYPE_USER, user1.login(), user1.getFields()); - esTester.index(INDEX, TYPE_USER, user2.login(), user2.getFields()); - esTester.index(INDEX, TYPE_USER, user3.login(), user3.getFields()); - esTester.index(INDEX, TYPE_USER, user4.login(), user4.getFields()); + esTester.putDocuments(INDEX, TYPE_USER, user1); + esTester.putDocuments(INDEX, TYPE_USER, user2); + esTester.putDocuments(INDEX, TYPE_USER, user3); + esTester.putDocuments(INDEX, TYPE_USER, user4); // restrict results to 3 users assertThat(index.getAtMostThreeActiveUsersForScmAccount(email)).hasSize(3); @@ -128,8 +128,8 @@ public class UserIndexTest { @Test public void searchUsers() throws Exception { - esTester.index(INDEX, TYPE_USER, USER1_LOGIN, newUser(USER1_LOGIN, Arrays.asList("user_1", "u1")).getFields()); - esTester.index(INDEX, TYPE_USER, USER2_LOGIN, newUser(USER2_LOGIN, Collections.emptyList()).getFields()); + esTester.putDocuments(INDEX, TYPE_USER, newUser(USER1_LOGIN, Arrays.asList("user_1", "u1"))); + esTester.putDocuments(INDEX, TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList())); assertThat(index.search(null, new SearchOptions()).getDocs()).hasSize(2); assertThat(index.search("user", new SearchOptions()).getDocs()).hasSize(2); diff --git a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexTest.java index 9d568321880..4f0210e6b95 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexTest.java @@ -42,8 +42,8 @@ public class ViewIndexTest { public void find_all_view_uuids() throws Exception { ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1")); ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2")); - esTester.index(INDEX, TYPE_VIEW, view1.uuid(), view1.getFields()); - esTester.index(INDEX, TYPE_VIEW, view2.uuid(), view2.getFields()); + esTester.putDocuments(INDEX, TYPE_VIEW, view1); + esTester.putDocuments(INDEX, TYPE_VIEW, view2); List result = newArrayList(index.findAllViewUuids()); @@ -62,9 +62,9 @@ public class ViewIndexTest { ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1")); ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4")); ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4")); - esTester.index(INDEX, TYPE_VIEW, view1.uuid(), view1.getFields()); - esTester.index(INDEX, TYPE_VIEW, view2.uuid(), view2.getFields()); - esTester.index(INDEX, TYPE_VIEW, view3.uuid(), view3.getFields()); + esTester.putDocuments(INDEX, TYPE_VIEW, view1); + esTester.putDocuments(INDEX, TYPE_VIEW, view2); + esTester.putDocuments(INDEX, TYPE_VIEW, view3); index.delete(asList(view1.uuid(), view2.uuid())); diff --git a/sonar-application/assembly.xml b/sonar-application/assembly.xml index b04e3a21437..a860949142d 100644 --- a/sonar-application/assembly.xml +++ b/sonar-application/assembly.xml @@ -42,6 +42,7 @@ false org.elasticsearch:elasticsearch + net.java.dev.jna:jna provided diff --git a/sonar-application/pom.xml b/sonar-application/pom.xml index 087e497a6ac..02f76e4a60d 100644 --- a/sonar-application/pom.xml +++ b/sonar-application/pom.xml @@ -238,8 +238,8 @@ - 110000000 - 120000000 + 120000000 + 125000000 ${project.build.directory}/sonarqube-${project.version}.zip diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java index 53bd63175ef..bc3c54e9868 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -93,7 +93,7 @@ public class PurgeDao implements Dao { List issueKeys = mapper.selectOldClosedIssueKeys(conf.rootProjectIdUuid().getUuid(), dateToLong(toDate)); executeLargeInputs(issueKeys, new DeleteIssueChangesFromIssueKeys(mapper)); executeLargeInputs(issueKeys, new DeleteIssuesFromKeys(mapper)); - listener.onIssuesRemoval(issueKeys); + listener.onIssuesRemoval(conf.rootProjectIdUuid().getUuid(), issueKeys); } private static void deleteAbortedBuilds(ResourceDto project, PurgeCommands commands) { diff --git a/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java b/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java index a2224c2d715..1d77fcc5a97 100644 --- a/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java +++ b/sonar-db/src/main/java/org/sonar/db/purge/PurgeListener.java @@ -30,12 +30,12 @@ public interface PurgeListener { } @Override - public void onIssuesRemoval(List issueKeys) { + public void onIssuesRemoval(String projectUuid, List issueKeys) { // do nothing } }; void onComponentDisabling(String uuid); - void onIssuesRemoval(List issueKeys); + void onIssuesRemoval(String projectUuid, List issueKeys); } diff --git a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java index 637a8f3e103..aace20f99df 100644 --- a/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/purge/PurgeDaoTest.java @@ -44,6 +44,9 @@ import static org.sonar.db.ce.CeTaskTypes.REPORT; public class PurgeDaoTest { + private static final String THE_PROJECT_UUID = "P1"; + private static final long THE_PROJECT_ID = 1L; + System2 system2 = mock(System2.class); @Rule @@ -79,7 +82,7 @@ public class PurgeDaoTest { @Test public void shouldDeleteHistoricalDataOfDirectoriesAndFiles() { dbTester.prepareDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles.xml"); - underTest.purge(new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[]{Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, new PurgeProfiler()); + underTest.purge(new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[] {Scopes.DIRECTORY, Scopes.FILE}, 30), PurgeListener.EMPTY, new PurgeProfiler()); dbTester.assertDbUnit(getClass(), "shouldDeleteHistoricalDataOfDirectoriesAndFiles-result.xml", "projects", "snapshots"); } @@ -95,18 +98,18 @@ public class PurgeDaoTest { @Test public void shouldDeleteSnapshots() { dbTester.prepareDbUnit(getClass(), "shouldDeleteSnapshots.xml"); - underTest.deleteSnapshots(PurgeSnapshotQuery.create().setIslast(false).setResourceId(1L), new PurgeProfiler()); + underTest.deleteSnapshots(PurgeSnapshotQuery.create().setIslast(false).setResourceId(THE_PROJECT_ID), new PurgeProfiler()); dbTester.assertDbUnit(getClass(), "shouldDeleteSnapshots-result.xml", "snapshots"); } @Test public void shouldSelectPurgeableSnapshots() { dbTester.prepareDbUnit(getClass(), "shouldSelectPurgeableSnapshots.xml"); - List snapshots = underTest.selectPurgeableSnapshots(1L); + List snapshots = underTest.selectPurgeableSnapshots(THE_PROJECT_ID); assertThat(snapshots).hasSize(3); - assertThat(getById(snapshots, 1L).isLast()).isTrue(); - assertThat(getById(snapshots, 1L).hasEvents()).isFalse(); + assertThat(getById(snapshots, THE_PROJECT_ID).isLast()).isTrue(); + assertThat(getById(snapshots, THE_PROJECT_ID).hasEvents()).isFalse(); assertThat(getById(snapshots, 4L).isLast()).isFalse(); assertThat(getById(snapshots, 4L).hasEvents()).isFalse(); assertThat(getById(snapshots, 5L).isLast()).isFalse(); @@ -182,16 +185,18 @@ public class PurgeDaoTest { dbTester.assertDbUnit(getClass(), "should_delete_old_closed_issues-result.xml", "issues", "issue_changes"); Class> listClass = (Class>)(Class)ArrayList.class; - ArgumentCaptor> argument = ArgumentCaptor.forClass(listClass); + ArgumentCaptor> issueKeys = ArgumentCaptor.forClass(listClass); + ArgumentCaptor projectUuid = ArgumentCaptor.forClass(String.class); - verify(purgeListener).onIssuesRemoval(argument.capture()); - assertThat(argument.getValue()).containsOnly("ISSUE-1", "ISSUE-2"); + verify(purgeListener).onIssuesRemoval(projectUuid.capture(), issueKeys.capture()); + assertThat(projectUuid.getValue()).isEqualTo(THE_PROJECT_UUID); + assertThat(issueKeys.getValue()).containsOnly("ISSUE-1", "ISSUE-2"); } @Test public void should_delete_all_closed_issues() { dbTester.prepareDbUnit(getClass(), "should_delete_all_closed_issues.xml"); - underTest.purge(new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 0), PurgeListener.EMPTY, new PurgeProfiler()); + underTest.purge(new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, "1"), new String[0], 0), PurgeListener.EMPTY, new PurgeProfiler()); dbTester.assertDbUnit(getClass(), "should_delete_all_closed_issues-result.xml", "issues", "issue_changes"); } @@ -222,10 +227,10 @@ public class PurgeDaoTest { } private static PurgeConfiguration newConfigurationWith30Days() { - return new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30); + return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30); } private static PurgeConfiguration newConfigurationWith30Days(System2 system2) { - return new PurgeConfiguration(new IdUuidPair(1L, "1"), new String[0], 30, system2); + return new PurgeConfiguration(new IdUuidPair(THE_PROJECT_ID, THE_PROJECT_UUID), new String[0], 30, system2); } } diff --git a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml index 122cc876b8a..4dad1831f6c 100644 --- a/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml +++ b/sonar-db/src/test/resources/org/sonar/db/purge/PurgeDaoTest/should_delete_old_closed_issues-result.xml @@ -1,6 +1,6 @@ - @@ -20,7 +20,7 @@ do not purge --> - @@ -19,7 +19,7 @@ do not purge --> do not purge -->