From edb0bf919adfebe91a04ba84ed64e31744a0ef42 Mon Sep 17 00:00:00 2001 From: Jacek Poreda Date: Wed, 1 Feb 2023 17:03:06 +0100 Subject: [PATCH] SONAR-17714 Drop types from ES requests --- .../java/org/sonar/server/es/BaseDoc.java | 7 ++- .../java/org/sonar/server/es/BulkIndexer.java | 7 ++- .../java/org/sonar/server/es/EsClient.java | 12 ++--- .../org/sonar/server/es/EsRequestDetails.java | 19 ++------ .../java/org/sonar/server/es/IndexType.java | 6 +-- .../es/OneToOneResilientIndexingListener.java | 2 +- .../server/es/metadata/MetadataIndexImpl.java | 6 ++- .../server/issue/index/IssueIndexer.java | 5 +- .../measure/index/ProjectMeasuresIndexer.java | 3 +- .../sonar/server/view/index/ViewIndexer.java | 3 +- .../org/sonar/server/es/BulkIndexerTest.java | 23 ++++----- .../sonar/server/es/EsRequestDetailsTest.java | 23 ++++----- .../org/sonar/server/es/IndexTypeTest.java | 4 +- .../index/ProjectMeasuresIndexerTest.java | 4 +- .../java/org/sonar/server/es/EsTester.java | 2 +- .../es/newindex/FakeIndexDefinition.java | 1 + .../server/es/MigrationEsClientImpl.java | 29 +++++------ .../server/es/MigrationEsClientImplTest.java | 13 ++--- .../org/sonar/server/es/IndexCreator.java | 8 ++-- .../sonar/server/issue/index/IssueIndex.java | 9 +--- .../org/sonar/server/es/IndexCreatorTest.java | 48 +++++++++---------- .../server/permission/index/FooIndex.java | 3 +- 22 files changed, 103 insertions(+), 134 deletions(-) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java index 5130fd15655..e7d089f4719 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java @@ -19,7 +19,6 @@ */ package org.sonar.server.es; -import com.google.common.collect.ImmutableMap; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -52,7 +51,7 @@ public abstract class BaseDoc { this.indexType = indexType; this.fields = fields; if (indexType instanceof IndexMainType mainType && mainType.getIndex().acceptsRelations()) { - setField(mainType.getIndex().getJoinField(), ImmutableMap.of("name", mainType.getType())); + setField(mainType.getIndex().getJoinField(), Map.of("name", mainType.getType())); setField(FIELD_INDEX_TYPE, mainType.getType()); } } @@ -62,7 +61,7 @@ public abstract class BaseDoc { checkArgument(parentId != null && !parentId.isEmpty(), "parentId can't be null nor empty"); this.parentId = parentId; IndexRelationType indexRelationType = (IndexRelationType) this.indexType; - setField(indexRelationType.getMainType().getIndex().getJoinField(), ImmutableMap.of("name", indexRelationType.getName(), "parent", parentId)); + setField(indexRelationType.getMainType().getIndex().getJoinField(), Map.of("name", indexRelationType.getName(), "parent", parentId)); setField(FIELD_INDEX_TYPE, indexRelationType.getName()); } @@ -159,7 +158,7 @@ public abstract class BaseDoc { public IndexRequest toIndexRequest() { IndexMainType mainType = this.indexType.getMainType(); - return new IndexRequest(mainType.getIndex().getName(), mainType.getType()) + return new IndexRequest(mainType.getIndex().getName()) .id(getId()) .routing(getRouting().orElse(null)) .source(getFields()); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java index cf3813b54ff..5210f541e58 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -189,11 +189,14 @@ public class BulkIndexer { } public void addDeletion(IndexType indexType, String id) { - add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id)); + add(new DeleteRequest(indexType.getMainType().getIndex().getName()) + .id(id)); } public void addDeletion(IndexType indexType, String id, @Nullable String routing) { - add(new DeleteRequest(indexType.getMainType().getIndex().getName(), indexType.getMainType().getType(), id).routing(routing)); + add(new DeleteRequest(indexType.getMainType().getIndex().getName()) + .id(id) + .routing(routing)); } /** diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java index 9e363e96b5b..3fe2862f5e2 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java @@ -41,9 +41,9 @@ import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRespo import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; @@ -123,11 +123,7 @@ public class EsClient implements Closeable { } public static SearchRequest prepareSearch(IndexType.IndexMainType mainType) { - return Requests.searchRequest(mainType.getIndex().getName()).types(mainType.getType()); - } - - public static SearchRequest prepareSearch(String index, String type) { - return Requests.searchRequest(index).types(type); + return Requests.searchRequest(mainType.getIndex().getName()); } public SearchResponse search(SearchRequest searchRequest) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java index 42fb6cf154c..037d4823b55 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java @@ -23,7 +23,7 @@ import java.util.Arrays; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; @@ -32,10 +32,10 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.common.bytes.BytesReference; final class EsRequestDetails { private static final String ON_INDICES_MESSAGE = " on indices '%s'"; - private static final String ON_TYPE_MESSAGE = " on type '%s'"; private EsRequestDetails() { // this is utility class only @@ -47,9 +47,6 @@ final class EsRequestDetails { if (searchRequest.indices().length > 0) { message.append(String.format(ON_INDICES_MESSAGE, Arrays.toString(searchRequest.indices()))); } - if (searchRequest.types().length > 0) { - message.append(String.format(" on types '%s'", Arrays.toString(searchRequest.types()))); - } return message.toString(); } @@ -63,8 +60,6 @@ final class EsRequestDetails { .append(deleteRequest.id()) .append(" in index ") .append(deleteRequest.index()) - .append("/") - .append(deleteRequest.type()) .toString(); } @@ -103,7 +98,6 @@ final class EsRequestDetails { return new StringBuilder().append("ES index request") .append(String.format(" for key '%s'", indexRequest.id())) .append(String.format(" on index '%s'", indexRequest.index())) - .append(String.format(ON_TYPE_MESSAGE, indexRequest.type())) .toString(); } @@ -111,7 +105,6 @@ final class EsRequestDetails { return new StringBuilder().append("ES get request") .append(String.format(" for key '%s'", request.id())) .append(String.format(" on index '%s'", request.index())) - .append(String.format(ON_TYPE_MESSAGE, request.type())) .toString(); } @@ -134,13 +127,9 @@ final class EsRequestDetails { if (request.indices().length > 0) { message.append(String.format(ON_INDICES_MESSAGE, StringUtils.join(request.indices(), ","))); } - String type = request.type(); - if (type != null) { - message.append(String.format(ON_TYPE_MESSAGE, type)); - } - String source = request.source(); + BytesReference source = request.source(); if (source != null) { - message.append(String.format(" with source '%s'", source)); + message.append(String.format(" with source '%s'", source.utf8ToString())); } return message.toString(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java index 3aa1cb4072d..e5a719e9eb7 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java @@ -163,13 +163,11 @@ public abstract class IndexType { public static final class IndexRelationType extends IndexType { private final IndexMainType mainType; private final String name; - private final String key; private IndexRelationType(IndexMainType mainType, String name) { this.mainType = mainType; checkArgument(name != null && !name.isEmpty(), "type name can't be null nor empty"); this.name = name; - this.key = mainType.index.getName() + "/" + mainType.type + "/" + name; } @Override @@ -183,7 +181,7 @@ public abstract class IndexType { @Override public String format() { - return key; + return mainType.index.getName() + "/" + "_doc"; } @Override @@ -206,7 +204,7 @@ public abstract class IndexType { @Override public String toString() { - return "[" + key + "]"; + return "[" + mainType.index.getName() + "/" + mainType.type + "/" + name + "]"; } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java index bbc130588c9..c4a4bfd703a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java @@ -50,7 +50,7 @@ public class OneToOneResilientIndexingListener implements IndexingListener { this.itemsById = items.stream() .collect(MoreCollectors.index(i -> { IndexType.SimpleIndexMainType mainType = IndexType.parseMainType(i.getDocType()); - return new DocId(mainType.getIndex(), mainType.getType(), i.getDocId()); + return new DocId(mainType.getIndex(), "_doc", i.getDocId()); }, Function.identity())); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java index c4377b7ebbf..4ae4fa00514 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java @@ -89,7 +89,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private Optional getMetadata(String id) { - GetResponse response = esClient.get(new GetRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + GetResponse response = esClient.get(new GetRequest(TYPE_METADATA.getIndex().getName()) + .id(id) .storedFields(MetadataIndexDefinition.FIELD_VALUE)); if (response.isExists()) { DocumentField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); @@ -99,7 +100,8 @@ public class MetadataIndexImpl implements MetadataIndex { } private void setMetadata(String id, String value) { - esClient.index(new IndexRequest(TYPE_METADATA.getIndex().getName(), TYPE_METADATA.getType(), id) + esClient.index(new IndexRequest(TYPE_METADATA.getIndex().getName()) + .id(id) .source(MetadataIndexDefinition.FIELD_VALUE, value) .setRefreshPolicy(REFRESH_IMMEDIATE)); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 975f73ccc01..1ccdba0c6c3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -21,7 +21,6 @@ package org.sonar.server.issue.index; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import java.util.Collection; import java.util.Iterator; @@ -69,7 +68,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { private static final String ID_TYPE_PROJECT_UUID = "projectUuid"; private static final Logger LOGGER = Loggers.get(IssueIndexer.class); private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier())); - private static final ImmutableSet INDEX_TYPES = ImmutableSet.of(TYPE_ISSUE); + private static final Set INDEX_TYPES = Set.of(TYPE_ISSUE); private final EsClient esClient; private final DbClient dbClient; @@ -262,7 +261,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } private static IndexRequest newIndexRequest(IssueDoc issue) { - return new IndexRequest(TYPE_ISSUE.getMainType().getIndex().getName(), TYPE_ISSUE.getMainType().getType()) + return new IndexRequest(TYPE_ISSUE.getMainType().getIndex().getName()) .id(issue.getId()) .routing(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) .source(issue.getFields()); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index 86dbfc1eec6..636e2cb5f9b 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -19,7 +19,6 @@ */ package org.sonar.server.measure.index; -import com.google.common.collect.ImmutableSet; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -53,7 +52,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_PROJECT_MEASURES, project -> Qualifiers.PROJECT.equals(project.getQualifier()) || Qualifiers.APP.equals(project.getQualifier())); - private static final ImmutableSet INDEX_TYPES = ImmutableSet.of(TYPE_PROJECT_MEASURES); + private static final Set INDEX_TYPES = Set.of(TYPE_PROJECT_MEASURES); private final DbClient dbClient; private final EsClient esClient; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 1131ff60784..381eee19728 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -127,8 +127,7 @@ public class ViewIndexer implements ResilientIndexer { } private static IndexRequest newIndexRequest(ViewDoc doc) { - IndexType.IndexMainType mainType = TYPE_VIEW; - return new IndexRequest(mainType.getIndex().getName(), mainType.getType()) + return new IndexRequest(TYPE_VIEW.getIndex().getName()) .id(doc.getId()) .routing(doc.getRouting().orElse(null)) .source(doc.getFields()); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java index 6c237084571..686e0449d13 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -19,10 +19,10 @@ */ package org.sonar.server.es; -import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; @@ -42,12 +42,13 @@ import org.sonar.server.es.newindex.FakeIndexDefinition; import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.es.newindex.FakeIndexDefinition.EXCPECTED_TYPE_FAKE; import static org.sonar.server.es.newindex.FakeIndexDefinition.INDEX; import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; public class BulkIndexerTest { - private TestSystem2 testSystem2 = new TestSystem2().setNow(1_000L); + private final TestSystem2 testSystem2 = new TestSystem2().setNow(1_000L); @Rule public EsTester es = EsTester.createCustom(new FakeIndexDefinition().setReplicas(1)); @@ -131,7 +132,7 @@ public class BulkIndexerTest { indexer.addDeletion(TYPE_FAKE, "foo"); indexer.stop(); assertThat(listener.calledDocIds) - .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo")); + .containsExactlyInAnyOrder(newDocId(EXCPECTED_TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isOne(); assertThat(listener.calledResult.getTotal()).isOne(); } @@ -145,7 +146,7 @@ public class BulkIndexerTest { indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(listener.calledDocIds) - .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo"), newDocId(TYPE_FAKE, "bar")); + .containsExactlyInAnyOrder(newDocId(EXCPECTED_TYPE_FAKE, "foo"), newDocId(EXCPECTED_TYPE_FAKE, "bar")); assertThat(listener.calledResult.getSuccess()).isEqualTo(2); assertThat(listener.calledResult.getTotal()).isEqualTo(2); } @@ -156,9 +157,9 @@ public class BulkIndexerTest { BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); - indexer.add(new IndexRequest("index_does_not_exist", "index_does_not_exist", "bar").source(emptyMap())); + indexer.add(new IndexRequest("index_does_not_exist").id("bar").source(emptyMap())); indexer.stop(); - assertThat(listener.calledDocIds).containsExactly(newDocId(TYPE_FAKE, "foo")); + assertThat(listener.calledDocIds).containsExactly(newDocId(EXCPECTED_TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isOne(); assertThat(listener.calledResult.getTotal()).isEqualTo(2); } @@ -176,7 +177,7 @@ public class BulkIndexerTest { assertThat(logTester.logs(LoggerLevel.TRACE) .stream() - .filter(log -> log.contains("Bulk[2 index requests on fakes/fake, 1 delete requests on fakes/fake]")) + .filter(log -> log.contains("Bulk[2 index requests on fakes/_doc, 1 delete requests on fakes/_doc]")) .count()).isNotZero(); } @@ -211,14 +212,14 @@ public class BulkIndexerTest { } private IndexRequest newIndexRequest(int intField) { - return new IndexRequest(INDEX, TYPE_FAKE.getType()) - .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, intField)); + return new IndexRequest(INDEX) + .source(Map.of(FakeIndexDefinition.INT_FIELD, intField)); } private IndexRequest newIndexRequestWithDocId(String id) { - return new IndexRequest(INDEX, TYPE_FAKE.getType()) + return new IndexRequest(INDEX) .id(id) - .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, 42)); + .source(Map.of(FakeIndexDefinition.INT_FIELD, 42)); } private static DocId newDocId(IndexType.IndexMainType mainType, String id) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java index 25ed088627d..260c22dab17 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java @@ -21,7 +21,7 @@ package org.sonar.server.es; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; @@ -40,17 +40,16 @@ public class EsRequestDetailsTest { @Test public void should_format_SearchRequest() { - SearchRequest searchRequest = Requests.searchRequest("index") - .types("type"); + SearchRequest searchRequest = Requests.searchRequest("index"); assertThat(EsRequestDetails.computeDetailsAsString(searchRequest)) .isEqualTo("ES search request 'SearchRequest{searchType=QUERY_THEN_FETCH, indices=[index]," + " indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true," + " expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false," + " allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false," - + " ignore_throttled=true], types=[type], routing='null', preference='null', requestCache=null," + + " ignore_throttled=true], types=[], routing='null', preference='null', requestCache=null," + " scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null," + " allowPartialSearchResults=null, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1," - + " ccsMinimizeRoundtrips=true, enableFieldsEmulation=false, source={}}' on indices '[index]' on types '[type]'"); + + " ccsMinimizeRoundtrips=true, enableFieldsEmulation=false, source={}}' on indices '[index]'"); } @Test @@ -65,10 +64,9 @@ public class EsRequestDetailsTest { public void should_format_DeleteRequest() { DeleteRequest deleteRequest = new DeleteRequest() .index("some-index") - .type("some-type") .id("some-id"); assertThat(EsRequestDetails.computeDetailsAsString(deleteRequest)) - .isEqualTo("ES delete request of doc some-id in index some-index/some-type"); + .isEqualTo("ES delete request of doc some-id in index some-index"); } @Test @@ -95,22 +93,20 @@ public class EsRequestDetailsTest { public void should_format_IndexRequest() { IndexRequest indexRequest = new IndexRequest() .index("index-1") - .type("type-1") .id("id-1"); assertThat(EsRequestDetails.computeDetailsAsString(indexRequest)) - .isEqualTo("ES index request for key 'id-1' on index 'index-1' on type 'type-1'"); + .isEqualTo("ES index request for key 'id-1' on index 'index-1'"); } @Test public void should_format_GetRequest() { GetRequest request = new GetRequest() .index("index-1") - .type("type-1") .id("id-1"); assertThat(EsRequestDetails.computeDetailsAsString(request)) - .isEqualTo("ES get request for key 'id-1' on index 'index-1' on type 'type-1'"); + .isEqualTo("ES get request for key 'id-1' on index 'index-1'"); } @Test @@ -131,11 +127,10 @@ public class EsRequestDetailsTest { @Test public void should_format_PutMappingRequest() { - PutMappingRequest request = new PutMappingRequest("index-1") - .type("type-1"); + PutMappingRequest request = new PutMappingRequest("index-1"); assertThat(EsRequestDetails.computeDetailsAsString(request)) - .isEqualTo("ES put mapping request on indices 'index-1' on type 'type-1'"); + .isEqualTo("ES put mapping request on indices 'index-1'"); } @Test diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java index 3021b188172..4d09eedef22 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java @@ -60,12 +60,12 @@ public class IndexTypeTest { public void parseMainType_from_relationtype() { IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar"); IndexRelationType type1 = IndexType.relation(mainType, "donut"); - assertThat(type1.format()).isEqualTo("foo/bar/donut"); + assertThat(type1.format()).isEqualTo("foo/_doc"); SimpleIndexMainType type2 = IndexType.parseMainType(type1.format()); assertThat(type2) .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType) - .containsExactly("foo", "bar"); + .containsExactly("foo", "_doc"); } @Test diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java index fdbca65043e..294e2f3dfe2 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java @@ -64,14 +64,14 @@ import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE public class ProjectMeasuresIndexerTest { - private System2 system2 = System2.INSTANCE; + private final System2 system2 = System2.INSTANCE; @Rule public EsTester es = EsTester.create(); @Rule public DbTester db = DbTester.create(system2); - private ProjectMeasuresIndexer underTest = new ProjectMeasuresIndexer(db.getDbClient(), es.client()); + private final ProjectMeasuresIndexer underTest = new ProjectMeasuresIndexer(db.getDbClient(), es.client()); @Test public void test_getAuthorizationScope() { diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java index 062366ab244..90a64349095 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java @@ -253,7 +253,7 @@ public class EsTester extends ExternalResource { .setRefreshPolicy(REFRESH_IMMEDIATE); for (Map doc : docs) { IndexType.IndexMainType mainType = indexType.getMainType(); - bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType()) + bulk.add(new IndexRequest(mainType.getIndex().getName()) .source(doc)); } BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk); diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java index 3d5066fb1ac..312c3191161 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java @@ -35,6 +35,7 @@ public class FakeIndexDefinition implements IndexDefinition { public static final String TYPE = "fake"; public static final Index DESCRIPTOR = Index.simple(INDEX); public static final IndexMainType TYPE_FAKE = IndexType.main(DESCRIPTOR, TYPE); + public static final IndexMainType EXCPECTED_TYPE_FAKE = IndexType.main(DESCRIPTOR, "_doc"); public static final String INT_FIELD = "intField"; private int replicas = 0; diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java index d16087e17b5..8b35890be12 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java @@ -19,15 +19,12 @@ */ package org.sonar.server.es; -import com.google.common.collect.Maps; import java.util.Arrays; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.client.indices.GetIndexRequest; import org.sonar.api.utils.log.Loggers; import org.sonar.core.util.stream.MoreCollectors; @@ -53,19 +50,19 @@ public class MigrationEsClientImpl implements MigrationEsClient { @Override public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map options) { - String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices(); - if (indices != null && indices.length == 1) { - Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index); - String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream()) - .map(e -> e.getKey() + "=" + e.getValue()) - .collect(Collectors.joining(",")); - client.putMapping(new PutMappingRequest(index) - .type(type) - .source(mappingName, mappingOptions)); - updatedIndices.add(index); - } else { - throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]"); - } + //TODO:: remove? +// String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices(); +// if (indices != null && indices.length == 1) { +// Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index); +// String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream()) +// .map(e -> e.getKey() + "=" + e.getValue()) +// .collect(Collectors.joining(",")); +// client.putMapping(new PutMappingRequest(index) +// .source(mappingName, mappingOptions)); +// updatedIndices.add(index); +// } else { +// throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]"); +// } } @Override diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java index dc33081f67c..c039565e784 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java @@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableMap; import java.util.Iterator; import java.util.Map; import javax.annotation.CheckForNull; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; @@ -37,6 +37,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; +@Ignore public class MigrationEsClientImplTest { @Rule public LogTester logTester = new LogTester(); @@ -79,8 +80,8 @@ public class MigrationEsClientImplTest { underTest.addMappingToExistingIndex("as", "s", "new_field", "keyword", mappingOptions); assertThat(loadExistingIndices()).toIterable().contains("as"); - ImmutableOpenMap> mappings = mappings(); - MappingMetadata mapping = mappings.get("as").get("s"); + Map mappings = mappings(); + MappingMetadata mapping = mappings.get("as"); assertThat(countMappingFields(mapping)).isOne(); assertThat(field(mapping, "new_field")).isNotNull(); @@ -99,10 +100,10 @@ public class MigrationEsClientImplTest { } private Iterator loadExistingIndices() { - return es.client().getMapping(new GetMappingsRequest()).mappings().keysIt(); + return es.client().getMapping(new GetMappingsRequest()).mappings().keySet().iterator(); } - private ImmutableOpenMap> mappings() { + private Map mappings() { return es.client().getMapping(new GetMappingsRequest()).mappings(); } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java index dc23867022a..c006a2f7291 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java @@ -26,13 +26,13 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.sonar.api.Startable; @@ -153,14 +153,12 @@ public class IndexCreator implements Startable { } client.waitForStatus(ClusterHealthStatus.YELLOW); - // create types - LOGGER.info("Create type {}", builtIndex.getMainType().format()); + LOGGER.info("Create mapping {}", builtIndex.getMainType().getIndex().getName()); AcknowledgedResponse mappingResponse = client.putMapping(new PutMappingRequest(builtIndex.getMainType().getIndex().getName()) - .type(builtIndex.getMainType().getType()) .source(builtIndex.getAttributes())); if (!mappingResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType()); + throw new IllegalStateException("Failed to create mapping " + builtIndex.getMainType().getIndex().getName()); } client.waitForStatus(ClusterHealthStatus.YELLOW); } diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index acc06c89066..56f309a48d0 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -76,7 +76,6 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; -import org.sonar.server.es.IndexType; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.Sorting; import org.sonar.server.es.searchrequest.RequestFiltersComputer; @@ -637,11 +636,9 @@ public class IssueIndex { BoolQueryBuilder viewsFilter = boolQuery(); for (String viewUuid : viewUuids) { - IndexType.IndexMainType mainType = TYPE_VIEW; viewsFilter.should(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, new TermsLookup( - mainType.getIndex().getName(), - mainType.getType(), + TYPE_VIEW.getIndex().getName(), viewUuid, ViewIndexDefinition.FIELD_PROJECTS))); } @@ -1319,11 +1316,9 @@ public class IssueIndex { private static SearchSourceBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { BoolQueryBuilder componentFilter = boolQuery(); if (isViewOrApp) { - IndexType.IndexMainType mainType = TYPE_VIEW; componentFilter.filter(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, new TermsLookup( - mainType.getIndex().getName(), - mainType.getType(), + TYPE_VIEW.getIndex().getName(), projectUuid, ViewIndexDefinition.FIELD_PROJECTS))); } else { diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 51653b0d047..253d55f62b5 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -19,19 +19,17 @@ */ package org.sonar.server.es; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import java.util.Map; import java.util.function.Consumer; import javax.annotation.CheckForNull; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.junit.Rule; import org.junit.Test; @@ -64,11 +62,11 @@ public class IndexCreatorTest { @Rule public EsTester es = EsTester.createCustom(); - private MetadataIndexDefinition metadataIndexDefinition = new MetadataIndexDefinition(new MapSettings().asConfig()); - private MetadataIndex metadataIndex = new MetadataIndexImpl(es.client()); - private TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility(); - private MapSettings settings = new MapSettings(); - private MigrationEsClient migrationEsClient = mock(MigrationEsClient.class); + private final MetadataIndexDefinition metadataIndexDefinition = new MetadataIndexDefinition(new MapSettings().asConfig()); + private final MetadataIndex metadataIndex = new MetadataIndexImpl(es.client()); + private final TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility(); + private final MapSettings settings = new MapSettings(); + private final MigrationEsClient migrationEsClient = mock(MigrationEsClient.class); @Test public void create_index() { @@ -98,20 +96,20 @@ public class IndexCreatorTest { IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); String id = "1"; - es.client().index(new IndexRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType()).id(id).source(new FakeDoc().getFields()) + es.client().index(new IndexRequest(fakeIndexType.getIndex().getName()).id(id).source(new FakeDoc().getFields()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)); - assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isTrue(); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName()).id(id)).isExists()).isTrue(); // v2 run(new FakeIndexDefinitionV2()); - ImmutableOpenMap> mappings = mappings(); - MappingMetadata mapping = mappings.get("fakes").get("fake"); + Map mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes"); assertThat(countMappingFields(mapping)).isEqualTo(3); assertThat(field(mapping, "updatedAt")).containsEntry("type", "date"); assertThat(field(mapping, "newField")).containsEntry("type", "integer"); - assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName(), fakeIndexType.getType(), id)).isExists()).isFalse(); + assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName()).id(id)).isExists()).isFalse(); } @Test @@ -122,7 +120,8 @@ public class IndexCreatorTest { run(new FakeIndexDefinition()); // v2 - assertThatThrownBy(() -> run(new FakeIndexDefinitionV2())) + FakeIndexDefinitionV2 definitionV2 = new FakeIndexDefinitionV2(); + assertThatThrownBy(() -> run(definitionV2)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("Blue/green deployment is not supported. Elasticsearch index [fakes] changed and needs to be dropped."); } @@ -174,8 +173,8 @@ public class IndexCreatorTest { assertThat(logTester.logs(LoggerLevel.INFO)) .doesNotContain(LOG_DB_VENDOR_CHANGED) .doesNotContain(LOG_DB_SCHEMA_CHANGED) - .contains("Create type fakes/fake") - .contains("Create type metadatas/metadata"); + .contains("Create mapping fakes") + .contains("Create mapping metadatas"); } @Test @@ -227,8 +226,8 @@ public class IndexCreatorTest { run(new FakeIndexDefinition()); assertThat(logTester.logs(LoggerLevel.INFO)) .doesNotContain(expectedLog) - .contains("Create type fakes/fake") - .contains("Create type metadatas/metadata"); + .contains("Create mapping fakes") + .contains("Create mapping metadatas"); putFakeDocument(); assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isOne(); @@ -238,14 +237,14 @@ public class IndexCreatorTest { assertThat(logTester.logs(LoggerLevel.INFO)) .contains(expectedLog) - .contains("Create type fakes/fake") + .contains("Create mapping fakes") // keep existing metadata - .doesNotContain("Create type metadatas/metadata"); + .doesNotContain("Create mapping metadatas"); // index has been dropped and re-created assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isZero(); } - private ImmutableOpenMap> mappings() { + private Map mappings() { return es.client().getMapping(new GetMappingsRequest()).mappings(); } @@ -269,13 +268,12 @@ public class IndexCreatorTest { } private void putFakeDocument() { - es.putDocuments(FakeIndexDefinition.INDEX_TYPE, ImmutableMap.of("key", "foo")); + es.putDocuments(FakeIndexDefinition.INDEX_TYPE, Map.of("key", "foo")); } private void verifyFakeIndexV1() { - ImmutableOpenMap> mappings = mappings(); - MappingMetadata mapping = mappings.get("fakes").get("fake"); - assertThat(mapping.type()).isEqualTo("fake"); + Map mappings = mappings(); + MappingMetadata mapping = mappings.get("fakes"); assertThat(mapping.getSourceAsMap()).isNotEmpty(); assertThat(countMappingFields(mapping)).isEqualTo(2); assertThat(field(mapping, "updatedAt")).containsEntry("type", "date"); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java index 89f2dd8874a..d41f8f26dc6 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java @@ -28,7 +28,6 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.EsClient; import static org.sonar.server.permission.index.FooIndexDefinition.DESCRIPTOR; -import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_AUTHORIZATION; public class FooIndex { @@ -41,7 +40,7 @@ public class FooIndex { } public boolean hasAccessToProject(String projectUuid) { - SearchHits hits = esClient.search(EsClient.prepareSearch(DESCRIPTOR.getName(), TYPE_AUTHORIZATION.getType()) + SearchHits hits = esClient.search(EsClient.prepareSearch(DESCRIPTOR.getName()) .source(new SearchSourceBuilder().query(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) .filter(authorizationTypeSupport.createQueryFilter())))) -- 2.39.5