From 6f5086ae866864ec4f70128a2d05db1351f617be Mon Sep 17 00:00:00 2001 From: Jacek Date: Tue, 29 Mar 2022 11:07:36 +0200 Subject: [PATCH] Fix Elasticsearch UT which is failing - Reason for failing is that documents are soft deleted in Elasticsearch (Lucene underneath). This was affecting documents score, which were wrongly calculated. The fix is about recreating index for this specific tests as this is costly operation. --- build.gradle | 1 + server/sonar-server-common/build.gradle | 1 + .../java/org/sonar/server/es/EsTester.java | 27 ++++++++++++++++--- .../index/ComponentIndexScoreTest.java | 1 + sonar-application/build.gradle | 1 - 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 8aabe14a27c..360e3e1568f 100644 --- a/build.gradle +++ b/build.gradle @@ -312,6 +312,7 @@ subprojects { dependency 'org.elasticsearch.plugin:transport-netty4-client:7.16.0' dependency 'org.elasticsearch:mocksocket:1.0' dependency 'org.codelibs.elasticsearch.module:analysis-common:7.16.0' + dependency 'org.codelibs.elasticsearch.module:reindex:7.16.0' dependency 'org.eclipse.jgit:org.eclipse.jgit:5.9.0.202009080501-r' dependency 'org.tmatesoft.svnkit:svnkit:1.10.1' dependency 'org.hamcrest:hamcrest-all:1.3' diff --git a/server/sonar-server-common/build.gradle b/server/sonar-server-common/build.gradle index a6ffdd9d140..4d768ea1f69 100644 --- a/server/sonar-server-common/build.gradle +++ b/server/sonar-server-common/build.gradle @@ -50,5 +50,6 @@ dependencies { testFixturesImplementation 'org.elasticsearch.plugin:transport-netty4-client' testFixturesImplementation 'org.codelibs.elasticsearch.module:analysis-common' + testFixturesImplementation 'org.codelibs.elasticsearch.module:reindex' testFixturesImplementation 'org.elasticsearch:mocksocket' } diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java index 006254c7c4e..79ea7a613bd 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java @@ -45,6 +45,7 @@ import org.apache.http.HttpHost; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BulkRequest; @@ -74,11 +75,13 @@ import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -168,6 +171,12 @@ public class EsTester extends ExternalResource { return new EsTester(true); } + public void recreateIndexes() { + deleteIndexIfExists(ALL_INDICES.getName()); + CORE_INDICES_CREATED.set(false); + create(); + } + @Override protected void after() { if (isCustom) { @@ -178,9 +187,17 @@ public class EsTester extends ExternalResource { .forEach(EsTester::deleteIndexIfExists); } - BulkIndexer.delete(ES_REST_CLIENT, IndexType.main(ALL_INDICES, "dummy"), - EsClient.prepareSearch(ALL_INDICES.getName()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))); + deleteAllDocumentsInIndexes(); + } + + private void deleteAllDocumentsInIndexes() { + try { + ES_REST_CLIENT.nativeClient() + .deleteByQuery(new DeleteByQueryRequest(ALL_INDICES.getName()).setQuery(QueryBuilders.matchAllQuery()).setRefresh(true).setWaitForActiveShards(1), RequestOptions.DEFAULT); + ES_REST_CLIENT.forcemerge(new ForceMergeRequest()); + } catch (IOException e) { + throw new IllegalStateException("Could not delete data from _all indices", e); + } } private static String[] getIndicesNames() { @@ -475,6 +492,9 @@ public class EsTester extends ExternalResource { .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put("logger.level", "INFO") .put("action.auto_create_index", false) + // allows to drop all indices at once using `_all` + // this parameter will default to true in ES 8.X + .put("action.destructive_requires_name", false) // Default the watermarks to absurdly low to prevent the tests // from failing on nodes without enough disk space .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b") @@ -489,6 +509,7 @@ public class EsTester extends ExternalResource { Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), null, null), ImmutableList.of( CommonAnalysisPlugin.class, + ReindexPlugin.class, // Netty4Plugin provides http and tcp transport Netty4Plugin.class, // install ParentJoin plugin required to create field of type "join" diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java index 490b7cfd519..419c10dc429 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java @@ -48,6 +48,7 @@ public class ComponentIndexScoreTest extends ComponentIndexTest { @Test public void should_prefer_key_matching_over_name_matching() { + es.recreateIndexes(); ComponentDto project1 = indexProject("quality", "SonarQube"); ComponentDto project2 = indexProject("sonarqube", "Quality Product"); diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle index 2c359571ccc..82a1c81657c 100644 --- a/sonar-application/build.gradle +++ b/sonar-application/build.gradle @@ -172,7 +172,6 @@ task zip(type: Zip, dependsOn: [configurations.compile, tasks.downloadLicenses, exclude '**/modules/mapper-version/**' exclude '**/modules/percolator/**' exclude '**/modules/rank-eval/**' - exclude '**/modules/reindex/**' exclude '**/modules/repositories-metering-api/**' exclude '**/modules/repository-encrypted/**' exclude '**/modules/repository-url/**' -- 2.39.5