Fix Elasticsearch UT which is failing

- Reason for failing is that documents are soft deleted in Elasticsearch (Lucene underneath). This was affecting documents score, which were wrongly calculated. The fix is about recreating index for this specific tests as this is costly operation.

diff --git a/build.gradle b/build.gradle
index 8aabe14a27ccac36b17b35c3ef78c7fa173e2c9f..360e3e1568fa39a0ae1f84db734fbbe82e1f3559 100644 (file)
--- a/build.gradle
+++ b/build.gradle
@@ -312,6 +312,7 @@ subprojects {
       dependency 'org.elasticsearch.plugin:transport-netty4-client:7.16.0'
       dependency 'org.elasticsearch:mocksocket:1.0'
       dependency 'org.codelibs.elasticsearch.module:analysis-common:7.16.0'
+      dependency 'org.codelibs.elasticsearch.module:reindex:7.16.0'
       dependency 'org.eclipse.jgit:org.eclipse.jgit:5.9.0.202009080501-r'
       dependency 'org.tmatesoft.svnkit:svnkit:1.10.1'
       dependency 'org.hamcrest:hamcrest-all:1.3'

diff --git a/server/sonar-server-common/build.gradle b/server/sonar-server-common/build.gradle
index a6ffdd9d14008fa5c970d9575c56659f75fc27ea..4d768ea1f694522747930bf7088519954cf46e65 100644 (file)
--- a/server/sonar-server-common/build.gradle
+++ b/server/sonar-server-common/build.gradle
@@ -50,5 +50,6 @@ dependencies {
   testFixturesImplementation 'org.elasticsearch.plugin:transport-netty4-client'
 
   testFixturesImplementation 'org.codelibs.elasticsearch.module:analysis-common'
+  testFixturesImplementation 'org.codelibs.elasticsearch.module:reindex'
   testFixturesImplementation 'org.elasticsearch:mocksocket'
 }

diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java
index 006254c7c4ee5224a4815990d0078597a696ab19..79ea7a613bd6f69e4d6c73e85729abd96d26afa1 100644 (file)
--- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java
+++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java
@@ -45,6 +45,7 @@ import org.apache.http.HttpHost;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -74,11 +75,13 @@ import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortOrder;
@@ -168,6 +171,12 @@ public class EsTester extends ExternalResource {
     return new EsTester(true);
   }
 
+  public void recreateIndexes() {
+    deleteIndexIfExists(ALL_INDICES.getName());
+    CORE_INDICES_CREATED.set(false);
+    create();
+  }
+
   @Override
   protected void after() {
     if (isCustom) {
@@ -178,9 +187,17 @@ public class EsTester extends ExternalResource {
         .forEach(EsTester::deleteIndexIfExists);
     }
 
-    BulkIndexer.delete(ES_REST_CLIENT, IndexType.main(ALL_INDICES, "dummy"),
-      EsClient.prepareSearch(ALL_INDICES.getName())
-        .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())));
+    deleteAllDocumentsInIndexes();
+  }
+
+  private void deleteAllDocumentsInIndexes() {
+    try {
+      ES_REST_CLIENT.nativeClient()
+        .deleteByQuery(new DeleteByQueryRequest(ALL_INDICES.getName()).setQuery(QueryBuilders.matchAllQuery()).setRefresh(true).setWaitForActiveShards(1), RequestOptions.DEFAULT);
+      ES_REST_CLIENT.forcemerge(new ForceMergeRequest());
+    } catch (IOException e) {
+      throw new IllegalStateException("Could not delete data from _all indices", e);
+    }
   }
 
   private static String[] getIndicesNames() {
@@ -475,6 +492,9 @@ public class EsTester extends ExternalResource {
       .put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), Integer.MAX_VALUE)
       .put("logger.level", "INFO")
       .put("action.auto_create_index", false)
+      // allows to drop all indices at once using `_all`
+      // this parameter will default to true in ES 8.X
+      .put("action.destructive_requires_name", false)
       // Default the watermarks to absurdly low to prevent the tests
       // from failing on nodes without enough disk space
       .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
@@ -489,6 +509,7 @@ public class EsTester extends ExternalResource {
     Node node = new Node(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), null, null),
       ImmutableList.of(
         CommonAnalysisPlugin.class,
+        ReindexPlugin.class,
         // Netty4Plugin provides http and tcp transport
         Netty4Plugin.class,
         // install ParentJoin plugin required to create field of type "join"

diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java
index 490b7cfd519644901c1b3fc935c1b54a5cdbe3ed..419c10dc429046abf4f7ac670444641dc06b7cf8 100644 (file)
--- a/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java
+++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java
@@ -48,6 +48,7 @@ public class ComponentIndexScoreTest extends ComponentIndexTest {
 
   @Test
   public void should_prefer_key_matching_over_name_matching() {
+    es.recreateIndexes();
     ComponentDto project1 = indexProject("quality", "SonarQube");
     ComponentDto project2 = indexProject("sonarqube", "Quality Product");
 

diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle
index 2c359571cccf24fdb578b0bcd1cadd271b29a547..82a1c81657c8dfae72b2f834d92e40e088683f69 100644 (file)
--- a/sonar-application/build.gradle
+++ b/sonar-application/build.gradle
@@ -172,7 +172,6 @@ task zip(type: Zip, dependsOn: [configurations.compile, tasks.downloadLicenses,
     exclude '**/modules/mapper-version/**'
     exclude '**/modules/percolator/**'
     exclude '**/modules/rank-eval/**'
-    exclude '**/modules/reindex/**'
     exclude '**/modules/repositories-metering-api/**'
     exclude '**/modules/repository-encrypted/**'
     exclude '**/modules/repository-url/**'