From: Simon Brandhof <simon.brandhof@sonarsource.com>
Date: Thu, 13 Jul 2017 13:15:44 +0000 (+0200)
Subject: SONAR-9480 fix removal of rules from index when too many failures
X-Git-Tag: 6.5-RC1~1
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=7c1a881ae846e4c9afa22243c4fc5884d1df0d81;p=sonarqube.git

SONAR-9480 fix removal of rules from index when too many failures
---

diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java
index 8a9f9d6f290..edb24020163 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java
@@ -48,7 +48,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
 
@@ -138,14 +138,14 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    List<RuleKey> ruleKeys = items
+    Set<RuleKey> ruleKeys = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE;
       })
       .map(i -> RuleKey.parse(i.getDocId()))
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleKeys,
       // only index requests, no deletion requests.
@@ -158,8 +158,10 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
 
     // the remaining items reference rows that don't exist in db. They must
     // be deleted from index.
-    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString()));
-    ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString()));
+    ruleKeys.forEach(r -> {
+      bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString());
+      bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString());
+    });
 
     return bulkIndexer.stop();
   }
@@ -168,14 +170,14 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
     BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
     bulkIndexer.start();
 
-    List<RuleExtensionId> docIds = items
+    Set<RuleExtensionId> docIds = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION;
       })
       .map(RuleIndexer::explodeRuleExtensionDocId)
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
       // only index requests, no deletion requests.
diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
index eaede45f8aa..7cead64cb48 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
@@ -45,7 +45,7 @@ import org.sonar.server.es.StartupIndexer;
 
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
 
 public class UserIndexer implements StartupIndexer, ResilientIndexer {
@@ -113,14 +113,14 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
     if (items.isEmpty()) {
       return new IndexingResult();
     }
-    List<String> logins = items
+    Set<String> logins = items
       .stream()
       .filter(i -> {
         requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
         return i.getDocType() == EsQueueDto.Type.USER;
       })
       .map(EsQueueDto::getDocId)
-      .collect(toArrayList(items.size()));
+      .collect(toHashSet(items.size()));
 
     ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
     dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);
diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java
index b3f42589d13..423a185cacb 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java
@@ -307,7 +307,7 @@ public class RecoveryIndexerTest {
       .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid());
 
     assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 USER");
-    assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]");
+    assertThatLogsContain(INFO, "Elasticsearch recovery - 1 documents processed [0 failures]");
   }
 
   private class ProxyUserIndexer extends UserIndexer {