SONAR-8809 always update component index, instead of drop+reindex

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java
index f8d4d6117ec6ac3a0f3bb52b7bbe33ab572b92ec..196efb18e56a82e72a646c1ea2bc260d0ea6836c 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java
@@ -21,7 +21,6 @@ package org.sonar.db.purge;
 
 import com.google.common.collect.Lists;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
@@ -62,7 +61,7 @@ public class PurgeDao implements Dao {
     deleteAbortedAnalyses(rootUuid, commands);
     deleteDataOfComponentsWithoutHistoricalData(session, rootUuid, conf.scopesWithoutHistoricalData(), commands);
     purgeAnalyses(commands, rootUuid);
-    purgeDisabledComponents(session, conf.getDisabledComponentUuids(), listener);
+    purgeDisabledComponents(session, conf, listener);
     deleteOldClosedIssues(conf, mapper, listener);
   }
 
@@ -123,18 +122,16 @@ public class PurgeDao implements Dao {
     purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids);
   }
 
-  private void purgeDisabledComponents(DbSession session, Collection<String> uuids, PurgeListener listener) {
+  private void purgeDisabledComponents(DbSession session, PurgeConfiguration conf, PurgeListener listener) {
     PurgeMapper mapper = mapper(session);
-    executeLargeInputs(uuids,
+    executeLargeInputs(conf.getDisabledComponentUuids(),
       input -> {
         mapper.deleteFileSourcesByUuid(input);
         mapper.resolveComponentIssuesNotAlreadyResolved(input, system2.now());
         return emptyList();
       });
 
-    for (String componentUuid : uuids) {
-      listener.onComponentDisabling(componentUuid);
-    }
+    listener.onComponentsDisabling(conf.rootProjectIdUuid().getUuid(), conf.getDisabledComponentUuids());
 
     session.commit();
   }

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java
index 4d5617fe102fe5220519dff227b9d7057ce9f2f0..1f5e374cbfdbe26dc55cfc13eeed8ed70646d25d 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java
@@ -19,13 +19,14 @@
  */
 package org.sonar.db.purge;
 
+import java.util.Collection;
 import java.util.List;
 
 public interface PurgeListener {
 
   PurgeListener EMPTY = new PurgeListener() {
     @Override
-    public void onComponentDisabling(String uuid) {
+    public void onComponentsDisabling(String uuid, Collection<String> disabledComponentUuids) {
       // do nothing
     }
 
@@ -35,7 +36,7 @@ public interface PurgeListener {
     }
   };
 
-  void onComponentDisabling(String uuid);
+  void onComponentsDisabling(String uuid, Collection<String> disabledComponentUuids);
 
   void onIssuesRemoval(String projectUuid, List<String> issueKeys);
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java
index d0a4b9ffc4e80e0757e7f0f763e25e1db48d09c0..936543707904f09159d383cf500e6c08d6dea65d 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java
@@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Throwables;
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -88,7 +89,6 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
       case PROJECT_CREATION:
       case PROJECT_KEY_UPDATE:
       case NEW_ANALYSIS:
-        deleteProject(projectUuid);
         doIndexByProjectUuid(projectUuid);
         break;
       default:
@@ -131,6 +131,13 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
           termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid))));
   }
 
+  public void delete(String projectUuid, Collection<String> disabledComponentUuids) {
+    BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS);
+    bulk.start();
+    disabledComponentUuids.stream().forEach(uuid -> bulk.addDeletion(INDEX_COMPONENTS, TYPE_COMPONENT, uuid, projectUuid));
+    bulk.stop();
+  }
+
   void index(ComponentDto... docs) {
     Future<?> submit = executor.submit(() -> indexNow(docs));
     try {

diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java
index 55909ffd52d50e5842682f8acdef578c1b7741f9..bc288db7380c669a0053b7aa62218414a7c14ce5 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java
@@ -19,9 +19,11 @@
  */
 package org.sonar.server.computation.dbcleaner;
 
+import java.util.Collection;
 import java.util.List;
 import org.sonar.api.server.ServerSide;
 import org.sonar.db.purge.PurgeListener;
+import org.sonar.server.component.index.ComponentIndexer;
 import org.sonar.server.issue.index.IssueIndexer;
 import org.sonar.server.test.index.TestIndexer;
 
@@ -29,14 +31,21 @@ import org.sonar.server.test.index.TestIndexer;
 public class IndexPurgeListener implements PurgeListener {
   private final TestIndexer testIndexer;
   private final IssueIndexer issueIndexer;
+  private final ComponentIndexer componentIndexer;
 
-  public IndexPurgeListener(TestIndexer testIndexer, IssueIndexer issueIndexer) {
+  public IndexPurgeListener(TestIndexer testIndexer, IssueIndexer issueIndexer, ComponentIndexer componentIndexer) {
     this.testIndexer = testIndexer;
     this.issueIndexer = issueIndexer;
+    this.componentIndexer = componentIndexer;
   }
 
   @Override
-  public void onComponentDisabling(String uuid) {
+  public void onComponentsDisabling(String projectUuid, Collection<String> disabledComponentUuids) {
+    componentIndexer.delete(projectUuid, disabledComponentUuids);
+    disabledComponentUuids.forEach(this::onComponentDisabling);
+  }
+
+  private void onComponentDisabling(String uuid) {
     testIndexer.deleteByFile(uuid);
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
index b3cb11a5ae6a3cd63334b281f1991fa07bb00b04..0aaf343869bd9ed47ace1c3f6bb245faeb37f1ca 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
@@ -32,7 +32,6 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -128,7 +127,7 @@ public class BulkIndexer implements Startable {
     progress.start();
   }
 
-  public void add(ActionRequest request) {
+  public void add(ActionRequest<?> request) {
     bulkRequest.request().add(request);
     if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) {
       executeBulk();
@@ -152,12 +151,12 @@ public class BulkIndexer implements Startable {
     while (true) {
       SearchHit[] hits = searchResponse.getHits().getHits();
       for (SearchHit hit : hits) {
-        DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.index(), hit.type(), hit.getId());
         SearchHitField routing = hit.field("_routing");
-        if (routing != null) {
-          deleteRequestBuilder.setRouting(routing.getValue());
+        if (routing == null) {
+          addDeletion(hit.index(), hit.type(), hit.getId());
+        } else {
+          addDeletion(hit.index(), hit.type(), hit.getId(), routing.getValue());
         }
-        add(deleteRequestBuilder.request());
       }
 
       String scrollId = searchResponse.getScrollId();
@@ -169,6 +168,14 @@ public class BulkIndexer implements Startable {
     }
   }
 
+  public void addDeletion(String index, String type, String id) {
+    add(client.prepareDelete(index, type, id).request());
+  }
+
+  public void addDeletion(String index, String type, String id, String routing) {
+    add(client.prepareDelete(index, type, id).setRouting(routing).request());
+  }
+
   /**
    * Delete all the documents matching the given search request. This method is blocking.
    * Index is refreshed, so docs are not searchable as soon as method is executed.

diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java
index b79895cfcee1311c0b8b2c3e2efa6b9cb7c6832d..5ac5980c7c14927865733153b8ed7bb1aa288143 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java
@@ -19,7 +19,10 @@
  */
 package org.sonar.server.computation.dbcleaner;
 
+import java.util.Arrays;
+import java.util.List;
 import org.junit.Test;
+import org.sonar.server.component.index.ComponentIndexer;
 import org.sonar.server.issue.index.IssueIndexer;
 import org.sonar.server.test.index.TestIndexer;
 
@@ -31,14 +34,19 @@ public class IndexPurgeListenerTest {
 
   TestIndexer testIndexer = mock(TestIndexer.class);
   IssueIndexer issueIndexer = mock(IssueIndexer.class);
+  ComponentIndexer componentIndexer = mock(ComponentIndexer.class);
 
-  IndexPurgeListener underTest = new IndexPurgeListener(testIndexer, issueIndexer);
+  IndexPurgeListener underTest = new IndexPurgeListener(testIndexer, issueIndexer, componentIndexer);
 
   @Test
   public void test_onComponentDisabling() {
-    underTest.onComponentDisabling("123456");
+    String uuid = "123456";
+    String projectUuid = "P789";
+    List<String> uuids = Arrays.asList(uuid);
+    underTest.onComponentsDisabling(projectUuid, uuids);
 
-    verify(testIndexer).deleteByFile("123456");
+    verify(testIndexer).deleteByFile(uuid);
+    verify(componentIndexer).delete(projectUuid, uuids);
   }
 
   @Test