From: Daniel Schwarz Date: Tue, 21 Feb 2017 10:09:35 +0000 (+0100) Subject: SONAR-8809 always update component index, instead of drop+reindex X-Git-Tag: 6.4-RC1~951 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=3716bfea7ebed5a9f50e11157c2adfbfce40d6e5;p=sonarqube.git SONAR-8809 always update component index, instead of drop+reindex --- diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java index f8d4d6117ec..196efb18e56 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java @@ -21,7 +21,6 @@ package org.sonar.db.purge; import com.google.common.collect.Lists; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -62,7 +61,7 @@ public class PurgeDao implements Dao { deleteAbortedAnalyses(rootUuid, commands); deleteDataOfComponentsWithoutHistoricalData(session, rootUuid, conf.scopesWithoutHistoricalData(), commands); purgeAnalyses(commands, rootUuid); - purgeDisabledComponents(session, conf.getDisabledComponentUuids(), listener); + purgeDisabledComponents(session, conf, listener); deleteOldClosedIssues(conf, mapper, listener); } @@ -123,18 +122,16 @@ public class PurgeDao implements Dao { purgeCommands.deleteComponentMeasures(analysisUuids, componentWithoutHistoricalDataUuids); } - private void purgeDisabledComponents(DbSession session, Collection uuids, PurgeListener listener) { + private void purgeDisabledComponents(DbSession session, PurgeConfiguration conf, PurgeListener listener) { PurgeMapper mapper = mapper(session); - executeLargeInputs(uuids, + executeLargeInputs(conf.getDisabledComponentUuids(), input -> { mapper.deleteFileSourcesByUuid(input); mapper.resolveComponentIssuesNotAlreadyResolved(input, system2.now()); return emptyList(); }); - for (String componentUuid : uuids) { - listener.onComponentDisabling(componentUuid); - } + listener.onComponentsDisabling(conf.rootProjectIdUuid().getUuid(), conf.getDisabledComponentUuids()); session.commit(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java index 4d5617fe102..1f5e374cbfd 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeListener.java @@ -19,13 +19,14 @@ */ package org.sonar.db.purge; +import java.util.Collection; import java.util.List; public interface PurgeListener { PurgeListener EMPTY = new PurgeListener() { @Override - public void onComponentDisabling(String uuid) { + public void onComponentsDisabling(String uuid, Collection disabledComponentUuids) { // do nothing } @@ -35,7 +36,7 @@ public interface PurgeListener { } }; - void onComponentDisabling(String uuid); + void onComponentsDisabling(String uuid, Collection disabledComponentUuids); void onIssuesRemoval(String projectUuid, List issueKeys); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index d0a4b9ffc4e..93654370790 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Throwables; import com.google.common.util.concurrent.Uninterruptibles; import java.util.Arrays; +import java.util.Collection; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -88,7 +89,6 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe case PROJECT_CREATION: case PROJECT_KEY_UPDATE: case NEW_ANALYSIS: - deleteProject(projectUuid); doIndexByProjectUuid(projectUuid); break; default: @@ -131,6 +131,13 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)))); } + public void delete(String projectUuid, Collection disabledComponentUuids) { + BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS); + bulk.start(); + disabledComponentUuids.stream().forEach(uuid -> bulk.addDeletion(INDEX_COMPONENTS, TYPE_COMPONENT, uuid, projectUuid)); + bulk.stop(); + } + void index(ComponentDto... docs) { Future submit = executor.submit(() -> indexNow(docs)); try { diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java index 55909ffd52d..bc288db7380 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/dbcleaner/IndexPurgeListener.java @@ -19,9 +19,11 @@ */ package org.sonar.server.computation.dbcleaner; +import java.util.Collection; import java.util.List; import org.sonar.api.server.ServerSide; import org.sonar.db.purge.PurgeListener; +import org.sonar.server.component.index.ComponentIndexer; import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.test.index.TestIndexer; @@ -29,14 +31,21 @@ import org.sonar.server.test.index.TestIndexer; public class IndexPurgeListener implements PurgeListener { private final TestIndexer testIndexer; private final IssueIndexer issueIndexer; + private final ComponentIndexer componentIndexer; - public IndexPurgeListener(TestIndexer testIndexer, IssueIndexer issueIndexer) { + public IndexPurgeListener(TestIndexer testIndexer, IssueIndexer issueIndexer, ComponentIndexer componentIndexer) { this.testIndexer = testIndexer; this.issueIndexer = issueIndexer; + this.componentIndexer = componentIndexer; } @Override - public void onComponentDisabling(String uuid) { + public void onComponentsDisabling(String projectUuid, Collection disabledComponentUuids) { + componentIndexer.delete(projectUuid, disabledComponentUuids); + disabledComponentUuids.forEach(this::onComponentDisabling); + } + + private void onComponentDisabling(String uuid) { testIndexer.deleteByFile(uuid); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index b3cb11a5ae6..0aaf343869b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -32,7 +32,6 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -128,7 +127,7 @@ public class BulkIndexer implements Startable { progress.start(); } - public void add(ActionRequest request) { + public void add(ActionRequest request) { bulkRequest.request().add(request); if (bulkRequest.request().estimatedSizeInBytes() >= flushByteSize) { executeBulk(); @@ -152,12 +151,12 @@ public class BulkIndexer implements Startable { while (true) { SearchHit[] hits = searchResponse.getHits().getHits(); for (SearchHit hit : hits) { - DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.index(), hit.type(), hit.getId()); SearchHitField routing = hit.field("_routing"); - if (routing != null) { - deleteRequestBuilder.setRouting(routing.getValue()); + if (routing == null) { + addDeletion(hit.index(), hit.type(), hit.getId()); + } else { + addDeletion(hit.index(), hit.type(), hit.getId(), routing.getValue()); } - add(deleteRequestBuilder.request()); } String scrollId = searchResponse.getScrollId(); @@ -169,6 +168,14 @@ public class BulkIndexer implements Startable { } } + public void addDeletion(String index, String type, String id) { + add(client.prepareDelete(index, type, id).request()); + } + + public void addDeletion(String index, String type, String id, String routing) { + add(client.prepareDelete(index, type, id).setRouting(routing).request()); + } + /** * Delete all the documents matching the given search request. This method is blocking. * Index is refreshed, so docs are not searchable as soon as method is executed. diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java index b79895cfcee..5ac5980c7c1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/dbcleaner/IndexPurgeListenerTest.java @@ -19,7 +19,10 @@ */ package org.sonar.server.computation.dbcleaner; +import java.util.Arrays; +import java.util.List; import org.junit.Test; +import org.sonar.server.component.index.ComponentIndexer; import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.test.index.TestIndexer; @@ -31,14 +34,19 @@ public class IndexPurgeListenerTest { TestIndexer testIndexer = mock(TestIndexer.class); IssueIndexer issueIndexer = mock(IssueIndexer.class); + ComponentIndexer componentIndexer = mock(ComponentIndexer.class); - IndexPurgeListener underTest = new IndexPurgeListener(testIndexer, issueIndexer); + IndexPurgeListener underTest = new IndexPurgeListener(testIndexer, issueIndexer, componentIndexer); @Test public void test_onComponentDisabling() { - underTest.onComponentDisabling("123456"); + String uuid = "123456"; + String projectUuid = "P789"; + List uuids = Arrays.asList(uuid); + underTest.onComponentsDisabling(projectUuid, uuids); - verify(testIndexer).deleteByFile("123456"); + verify(testIndexer).deleteByFile(uuid); + verify(componentIndexer).delete(projectUuid, uuids); } @Test