From 0832a517c9b7cb70d51ba6557094a605a8a9efdc Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Mon, 23 Jan 2017 16:08:49 +0100 Subject: [PATCH] SONAR-8238 when updating a project, clear ALL components from index --- .../component/index/ComponentIndexer.java | 62 +++--- .../component/index/ComponentIndexerTest.java | 30 ++- .../org/sonar/db/component/ComponentDao.java | 11 +- .../sonar/db/component/ComponentMapper.java | 2 +- .../sonar/db/component/ComponentMapper.xml | 13 +- .../sonar/db/component/ComponentDaoTest.java | 35 ++++ .../ComponentDaoTest/selectForIndexing.xml | 185 ++++++++++++++++++ 7 files changed, 299 insertions(+), 39 deletions(-) create mode 100644 sonar-db/src/test/resources/org/sonar/db/component/ComponentDaoTest/selectForIndexing.xml diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index 9f628f9ac75..31a5e4c0e7d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -28,6 +28,7 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; import org.sonar.api.Startable; import org.sonar.db.DbClient; @@ -36,6 +37,9 @@ import org.sonar.db.component.ComponentDto; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; +import static java.util.Objects.requireNonNull; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_COMPONENTS; import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_AUTHORIZATION; import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; @@ -55,47 +59,53 @@ public class ComponentIndexer implements Startable { /** * Copy all components of all projects to the elastic search index. *

- * Warning: This should only be called on an empty index. It does not delete anything. + * Warning: This should only be called on an empty index and it should only be called during startup. */ public void index() { - try (DbSession dbSession = dbClient.openSession(false)) { - BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS); - bulk.setLarge(true); - bulk.start(); - dbClient.componentDao() - .selectAll(dbSession, context -> { - ComponentDto dto = (ComponentDto) context.getResultObject(); - bulk.add(newIndexRequest(toDocument(dto))); - }); - bulk.stop(); - } + doIndexByProjectUuid(null); } /** * Update the index for one specific project. The current data from the database is used. */ public void indexByProjectUuid(String projectUuid) { + requireNonNull(projectUuid); + deleteComponentsByProjectUuid(projectUuid); + doIndexByProjectUuid(projectUuid); + } + + /** + * @param projectUuid the uuid of the project to analyze, or null if all content should be indexed.
+ * Warning: only use null during startup. + */ + private void doIndexByProjectUuid(@Nullable String projectUuid) { + BulkIndexer bulk = new BulkIndexer(esClient, INDEX_COMPONENTS); + + // setLarge must be enabled only during server startup because it disables replicas + bulk.setLarge(projectUuid == null); + + bulk.start(); try (DbSession dbSession = dbClient.openSession(false)) { - deleteComponentsByProjectUuid(projectUuid); - index( - dbClient - .componentDao() - .selectByProjectUuid(projectUuid, dbSession) - .toArray(new ComponentDto[0])); + dbClient.componentDao() + .selectForIndexing(dbSession, projectUuid, context -> { + ComponentDto dto = (ComponentDto) context.getResultObject(); + bulk.add(newIndexRequest(toDocument(dto))); + }); } + bulk.stop(); } - public void deleteByProjectUuid(String uuid) { - deleteComponentsByProjectUuid(uuid); - deleteAuthorizationByProjectUuid(uuid); + public void deleteByProjectUuid(String projectUuid) { + requireNonNull(projectUuid); + deleteComponentsByProjectUuid(projectUuid); + deleteAuthorizationByProjectUuid(projectUuid); } private void deleteComponentsByProjectUuid(String projectUuid) { - esClient - .prepareDelete(INDEX_COMPONENTS, TYPE_COMPONENT, projectUuid) - .setRouting(projectUuid) - .setRefresh(true) - .get(); + BulkIndexer.delete(esClient, INDEX_COMPONENTS, esClient.prepareSearch(INDEX_COMPONENTS) + .setQuery(boolQuery() + .filter( + termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)))); } private void deleteAuthorizationByProjectUuid(String projectUuid) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java index 23bea430a5b..789105833f8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java @@ -101,7 +101,7 @@ public class ComponentIndexerTest { } @Test - public void reindex_project() { + public void index_and_update_and_reindex_project() { // insert ComponentDto component = ComponentTesting.newProjectDto(organization, "UUID-1").setName("OldName"); @@ -121,16 +121,36 @@ public class ComponentIndexerTest { assertMatches("NewName", 1); } + @Test + public void index_and_update_and_reindex_project_with_files() { + + // insert + ComponentDto project = dbTester.components().insertProject(); + ComponentDto file = dbTester.components().insertComponent(ComponentTesting.newFileDto(project).setName("OldFile")); + + // verify insert + index(project); + assertMatches("OldFile", 1); + + // modify + file.setName("NewFile"); + update(file); + + // verify modification + index(project); + assertMatches("OldFile", 0); + assertMatches("NewFile", 1); + } + private void insert(ComponentDto component) { - dbClient.componentDao().insert(dbSession, component); - dbSession.commit(); + dbTester.components().insertComponent(component); } private void update(ComponentDto component) { ComponentUpdateDto updateComponent = ComponentUpdateDto.copyFrom(component); updateComponent.setBChanged(true); dbClient.componentDao().update(dbSession, updateComponent); - dbClient.componentDao().applyBChangesForRootComponentUuid(dbSession, "UUID-1"); + dbClient.componentDao().applyBChangesForRootComponentUuid(dbSession, component.getRootUuid()); dbSession.commit(); } @@ -158,7 +178,7 @@ public class ComponentIndexerTest { .setQuery(termQuery(FIELD_NAME, nameQuery)) .get() .getHits() - .getHits().length).isEqualTo(numberOfMatches); + .getTotalHits()).isEqualTo(numberOfMatches); } private ComponentIndexer createIndexer() { diff --git a/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java b/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java index 8ec27d1c2a6..0bc027afb7a 100644 --- a/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java +++ b/sonar-db/src/main/java/org/sonar/db/component/ComponentDao.java @@ -28,6 +28,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import javax.annotation.Nullable; import org.apache.ibatis.session.ResultHandler; @@ -230,10 +231,14 @@ public class ComponentDao implements Dao { } /** - * Selects all enabled components. The result is not returned (since it is usually too big), but handed over to the handler + * Selects all components that are relevant for indexing. The result is not returned (since it is usually too big), but handed over to the handler + * @param session the database session + * @param projectUuid the project uuid, which is selected with all of its children + * @param handler the action to be applied to every result */ - public void selectAll(DbSession session, ResultHandler handler) { - mapper(session).selectAll(handler); + public void selectForIndexing(DbSession session, @Nullable String projectUuid, ResultHandler handler) { + Objects.requireNonNull(handler); + mapper(session).selectForIndexing(projectUuid, handler); } /** diff --git a/sonar-db/src/main/java/org/sonar/db/component/ComponentMapper.java b/sonar-db/src/main/java/org/sonar/db/component/ComponentMapper.java index 7b2e26eab24..a32a4177b58 100644 --- a/sonar-db/src/main/java/org/sonar/db/component/ComponentMapper.java +++ b/sonar-db/src/main/java/org/sonar/db/component/ComponentMapper.java @@ -116,7 +116,7 @@ public interface ComponentMapper { long countGhostProjects(Map parameters); - void selectAll(ResultHandler handler); + void selectForIndexing(@Param("projectUuid") @Nullable String projectUuid, ResultHandler handler); void insert(ComponentDto componentDto); diff --git a/sonar-db/src/main/resources/org/sonar/db/component/ComponentMapper.xml b/sonar-db/src/main/resources/org/sonar/db/component/ComponentMapper.xml index a9d12f01871..92c23601b28 100644 --- a/sonar-db/src/main/resources/org/sonar/db/component/ComponentMapper.xml +++ b/sonar-db/src/main/resources/org/sonar/db/component/ComponentMapper.xml @@ -414,11 +414,16 @@ - + select + from projects p - where p.enabled=${_true} + where + p.enabled=${_true} + and p.copy_component_uuid is null + + and p.project_uuid=#{projectUuid} + diff --git a/sonar-db/src/test/java/org/sonar/db/component/ComponentDaoTest.java b/sonar-db/src/test/java/org/sonar/db/component/ComponentDaoTest.java index 039891ac0a1..8d2b2d83c66 100644 --- a/sonar-db/src/test/java/org/sonar/db/component/ComponentDaoTest.java +++ b/sonar-db/src/test/java/org/sonar/db/component/ComponentDaoTest.java @@ -20,9 +20,14 @@ package org.sonar.db.component; import com.google.common.base.Optional; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; +import org.apache.ibatis.session.ResultContext; +import org.apache.ibatis.session.ResultHandler; +import org.assertj.core.api.ListAssert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -563,6 +568,36 @@ public class ComponentDaoTest { assertThat(components).extracting("id").containsOnly(1L, 2L, 3L, 4L); } + @Test + public void selectForIndexing_all() { + assertSelectForIndexing(null) + .doesNotContain("DIS7") + .doesNotContain("COPY8") + .containsOnly("U1", "U2", "U3", "U4", "U5", "U6"); + } + + @Test + public void selectForIndexing_project() { + assertSelectForIndexing("U1") + .doesNotContain("DIS7") + .doesNotContain("COPY8") + .containsOnly("U1", "U2", "U3", "U4"); + } + + private ListAssert assertSelectForIndexing(@Nullable String projectUuid) { + db.prepareDbUnit(getClass(), "selectForIndexing.xml"); + + List components = new ArrayList<>(); + underTest.selectForIndexing(dbSession, projectUuid, new ResultHandler() { + + @Override + public void handleResult(ResultContext context) { + components.add((ComponentDto) context.getResultObject()); + } + }); + return assertThat(components).extracting(ComponentDto::uuid); + } + @Test public void insert() { db.prepareDbUnit(getClass(), "empty.xml"); diff --git a/sonar-db/src/test/resources/org/sonar/db/component/ComponentDaoTest/selectForIndexing.xml b/sonar-db/src/test/resources/org/sonar/db/component/ComponentDaoTest/selectForIndexing.xml new file mode 100644 index 00000000000..7c4f24eda93 --- /dev/null +++ b/sonar-db/src/test/resources/org/sonar/db/component/ComponentDaoTest/selectForIndexing.xml @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- 2.39.5