From: Léo Geoffroy Date: Wed, 12 Jul 2023 08:51:12 +0000 (+0200) Subject: SONAR-19850 Add indexer event for switch of main branch X-Git-Tag: 10.2.0.77647~336 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f5ff20dbd80554f8bf51e7af68db868256e16bac;p=sonarqube.git SONAR-19850 Add indexer event for switch of main branch --- diff --git a/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java b/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java index e06344017f6..54e6ab950fa 100644 --- a/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java +++ b/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java @@ -62,6 +62,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.groups.Tuple.tuple; import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.server.es.Indexers.BranchEvent.DELETION; import static org.sonar.server.es.Indexers.EntityEvent.PROJECT_KEY_UPDATE; @@ -257,7 +258,7 @@ public class IssueIndexerIT { assertThatIndexHasSize(3); - IndexingResult result = indexBranch(branch1.getUuid(), DELETION); + IndexingResult result = indexBranches(List.of(branch1.getUuid()), DELETION); assertThat(result.getTotal()).isEqualTo(2); assertThat(result.getSuccess()).isEqualTo(2); @@ -414,7 +415,7 @@ public class IssueIndexerIT { es.lockWrites(TYPE_ISSUE); - IndexingResult result = indexBranch(project.uuid(), DELETION); + IndexingResult result = indexBranches(List.of(project.uuid()), DELETION); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); @@ -433,8 +434,8 @@ public class IssueIndexerIT { assertThatEsQueueTableHasSize(0); } - private IndexingResult indexBranch(String branchUuid, Indexers.BranchEvent cause) { - Collection items = underTest.prepareForRecoveryOnBranchEvent(db.getSession(), singletonList(branchUuid), cause); + private IndexingResult indexBranches(List branchUuids, Indexers.BranchEvent cause) { + Collection items = underTest.prepareForRecoveryOnBranchEvent(db.getSession(), branchUuids, cause); db.commit(); return underTest.index(db.getSession(), items); } @@ -461,7 +462,7 @@ public class IssueIndexerIT { @Test public void deleteByKeys_shouldNotRecoverFromErrors() { - addIssueToIndex("P1", "B1","Issue1"); + addIssueToIndex("P1", "B1", "Issue1"); es.lockWrites(TYPE_ISSUE); List issues = List.of("Issue1"); @@ -519,6 +520,34 @@ public class IssueIndexerIT { assertThat(doc.scope()).isEqualTo(IssueScope.MAIN); } + @Test + public void indexIssue_whenSwitchMainBranch_shouldIndexIsMainBranch() { + RuleDto rule = db.rules().insert(); + ProjectData projectData = db.components().insertPrivateProject(); + BranchDto mainBranchDto = projectData.getMainBranchDto(); + ComponentDto mainBranchComponent = projectData.getMainBranchComponent(); + BranchDto newMainBranchDto = db.components().insertProjectBranch(projectData.getProjectDto(), b -> b.setKey("newMainBranch")); + ComponentDto newMainBranchComponent = db.components().getComponentDto(newMainBranchDto); + IssueDto issue1 = createIssue(rule, mainBranchComponent); + IssueDto issue2 = createIssue(rule, newMainBranchComponent); + underTest.indexAllIssues(); + assertThat(es.getDocuments(TYPE_ISSUE, IssueDoc.class)).extracting(IssueDoc::branchUuid, IssueDoc::isMainBranch) + .containsExactlyInAnyOrder(tuple(issue1.getProjectUuid(), true), tuple(issue2.getProjectUuid(), false)); + + db.getDbClient().branchDao().updateIsMain(db.getSession(), projectData.getMainBranchDto().getUuid(), false); + db.getDbClient().branchDao().updateIsMain(db.getSession(), newMainBranchDto.getUuid(), true); + indexBranches(List.of(mainBranchDto.getUuid(), newMainBranchDto.getUuid()), Indexers.BranchEvent.SWITCH_OF_MAIN_BRANCH); + + assertThat(es.getDocuments(TYPE_ISSUE, IssueDoc.class)).extracting(IssueDoc::branchUuid, IssueDoc::isMainBranch) + .containsExactlyInAnyOrder(tuple(issue1.getProjectUuid(), false), tuple(issue2.getProjectUuid(), true)); + } + + private IssueDto createIssue(RuleDto rule, ComponentDto branch) { + ComponentDto dir2 = db.components().insertComponent(ComponentTesting.newDirectory(branch, "src/main/java/foo")); + ComponentDto file2 = db.components().insertComponent(newFileDto(branch, dir2)); + return db.issues().insert(rule, branch, file2); + } + @Test public void issue_on_test_file_has_test_scope() { RuleDto rule = db.rules().insert(); diff --git a/server/sonar-server-common/src/it/java/org/sonar/server/measure/index/ProjectMeasuresIndexerIT.java b/server/sonar-server-common/src/it/java/org/sonar/server/measure/index/ProjectMeasuresIndexerIT.java index 9775e32eeae..24ce56bd724 100644 --- a/server/sonar-server-common/src/it/java/org/sonar/server/measure/index/ProjectMeasuresIndexerIT.java +++ b/server/sonar-server-common/src/it/java/org/sonar/server/measure/index/ProjectMeasuresIndexerIT.java @@ -21,13 +21,17 @@ package org.sonar.server.measure.index; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Rule; import org.junit.Test; +import org.sonar.api.measures.CoreMetrics; import org.sonar.api.resources.Qualifiers; import org.sonar.api.utils.System2; import org.sonar.db.DbSession; @@ -37,6 +41,7 @@ import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ProjectData; import org.sonar.db.component.SnapshotDto; import org.sonar.db.es.EsQueueDto; +import org.sonar.db.metric.MetricDto; import org.sonar.db.project.ProjectDto; import org.sonar.server.es.EsTester; import org.sonar.server.es.Indexers; @@ -49,6 +54,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; @@ -58,6 +64,9 @@ import static org.sonar.server.es.Indexers.EntityEvent.CREATION; import static org.sonar.server.es.Indexers.EntityEvent.DELETION; import static org.sonar.server.es.Indexers.EntityEvent.PROJECT_KEY_UPDATE; import static org.sonar.server.es.Indexers.EntityEvent.PROJECT_TAGS_UPDATE; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_KEY; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_VALUE; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_UUID; @@ -239,6 +248,27 @@ public class ProjectMeasuresIndexerIT { assertThat(result.getSuccess()).isOne(); } + @Test + public void prepareForRecoveryOnEntityEvent_shouldReindexProject_whenSwitchMainBranch() { + ProjectData projectData = db.components().insertPrivateProject(defaults(), p -> p.setTagsString("foo")); + ProjectDto project = projectData.getProjectDto(); + BranchDto oldMainBranchDto = projectData.getMainBranchDto(); + BranchDto newMainBranchDto = db.components().insertProjectBranch(project); + MetricDto nloc = db.measures().insertMetric(m -> m.setKey(CoreMetrics.NCLOC_KEY)); + db.measures().insertLiveMeasure(oldMainBranchDto, nloc, e -> e.setValue(1d)); + db.measures().insertLiveMeasure(newMainBranchDto, nloc, e -> e.setValue(2d)); + indexProject(project, CREATION); + assertThatProjectHasMeasure(project, CoreMetrics.NCLOC_KEY, 1d); + + db.getDbClient().branchDao().updateIsMain(db.getSession(), oldMainBranchDto.getUuid(), false); + db.getDbClient().branchDao().updateIsMain(db.getSession(), newMainBranchDto.getUuid(), true); + IndexingResult result = indexBranches(List.of(oldMainBranchDto, newMainBranchDto), Indexers.BranchEvent.SWITCH_OF_MAIN_BRANCH); + + assertThatProjectHasMeasure(project, CoreMetrics.NCLOC_KEY, 2d); + assertThat(result.getTotal()).isOne(); + assertThat(result.getSuccess()).isOne(); + } + @Test public void delete_doc_from_index_when_project_is_deleted() { ProjectDto project = db.components().insertPrivateProject().getProjectDto(); @@ -306,6 +336,13 @@ public class ProjectMeasuresIndexerIT { return underTest.index(dbSession, items); } + private IndexingResult indexBranches(List branches, Indexers.BranchEvent cause) { + DbSession dbSession = db.getSession(); + Collection items = underTest.prepareForRecoveryOnBranchEvent(dbSession, branches.stream().map(BranchDto::getUuid).collect(Collectors.toSet()), cause); + dbSession.commit(); + return underTest.index(dbSession, items); + } + private void assertThatProjectHasTag(ProjectDto project, String expectedTag) { SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) .source(new SearchSourceBuilder() @@ -318,6 +355,22 @@ public class ProjectMeasuresIndexerIT { .contains(project.getUuid()); } + private void assertThatProjectHasMeasure(ProjectDto project, String metric, Double value) { + SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .source(new SearchSourceBuilder() + .query(nestedQuery( + FIELD_MEASURES, + boolQuery() + .filter(termQuery(FIELD_MEASURES_MEASURE_KEY, metric)) + .filter(termQuery(FIELD_MEASURES_MEASURE_VALUE, value)), + ScoreMode.Avg + ))); + + assertThat(es.client().search(request).getHits().getHits()) + .extracting(SearchHit::getId) + .contains(project.getUuid()); + } + private void assertThatEsQueueTableHasSize(int expectedSize) { assertThat(db.countRowsOfTable("es_queue")).isEqualTo(expectedSize); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/Indexers.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/Indexers.java index 00e23edbe4b..086bdb7ca04 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/Indexers.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/Indexers.java @@ -37,7 +37,8 @@ public interface Indexers { enum BranchEvent { // Note that when a project/app is deleted, no events are sent for each branch removed as part of that process DELETION, - MEASURE_CHANGE + MEASURE_CHANGE, + SWITCH_OF_MAIN_BRANCH } /** diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index fdab6b14835..f41b612a9de 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -161,7 +161,8 @@ public class IssueIndexer implements EventIndexer, AnalysisIndexer, NeedAuthoriz // Measures, permissions, project key and tags are not used in type issues/issue emptyList(); - case DELETION -> { + case DELETION, SWITCH_OF_MAIN_BRANCH -> { + //switch of main branch requires to reindex the project issues List items = createBranchRecoveryItems(branchUuids); yield dbClient.esQueueDao().insert(dbSession, items); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index d5d0aefbc9e..e515469bf1d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -28,6 +28,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.jetbrains.annotations.NotNull; import org.sonar.api.resources.Qualifiers; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -115,16 +116,20 @@ public class ProjectMeasuresIndexer implements EventIndexer, AnalysisIndexer, Ne public Collection prepareForRecoveryOnBranchEvent(DbSession dbSession, Collection branchUuids, Indexers.BranchEvent cause) { return switch (cause) { case DELETION -> Collections.emptyList(); - case MEASURE_CHANGE -> { - // when MEASURE_CHANGE or PROJECT_KEY_UPDATE project must be re-indexed because key is used in this index - Set entityUuids = dbClient.branchDao().selectByUuids(dbSession, branchUuids) - .stream().map(BranchDto::getProjectUuid) - .collect(Collectors.toSet()); - yield prepareForRecovery(dbSession, entityUuids); + case MEASURE_CHANGE, SWITCH_OF_MAIN_BRANCH -> { + Set projectUuids = retrieveProjectUuidsFromBranchUuids(dbSession, branchUuids); + yield prepareForRecovery(dbSession, projectUuids); } }; } + @NotNull + private Set retrieveProjectUuidsFromBranchUuids(DbSession dbSession, Collection branchUuids) { + return dbClient.branchDao().selectByUuids(dbSession, branchUuids) + .stream().map(BranchDto::getProjectUuid) + .collect(Collectors.toSet()); + } + private Collection prepareForRecovery(DbSession dbSession, Collection entityUuids) { List items = entityUuids.stream() .map(entityUuid -> EsQueueDto.create(TYPE_PROJECT_MEASURES.format(), entityUuid, null, entityUuid)) diff --git a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/branch/ws/SetMainBranchActionIT.java b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/branch/ws/SetMainBranchActionIT.java index 6bf46806a6c..bff8917f060 100644 --- a/server/sonar-webserver-webapi/src/it/java/org/sonar/server/branch/ws/SetMainBranchActionIT.java +++ b/server/sonar-webserver-webapi/src/it/java/org/sonar/server/branch/ws/SetMainBranchActionIT.java @@ -19,6 +19,7 @@ */ package org.sonar.server.branch.ws; +import java.util.List; import java.util.Optional; import java.util.Set; import org.junit.Rule; @@ -33,6 +34,7 @@ import org.sonar.db.component.BranchDto; import org.sonar.db.component.ComponentTesting; import org.sonar.db.component.ProjectData; import org.sonar.db.project.ProjectDto; +import org.sonar.server.es.Indexers; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.exceptions.NotFoundException; import org.sonar.server.exceptions.UnauthorizedException; @@ -44,6 +46,8 @@ import org.sonar.server.ws.WsActionTester; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.sonar.server.branch.ws.ProjectBranchesParameters.ACTION_SET_MAIN_BRANCH; @@ -59,7 +63,9 @@ public class SetMainBranchActionIT { @Rule public LogTester logTester = new LogTester().setLevel(Level.INFO); ProjectLifeCycleListeners projectLifeCycleListeners = mock(ProjectLifeCycleListeners.class); - private WsActionTester tester = new WsActionTester(new SetMainBranchAction(db.getDbClient(), userSession, projectLifeCycleListeners)); + + private final Indexers indexers = mock(Indexers.class); + private WsActionTester tester = new WsActionTester(new SetMainBranchAction(db.getDbClient(), userSession, projectLifeCycleListeners, indexers)); @Test public void testDefinition() { @@ -214,6 +220,7 @@ public class SetMainBranchActionIT { .setParam(PARAM_BRANCH, newMainBranch.getKey()).execute(); checkCallToProjectLifeCycleListenersOnProjectBranchesChanges(projectData.getProjectDto()); + verify(indexers).commitAndIndexBranches(any(), eq(List.of(projectData.getMainBranchDto(), newMainBranch)), eq(Indexers.BranchEvent.SWITCH_OF_MAIN_BRANCH)); checkNewMainBranch(projectData.projectUuid(), newMainBranch.getUuid()); checkPreviousMainBranch(projectData); assertThat(logTester.logs(Level.INFO)) diff --git a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/branch/ws/SetMainBranchAction.java b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/branch/ws/SetMainBranchAction.java index 8f5f2cb33dc..77861127ab6 100644 --- a/server/sonar-webserver-webapi/src/main/java/org/sonar/server/branch/ws/SetMainBranchAction.java +++ b/server/sonar-webserver-webapi/src/main/java/org/sonar/server/branch/ws/SetMainBranchAction.java @@ -19,6 +19,7 @@ */ package org.sonar.server.branch.ws; +import java.util.List; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,7 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.BranchDto; import org.sonar.db.project.ProjectDto; +import org.sonar.server.es.Indexers; import org.sonar.server.project.Project; import org.sonar.server.project.ProjectLifeCycleListeners; import org.sonar.server.user.UserSession; @@ -46,11 +48,13 @@ public class SetMainBranchAction implements BranchWsAction { private final DbClient dbClient; private final UserSession userSession; private final ProjectLifeCycleListeners projectLifeCycleListeners; + private final Indexers indexers; - public SetMainBranchAction(DbClient dbClient, UserSession userSession, ProjectLifeCycleListeners projectLifeCycleListeners) { + public SetMainBranchAction(DbClient dbClient, UserSession userSession, ProjectLifeCycleListeners projectLifeCycleListeners, Indexers indexers) { this.dbClient = dbClient; this.userSession = userSession; this.projectLifeCycleListeners = projectLifeCycleListeners; + this.indexers = indexers; } @Override @@ -94,7 +98,7 @@ public class SetMainBranchAction implements BranchWsAction { } configureProjectWithNewMainBranch(dbSession, projectDto.getKey(), oldMainBranch, newMainBranch); refreshApplicationsAndPortfoliosComputedByProject(projectDto); - // todo : refresh elasticSearchIndexes + indexers.commitAndIndexBranches(dbSession, List.of(oldMainBranch, newMainBranch), Indexers.BranchEvent.SWITCH_OF_MAIN_BRANCH); dbSession.commit(); response.noContent();