From f8fd928e5a6627c245b85342af5f6d367f49eb28 Mon Sep 17 00:00:00 2001 From: Jacek Date: Fri, 5 Jan 2024 14:58:48 +0100 Subject: [PATCH] SONAR-18963 Index peristed issues only --- .../step/IndexAnalysisStepIT.java | 61 +++++-- .../component/FileStatuses.java | 5 - .../component/FileStatusesImpl.java | 7 - ...ProjectAnalysisTaskContainerPopulator.java | 4 + .../index/IndexDiffResolver.java | 27 +++ .../index/IndexDiffResolverImpl.java | 41 +++++ .../index/NopDiffResolverImpl.java | 36 ++++ .../projectanalysis/index/package-info.java | 23 +++ .../ChangedIssuesRepository.java} | 26 +-- .../step/IndexAnalysisStep.java | 43 +++-- .../step/LoadChangedIssuesStep.java | 70 ++++++++ .../step/ReportComputationSteps.java | 1 + .../component/FileStatusesImplTest.java | 28 ---- .../index/IndexDiffResolverImplTest.java | 56 +++++++ .../index/NopDiffResolverImplTest.java | 37 +++++ .../ChangedIssuesRepositoryTest.java} | 25 +-- .../step/LoadChangedIssuesStepTest.java | 157 ++++++++++++++++++ .../server/issue/index/IssueIndexerIT.java | 17 +- .../index/EntityDefinitionIndexer.java | 5 - .../org/sonar/server/es/AnalysisIndexer.java | 23 ++- .../server/issue/index/IssueIndexer.java | 32 ++-- .../measure/index/ProjectMeasuresIndexer.java | 5 - .../sonar/server/es/AnalysisIndexerTest.java | 68 ++++++++ .../server/permission/index/FooIndexer.java | 12 +- 24 files changed, 660 insertions(+), 149 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolver.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImpl.java create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/package-info.java rename server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/{component/NopFileStatuses.java => issue/ChangedIssuesRepository.java} (66%) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStep.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImplTest.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImplTest.java rename server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/{component/NopFileStatusesTest.java => issue/ChangedIssuesRepositoryTest.java} (60%) create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStepTest.java create mode 100644 server/sonar-server-common/src/test/java/org/sonar/server/es/AnalysisIndexerTest.java diff --git a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStepIT.java b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStepIT.java index a1ac674645b..a9edff59ac0 100644 --- a/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStepIT.java +++ b/server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStepIT.java @@ -19,20 +19,25 @@ */ package org.sonar.ce.task.projectanalysis.step; +import java.util.Optional; import java.util.Set; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.sonar.ce.task.projectanalysis.batch.BatchReportReaderRule; import org.sonar.ce.task.projectanalysis.component.Component; -import org.sonar.ce.task.projectanalysis.component.FileStatuses; import org.sonar.ce.task.projectanalysis.component.ReportComponent; import org.sonar.ce.task.projectanalysis.component.TreeRootHolderRule; import org.sonar.ce.task.projectanalysis.component.ViewsComponent; +import org.sonar.ce.task.projectanalysis.index.IndexDiffResolver; import org.sonar.ce.task.step.ComputationStep; import org.sonar.ce.task.step.TestComputationStepContext; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.ce.CeActivityDao; +import org.sonar.db.ce.CeActivityDto; +import org.sonar.db.ce.CeQueueDto; +import org.sonar.db.ce.CeTaskTypes; import org.sonar.db.component.BranchDao; import org.sonar.server.es.AnalysisIndexer; @@ -53,11 +58,12 @@ public class IndexAnalysisStepIT extends BaseStepTest { public BatchReportReaderRule reportReader = new BatchReportReaderRule(); private final DbClient dbClient = mock(DbClient.class); - private final FileStatuses fileStatuses = mock(FileStatuses.class); + private final IndexDiffResolver indexDiffResolver = mock(IndexDiffResolver.class); private final AnalysisIndexer analysisIndexer = mock(AnalysisIndexer.class); private final DbSession dbSession = mock(DbSession.class); private final BranchDao branchDao = mock(BranchDao.class); - private final IndexAnalysisStep underTest = new IndexAnalysisStep(treeRootHolder, fileStatuses, dbClient, analysisIndexer); + private final CeActivityDao ceActivityDao = mock(CeActivityDao.class); + private final IndexAnalysisStep underTest = new IndexAnalysisStep(treeRootHolder, indexDiffResolver, dbClient, analysisIndexer); private TestComputationStepContext testComputationStepContext; @@ -67,6 +73,7 @@ public class IndexAnalysisStepIT extends BaseStepTest { when(dbClient.openSession(false)).thenReturn(dbSession); when(dbClient.branchDao()).thenReturn(branchDao); + when(dbClient.ceActivityDao()).thenReturn(ceActivityDao); } @Test @@ -76,36 +83,66 @@ public class IndexAnalysisStepIT extends BaseStepTest { underTest.execute(testComputationStepContext); - verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID, Set.of()); + verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID); } @Test - public void call_indexByProjectUuid_of_indexer_for_view() { + public void execute_indexByProjectUuid_of_indexer_for_view() { Component view = ViewsComponent.builder(VIEW, PROJECT_KEY).setUuid(PROJECT_UUID).build(); treeRootHolder.setRoot(view); underTest.execute(testComputationStepContext); - verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID, Set.of()); + verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID); } @Test - public void execute_whenMarkAsUnchangedFlagActivated_shouldCallIndexOnAnalysisWithChangedComponents() { + public void execute_whenBranchIsNeedIssueSync_shouldReindexEverything() { Component project = ReportComponent.builder(PROJECT, 1).setUuid(PROJECT_UUID).setKey(PROJECT_KEY).build(); treeRootHolder.setRoot(project); - Set anyUuids = Set.of("any-uuid"); - when(fileStatuses.getFileUuidsMarkedAsUnchanged()).thenReturn(anyUuids); + when(branchDao.isBranchNeedIssueSync(dbSession, PROJECT_UUID)).thenReturn(true); underTest.execute(testComputationStepContext); - verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID, anyUuids); + verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID); } @Test - public void execute_whenBranchIsNeedIssueSync_shouldReindexEverything() { + public void execute_whenConditionsForDiffMet_shouldReindexDifferenceOnly() { Component project = ReportComponent.builder(PROJECT, 1).setUuid(PROJECT_UUID).setKey(PROJECT_KEY).build(); treeRootHolder.setRoot(project); - when(branchDao.isBranchNeedIssueSync(dbSession, PROJECT_UUID)).thenReturn(true); + when(ceActivityDao.selectLastByComponentUuidAndTaskType(dbSession, PROJECT_UUID, CeTaskTypes.REPORT)) + .thenReturn(Optional.of(new CeActivityDto(new CeQueueDto()).setStatus(org.sonar.db.ce.CeActivityDto.Status.SUCCESS))); + when(analysisIndexer.supportDiffIndexing()).thenReturn(true); + when(indexDiffResolver.resolve(analysisIndexer.getClass())).thenReturn(Set.of("foo", "bar")); + + underTest.execute(testComputationStepContext); + + verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID, Set.of("foo", "bar")); + } + + @Test + public void execute_whenFailedCETask_shouldReindexFully() { + Component project = ReportComponent.builder(PROJECT, 1).setUuid(PROJECT_UUID).setKey(PROJECT_KEY).build(); + treeRootHolder.setRoot(project); + when(ceActivityDao.selectLastByComponentUuidAndTaskType(dbSession, PROJECT_UUID, CeTaskTypes.REPORT)) + .thenReturn(Optional.of(new CeActivityDto(new CeQueueDto()).setStatus(CeActivityDto.Status.FAILED))); + when(analysisIndexer.supportDiffIndexing()).thenReturn(true); + when(indexDiffResolver.resolve(analysisIndexer.getClass())).thenReturn(Set.of("foo", "bar")); + + underTest.execute(testComputationStepContext); + + verify(analysisIndexer).indexOnAnalysis(PROJECT_UUID); + } + + @Test + public void execute_whenNoCETask_shouldReindexFully() { + Component project = ReportComponent.builder(PROJECT, 1).setUuid(PROJECT_UUID).setKey(PROJECT_KEY).build(); + treeRootHolder.setRoot(project); + when(ceActivityDao.selectLastByComponentUuidAndTaskType(dbSession, PROJECT_UUID, CeTaskTypes.REPORT)) + .thenReturn(Optional.empty()); + when(analysisIndexer.supportDiffIndexing()).thenReturn(true); + when(indexDiffResolver.resolve(analysisIndexer.getClass())).thenReturn(Set.of("foo", "bar")); underTest.execute(testComputationStepContext); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java index fd90e78737e..5df76c4bab3 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatuses.java @@ -19,8 +19,6 @@ */ package org.sonar.ce.task.projectanalysis.component; -import java.util.Set; - public interface FileStatuses { /** * A file is unchanged compared to the last analysis if it was detected as unchanged by the scanner and @@ -29,7 +27,4 @@ public interface FileStatuses { boolean isUnchanged(Component component); boolean isDataUnchanged(Component component); - - Set getFileUuidsMarkedAsUnchanged(); - } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java index 6707d0f43ed..5f0e34472ba 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImpl.java @@ -27,7 +27,6 @@ import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; import org.sonar.db.source.FileHashesDto; import static com.google.common.base.Preconditions.checkState; -import static java.util.Collections.unmodifiableSet; import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER; public class FileStatusesImpl implements FileStatuses { @@ -88,12 +87,6 @@ public class FileStatusesImpl implements FileStatuses { return fileUuidsMarkedAsUnchanged.contains(component.getUuid()); } - @Override - public Set getFileUuidsMarkedAsUnchanged() { - failIfNotInitialized(); - return unmodifiableSet(fileUuidsMarkedAsUnchanged); - } - private boolean hashEquals(Component component) { Optional dbHash = previousSourceHashRepository.getDbFile(component).map(FileHashesDto::getSrcHash); return dbHash.map(hash -> hash.equals(sourceHashRepository.getRawSourceHash(component))).orElse(false); diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index d8ae9abb6f7..1f8615159df 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -52,7 +52,9 @@ import org.sonar.ce.task.projectanalysis.filemove.ScoreMatrixDumperImpl; import org.sonar.ce.task.projectanalysis.filemove.SourceSimilarityImpl; import org.sonar.ce.task.projectanalysis.filesystem.ComputationTempFolderProvider; import org.sonar.ce.task.projectanalysis.issue.AnticipatedTransitionRepositoryImpl; +import org.sonar.ce.task.projectanalysis.index.IndexDiffResolverImpl; import org.sonar.ce.task.projectanalysis.issue.BaseIssuesLoader; +import org.sonar.ce.task.projectanalysis.issue.ChangedIssuesRepository; import org.sonar.ce.task.projectanalysis.issue.CloseIssuesOnRemovedComponentsVisitor; import org.sonar.ce.task.projectanalysis.issue.ClosedIssuesInputFactory; import org.sonar.ce.task.projectanalysis.issue.ComponentIssuesLoader; @@ -194,6 +196,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop new ComputationTempFolderProvider(), FileStatusesImpl.class, + IndexDiffResolverImpl.class, new MetricModule(), // holders @@ -271,6 +274,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop IssuesRepositoryVisitor.class, RemoveProcessedComponentsVisitor.class, IssueOnReferenceBranchVisitor.class, + ChangedIssuesRepository.class, // visitors : order is important, measure computers must be executed at the end in order to access to every measures / issues AnalysisFromSonarQube94Visitor.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolver.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolver.java new file mode 100644 index 00000000000..5c6ad50f541 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolver.java @@ -0,0 +1,27 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.index; + +import java.util.Collection; +import org.sonar.server.es.AnalysisIndexer; + +public interface IndexDiffResolver { + Collection resolve(Class clazz); +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImpl.java new file mode 100644 index 00000000000..33d2878fc78 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImpl.java @@ -0,0 +1,41 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.index; + +import java.util.Collection; +import org.sonar.ce.task.projectanalysis.issue.ChangedIssuesRepository; +import org.sonar.server.es.AnalysisIndexer; +import org.sonar.server.issue.index.IssueIndexer; + +public class IndexDiffResolverImpl implements IndexDiffResolver { + private final ChangedIssuesRepository changedIssuesRepository; + + public IndexDiffResolverImpl(ChangedIssuesRepository changedIssuesRepository) { + this.changedIssuesRepository = changedIssuesRepository; + } + + @Override + public Collection resolve(Class clazz) { + if (clazz.isAssignableFrom(IssueIndexer.class)) { + return changedIssuesRepository.getChangedIssuesKeys(); + } + throw new UnsupportedOperationException("Unsupported indexer: " + clazz); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImpl.java new file mode 100644 index 00000000000..3611b6d6246 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImpl.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.index; + +import java.util.Collection; +import java.util.Collections; +import org.sonar.server.es.AnalysisIndexer; + +public class NopDiffResolverImpl implements IndexDiffResolver { + + public NopDiffResolverImpl() { + //nothing to do + } + + @Override + public Collection resolve(Class clazz) { + return Collections.emptyList(); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/package-info.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/package-info.java new file mode 100644 index 00000000000..459ee2c2949 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/index/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.ce.task.projectanalysis.index; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/NopFileStatuses.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepository.java similarity index 66% rename from server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/NopFileStatuses.java rename to server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepository.java index 5740b312fe5..0138f09bccf 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/NopFileStatuses.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepository.java @@ -17,29 +17,19 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.ce.task.projectanalysis.component; - +package org.sonar.ce.task.projectanalysis.issue; +import java.util.HashSet; import java.util.Set; -/** - * No operation implementation of {@link FileStatuses} - */ -public final class NopFileStatuses implements FileStatuses { - - - @Override - public boolean isUnchanged(Component component) { - return false; - } +public class ChangedIssuesRepository { + private final Set changedIssuesKeys = new HashSet<>(); - @Override - public boolean isDataUnchanged(Component component) { - return false; + public void addIssueKey(String issueKey) { + changedIssuesKeys.add(issueKey); } - @Override - public Set getFileUuidsMarkedAsUnchanged() { - return Set.of(); + public Set getChangedIssuesKeys() { + return changedIssuesKeys; } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStep.java index 21967541467..33ac38cbd26 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/IndexAnalysisStep.java @@ -19,48 +19,59 @@ */ package org.sonar.ce.task.projectanalysis.step; -import java.util.Set; -import java.util.function.Consumer; +import java.util.Collection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.ce.task.projectanalysis.component.FileStatuses; +import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.TreeRootHolder; +import org.sonar.ce.task.projectanalysis.index.IndexDiffResolver; import org.sonar.ce.task.step.ComputationStep; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.ce.CeActivityDto; +import org.sonar.db.ce.CeTaskTypes; import org.sonar.server.es.AnalysisIndexer; public class IndexAnalysisStep implements ComputationStep { - private static final Logger LOGGER = LoggerFactory.getLogger(IndexAnalysisStep.class); - private final TreeRootHolder treeRootHolder; - private final FileStatuses fileStatuses; + private final IndexDiffResolver indexDiffResolver; private final AnalysisIndexer[] indexers; private final DbClient dbClient; - public IndexAnalysisStep(TreeRootHolder treeRootHolder, FileStatuses fileStatuses, DbClient dbClient, AnalysisIndexer... indexers) { + public IndexAnalysisStep(TreeRootHolder treeRootHolder, IndexDiffResolver indexDiffResolver, DbClient dbClient, AnalysisIndexer... indexers) { this.treeRootHolder = treeRootHolder; - this.fileStatuses = fileStatuses; + this.indexDiffResolver = indexDiffResolver; this.indexers = indexers; this.dbClient = dbClient; } @Override public void execute(ComputationStep.Context context) { - String branchUuid = treeRootHolder.getRoot().getUuid(); - Consumer analysisIndexerConsumer = getAnalysisIndexerConsumer(branchUuid); + Component root = treeRootHolder.getRoot(); + String branchUuid = root.getUuid(); + for (AnalysisIndexer indexer : indexers) { LOGGER.debug("Call {}", indexer); - analysisIndexerConsumer.accept(indexer); + if (isDiffIndexingSupported(root, indexer) && hasPreviousAnalysisSucceeded(branchUuid) && !isBranchNeedIssueSync(branchUuid)) { + Collection diffSet = indexDiffResolver.resolve(indexer.getClass()); + indexer.indexOnAnalysis(branchUuid, diffSet); + } else { + indexer.indexOnAnalysis(branchUuid); + } + } + } + + private boolean hasPreviousAnalysisSucceeded(String branchUuid) { + try (DbSession dbSession = dbClient.openSession(false)) { + return dbClient.ceActivityDao().selectLastByComponentUuidAndTaskType(dbSession, branchUuid, CeTaskTypes.REPORT) + .filter(activityDto -> CeActivityDto.Status.SUCCESS.equals(activityDto.getStatus())) + .isPresent(); } } - private Consumer getAnalysisIndexerConsumer(String branchUuid) { - Set fileUuidsMarkedAsUnchanged = fileStatuses.getFileUuidsMarkedAsUnchanged(); - return isBranchNeedIssueSync(branchUuid) - ? (indexer -> indexer.indexOnAnalysis(branchUuid)) - : (indexer -> indexer.indexOnAnalysis(branchUuid, fileUuidsMarkedAsUnchanged)); + private static boolean isDiffIndexingSupported(Component root, AnalysisIndexer indexer) { + return Component.Type.PROJECT.equals(root.getType()) && indexer.supportDiffIndexing(); } private boolean isBranchNeedIssueSync(String branchUuid) { diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStep.java new file mode 100644 index 00000000000..483688ab950 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStep.java @@ -0,0 +1,70 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import org.sonar.ce.task.projectanalysis.issue.ChangedIssuesRepository; +import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache; +import org.sonar.ce.task.projectanalysis.period.PeriodHolder; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.issue.DefaultIssue; +import org.sonar.core.util.CloseableIterator; +import org.sonar.db.newcodeperiod.NewCodePeriodType; + +public class LoadChangedIssuesStep implements ComputationStep { + private final PeriodHolder periodHolder; + private final ProtoIssueCache protoIssueCache; + private final ChangedIssuesRepository changedIssuesRepository; + + public LoadChangedIssuesStep(PeriodHolder periodHolder, ProtoIssueCache protoIssueCache, ChangedIssuesRepository changedIssuesRepository) { + this.periodHolder = periodHolder; + this.protoIssueCache = protoIssueCache; + this.changedIssuesRepository = changedIssuesRepository; + } + + @Override + public void execute(Context context) { + try (CloseableIterator issues = protoIssueCache.traverse()) { + while (issues.hasNext()) { + DefaultIssue issue = issues.next(); + if (shouldUpdateIndexForIssue(issue)) { + changedIssuesRepository.addIssueKey(issue.key()); + } + } + } + } + + private boolean shouldUpdateIndexForIssue(DefaultIssue issue) { + return issue.isNew() || issue.isCopied() || issue.isChanged() + || (isOnBranchUsingReferenceBranch() && (issue.isNoLongerNewCodeReferenceIssue() || issue.isToBeMigratedAsNewCodeReferenceIssue())); + } + + private boolean isOnBranchUsingReferenceBranch() { + if (periodHolder.hasPeriod()) { + return NewCodePeriodType.REFERENCE_BRANCH.name().equals(periodHolder.getPeriod().getMode()); + } + return false; + } + + @Override + public String getDescription() { + return "Load changed issues for indexation"; + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java index d5d3f4cb578..d0783b2708f 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java @@ -114,6 +114,7 @@ public class ReportComputationSteps extends AbstractComputationSteps { UpdateQualityProfilesLastUsedDateStep.class, PurgeDatastoresStep.class, + LoadChangedIssuesStep.class, IndexAnalysisStep.class, UpdateNeedIssueSyncStep.class, ProjectNclocComputationStep.class, diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java index 0288d225d2e..e6e936084f4 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/FileStatusesImplTest.java @@ -29,7 +29,6 @@ import org.sonar.ce.task.projectanalysis.source.SourceHashRepository; import org.sonar.db.source.FileHashesDto; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -174,33 +173,6 @@ public class FileStatusesImplTest { verify(previousSourceHashRepository).getDbFile(file1); } - @Test - public void getFileUuidsMarkedAsUnchanged_whenNotInitialized_shouldFail() { - assertThatThrownBy(fileStatuses::getFileUuidsMarkedAsUnchanged) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Not initialized"); - } - - @Test - public void getFileUuidsMarkedAsUnchanged_shouldReturnMarkAsUnchangedFileUuids() { - Component file1 = ReportComponent.builder(Component.Type.FILE, 2, "FILE1_KEY").setStatus(Component.Status.SAME) - .setFileAttributes(new FileAttributes(false, null, 10, true, null)).build(); - Component file2 = ReportComponent.builder(Component.Type.FILE, 3, "FILE2_KEY").setStatus(Component.Status.SAME).build(); - addDbFileHash(file1, "hash1"); - addDbFileHash(file2, "hash2"); - addReportFileHash(file1, "hash1"); - addReportFileHash(file2, "hash2"); - Component project = ReportComponent.builder(Component.Type.PROJECT, 1) - .setUuid(PROJECT_UUID) - .setKey(PROJECT_KEY) - .addChildren(file1, file2) - .build(); - treeRootHolder.setRoot(project); - fileStatuses.initialize(); - - assertThat(fileStatuses.getFileUuidsMarkedAsUnchanged()).contains(file1.getUuid()); - } - private void addDbFileHash(Component file, String hash) { FileHashesDto fileHashesDto = new FileHashesDto().setSrcHash(hash); when(previousSourceHashRepository.getDbFile(file)).thenReturn(Optional.of(fileHashesDto)); diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImplTest.java new file mode 100644 index 00000000000..ea521426a6e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/IndexDiffResolverImplTest.java @@ -0,0 +1,56 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.index; + +import java.util.Collection; +import java.util.Set; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.issue.ChangedIssuesRepository; +import org.sonar.server.issue.index.IssueIndexer; +import org.sonar.server.measure.index.ProjectMeasuresIndexer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IndexDiffResolverImplTest { + + private final ChangedIssuesRepository changedIssuesRepository = mock(ChangedIssuesRepository.class); + + private final IndexDiffResolverImpl underTest = new IndexDiffResolverImpl(changedIssuesRepository); + + @Test + public void resolve_whenIssueIndexer_shouldReturnChangedIssueKeys() { + when(changedIssuesRepository.getChangedIssuesKeys()).thenReturn(Set.of("key1", "key2","key3")); + Collection resolvedDiff = underTest.resolve(IssueIndexer.class); + + assertThat(resolvedDiff) + .containsExactlyInAnyOrder("key1", "key2","key3"); + } + + @Test + public void resolve_whenUnsupportedIndexer_shouldThrowUPE() { + when(changedIssuesRepository.getChangedIssuesKeys()).thenReturn(Set.of("key1", "key2","key3")); + assertThatThrownBy(() ->underTest.resolve(ProjectMeasuresIndexer.class)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Unsupported indexer: class org.sonar.server.measure.index.ProjectMeasuresIndexer"); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImplTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImplTest.java new file mode 100644 index 00000000000..5e1e1126b52 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/index/NopDiffResolverImplTest.java @@ -0,0 +1,37 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.index; + +import org.junit.Test; +import org.sonar.server.issue.index.IssueIndexer; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NopDiffResolverImplTest { + + private final NopDiffResolverImpl underTest = new NopDiffResolverImpl(); + + @Test + public void resolve_shouldDoNothing() { + assertThat(underTest.resolve(IssueIndexer.class)) + .isEmpty(); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/NopFileStatusesTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepositoryTest.java similarity index 60% rename from server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/NopFileStatusesTest.java rename to server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepositoryTest.java index 8040f40b923..40063f362f5 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/component/NopFileStatusesTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/issue/ChangedIssuesRepositoryTest.java @@ -17,31 +17,24 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.ce.task.projectanalysis.component; +package org.sonar.ce.task.projectanalysis.issue; import org.junit.Test; -import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; -public class NopFileStatusesTest { +public class ChangedIssuesRepositoryTest { - private final NopFileStatuses nopFileStatuses = new NopFileStatuses(); - - private final Component component = Mockito.mock(Component.class); + private final ChangedIssuesRepository underTest = new ChangedIssuesRepository(); @Test - public void isUnchanged() { - assertThat(nopFileStatuses.isUnchanged(component)).isFalse(); - } + public void addIssueKey_shouldAddKeysToRepository() { + underTest.addIssueKey("key1"); + underTest.addIssueKey("key2"); + underTest.addIssueKey("key3"); - @Test - public void isDataUnchanged() { - assertThat(nopFileStatuses.isDataUnchanged(component)).isFalse(); + assertThat(underTest.getChangedIssuesKeys()) + .containsExactlyInAnyOrder("key1", "key2", "key3"); } - @Test - public void getFileUuidsMarkedAsUnchanged() { - assertThat(nopFileStatuses.getFileUuidsMarkedAsUnchanged()).isEmpty(); - } } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStepTest.java new file mode 100644 index 00000000000..ae7d40a51ac --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/LoadChangedIssuesStepTest.java @@ -0,0 +1,157 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.step; + +import java.io.IOException; +import java.util.Date; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.issue.impact.Severity; +import org.sonar.api.issue.impact.SoftwareQuality; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rules.RuleType; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.projectanalysis.issue.ChangedIssuesRepository; +import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache; +import org.sonar.ce.task.projectanalysis.period.Period; +import org.sonar.ce.task.projectanalysis.period.PeriodHolder; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.issue.DefaultIssue; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.api.issue.Issue.STATUS_OPEN; +import static org.sonar.api.rule.Severity.BLOCKER; + +public class LoadChangedIssuesStepTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private final PeriodHolder periodHolder = mock(PeriodHolder.class); + private ProtoIssueCache protoIssueCache; + + private final ChangedIssuesRepository changedIssuesRepository = mock(ChangedIssuesRepository.class); + + private LoadChangedIssuesStep underTest; + + @Before + public void before() throws IOException { + protoIssueCache = new ProtoIssueCache(temp.newFile(), System2.INSTANCE); + underTest = new LoadChangedIssuesStep(periodHolder, protoIssueCache, changedIssuesRepository); + } + + @Test + public void getDescription_shouldReturnDescription() { + assertThat(underTest.getDescription()).isEqualTo("Load changed issues for indexation"); + } + + @Test + public void execute_whenIssueIsNew_shouldLoadIssue() { + protoIssueCache.newAppender() + .append(newDefaultIssue().setNew(true)) + .close(); + + underTest.execute(mock(ComputationStep.Context.class)); + + verify(changedIssuesRepository).addIssueKey("issueKey1"); + } + + @Test + public void execute_whenIssueIssCopied_shouldLoadIssue() { + protoIssueCache.newAppender() + .append(newDefaultIssue().setCopied(true)) + .close(); + + underTest.execute(mock(ComputationStep.Context.class)); + + verify(changedIssuesRepository).addIssueKey("issueKey1"); + } + + @Test + public void execute_whenIssueIsChanged_shouldLoadIssue() { + protoIssueCache.newAppender() + .append(newDefaultIssue().setChanged(true)) + .close(); + + underTest.execute(mock(ComputationStep.Context.class)); + + verify(changedIssuesRepository).addIssueKey("issueKey1"); + } + + @Test + public void execute_whenIssueIsNoLongerNewCodeReferenceIssue_shouldLoadIssue() { + when(periodHolder.hasPeriod()).thenReturn(true); + when(periodHolder.getPeriod()).thenReturn(new Period("REFERENCE_BRANCH", null, null)); + + protoIssueCache.newAppender() + .append(newDefaultIssue() + .setIsNoLongerNewCodeReferenceIssue(true) + .setNew(false) + .setCopied(false) + .setChanged(false)) + .close(); + + underTest.execute(mock(ComputationStep.Context.class)); + + verify(changedIssuesRepository).addIssueKey("issueKey1"); + } + + @Test + public void execute_whenIssueIsToBeMigratedAsNewCodeReferenceIssue_shouldLoadIssue() { + when(periodHolder.hasPeriod()).thenReturn(true); + when(periodHolder.getPeriod()).thenReturn(new Period("REFERENCE_BRANCH", null, null)); + + protoIssueCache.newAppender() + .append(newDefaultIssue() + .setIsOnChangedLine(true) + .setIsNewCodeReferenceIssue(false) + .setIsNoLongerNewCodeReferenceIssue(false) + .setNew(false) + .setCopied(false) + .setChanged(false)) + .close(); + + underTest.execute(mock(ComputationStep.Context.class)); + + verify(changedIssuesRepository).addIssueKey("issueKey1"); + } + + private static DefaultIssue newDefaultIssue() { + return new DefaultIssue() + .setKey("issueKey1") + .setType(RuleType.CODE_SMELL) + .setRuleKey(RuleKey.of("repo", "ruleKey1")) + .setComponentUuid("fileUuid") + .setComponentKey("fileKey") + .setProjectUuid("projectUuid") + .setProjectKey("projectKey") + .setSeverity(BLOCKER) + .setStatus(STATUS_OPEN) + .setCreationDate(new Date()) + .setSelectedAt(1L) + .addImpact(SoftwareQuality.SECURITY, Severity.MEDIUM); + } + +} diff --git a/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java b/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java index 8271c9b8989..55fbab36527 100644 --- a/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java +++ b/server/sonar-server-common/src/it/java/org/sonar/server/issue/index/IssueIndexerIT.java @@ -609,7 +609,7 @@ public class IssueIndexerIT { } @Test - public void indexOnAnalysis_whenChangedComponents_shouldReindexOnlyChangedComponents() { + public void indexOnAnalysis_whenDiffProvided_shouldReindexOnlyIssueDifference() { RuleDto rule = db.rules().insert(); ComponentDto mainBranchComponent = db.components().insertPrivateProject().getMainBranchComponent(); ComponentDto changedComponent1 = db.components().insertComponent(newFileDto(mainBranchComponent)); @@ -621,22 +621,22 @@ public class IssueIndexerIT { db.issues().insert(rule, mainBranchComponent, unchangedComponent); db.issues().insert(rule, mainBranchComponent, unchangedComponent); - underTest.indexOnAnalysis(mainBranchComponent.uuid(), Set.of(unchangedComponent.uuid())); + underTest.indexOnAnalysis(mainBranchComponent.uuid(), Set.of(changedIssue1.getKee(), changedIssue2.getKee(), changedIssue3.getKee())); assertThatIndexHasOnly(changedIssue1, changedIssue2, changedIssue3); } @Test - public void indexOnAnalysis_whenEmptyUnchangedComponents_shouldReindexEverything() { + public void indexOnAnalysis_whenEmptyDiffToIndex_shouldSkipIndexing() { RuleDto rule = db.rules().insert(); ComponentDto mainBranchComponent = db.components().insertPrivateProject().getMainBranchComponent(); ComponentDto changedComponent = db.components().insertComponent(newFileDto(mainBranchComponent)); - IssueDto changedIssue1 = db.issues().insert(rule, mainBranchComponent, changedComponent); - IssueDto changedIssue2 = db.issues().insert(rule, mainBranchComponent, changedComponent); + db.issues().insert(rule, mainBranchComponent, changedComponent); + db.issues().insert(rule, mainBranchComponent, changedComponent); underTest.indexOnAnalysis(mainBranchComponent.uuid(), Set.of()); - assertThatIndexHasOnly(changedIssue1, changedIssue2); + assertThatIndexHasSize(0); } @Test @@ -650,6 +650,11 @@ public class IssueIndexerIT { assertThat(es.getDocuments(TYPE_ISSUE)).isEmpty(); } + @Test + public void supportDiffIndex_shouldReturnTrue() { + assertThat(underTest.supportDiffIndexing()).isTrue(); + } + private void addIssueToIndex(String projectUuid, String branchUuid, String issueKey) { es.putDocuments(TYPE_ISSUE, newDoc().setKey(issueKey).setProjectUuid(projectUuid).setBranchUuid(branchUuid)); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/EntityDefinitionIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/EntityDefinitionIndexer.java index d6a42747bf3..500357ba539 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/EntityDefinitionIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/EntityDefinitionIndexer.java @@ -84,11 +84,6 @@ public class EntityDefinitionIndexer implements EventIndexer, AnalysisIndexer, N @Override public void indexOnAnalysis(String branchUuid) { - indexOnAnalysis(branchUuid, Set.of()); - } - - @Override - public void indexOnAnalysis(String branchUuid, Set unchangedComponentUuids) { try (DbSession dbSession = dbClient.openSession(false)) { Optional branchDto = dbClient.branchDao().selectByUuid(dbSession, branchUuid); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/AnalysisIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/AnalysisIndexer.java index e76d1f95d18..e7d535d8dfd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/AnalysisIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/AnalysisIndexer.java @@ -19,7 +19,7 @@ */ package org.sonar.server.es; -import java.util.Set; +import java.util.Collection; /** * Indexers that should be called when a project branch is analyzed @@ -33,11 +33,22 @@ public interface AnalysisIndexer { void indexOnAnalysis(String branchUuid); /** - * This method is called when an analysis must be indexed. + * This method is called when {@link #supportDiffIndexing()} is true. * - * @param branchUuid UUID of a project or application branch - * @param unchangedComponentUuids UUIDs of components that didn't change in this analysis. - * Indexers can be optimized by not re-indexing data related to these components. + * @param diffToIndex Diff of uuids of indexed entities (issue keys, project uuids, etc.) + */ + default void indexOnAnalysis(String branchUuid, Collection diffToIndex) { + if (!supportDiffIndexing()) { + throw new IllegalStateException("Diff indexing is not supported by this indexer " + getClass().getName()); + } + } + + /** + * This method indicates if the indexer supports diff indexing during analysis. + * + * @return true if it is supported, false otherwise */ - void indexOnAnalysis(String branchUuid, Set unchangedComponentUuids); + default boolean supportDiffIndexing() { + return false; + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 17d917b2581..e971c5d313a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -117,24 +117,32 @@ public class IssueIndexer implements EventIndexer, AnalysisIndexer, NeedAuthoriz public void indexAllIssues() { try (IssueIterator issues = issueIteratorFactory.createForAll()) { - doIndex(issues, Set.of()); + doIndex(issues); } } @Override public void indexOnAnalysis(String branchUuid) { try (IssueIterator issues = issueIteratorFactory.createForBranch(branchUuid)) { - doIndex(issues, Set.of()); + doIndex(issues); } } @Override - public void indexOnAnalysis(String branchUuid, Set unchangedComponentUuids) { - try (IssueIterator issues = issueIteratorFactory.createForBranch(branchUuid)) { - doIndex(issues, unchangedComponentUuids); + public void indexOnAnalysis(String branchUuid, Collection diffToIndex) { + if (diffToIndex.isEmpty()) { + return; + } + try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(diffToIndex)) { + doIndex(issues); } } + @Override + public boolean supportDiffIndexing() { + return true; + } + public void indexProject(String projectUuid) { asyncIssueIndexing.triggerForProject(projectUuid); } @@ -162,7 +170,7 @@ public class IssueIndexer implements EventIndexer, AnalysisIndexer, NeedAuthoriz emptyList(); case DELETION, SWITCH_OF_MAIN_BRANCH -> { - //switch of main branch requires to reindex the project issues + // switch of main branch requires to reindex the project issues List items = createBranchRecoveryItems(branchUuids); yield dbClient.esQueueDao().insert(dbSession, items); } @@ -304,25 +312,19 @@ public class IssueIndexer implements EventIndexer, AnalysisIndexer, NeedAuthoriz @VisibleForTesting protected void index(Iterator issues) { - doIndex(issues, Set.of()); + doIndex(issues); } - private void doIndex(Iterator issues, Set unchangedComponentUuids) { + private void doIndex(Iterator issues) { BulkIndexer bulk = createBulkIndexer(IndexingListener.FAIL_ON_ERROR); bulk.start(); while (issues.hasNext()) { IssueDoc issue = issues.next(); - if (shouldReindexIssue(issue, unchangedComponentUuids)) { - bulk.add(newIndexRequest(issue)); - } + bulk.add(newIndexRequest(issue)); } bulk.stop(); } - private static boolean shouldReindexIssue(IssueDoc issue, Set unchangedComponentUuids) { - return issue.getFields().get(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID) == null || !unchangedComponentUuids.contains(issue.componentUuid()); - } - private static IndexRequest newIndexRequest(IssueDoc issue) { return new IndexRequest(TYPE_ISSUE.getMainType().getIndex().getName()) .id(issue.getId()) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index 03114a93a49..d698e08a619 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -92,11 +92,6 @@ public class ProjectMeasuresIndexer implements EventIndexer, AnalysisIndexer, Ne @Override public void indexOnAnalysis(String branchUuid) { - indexOnAnalysis(branchUuid, Set.of()); - } - - @Override - public void indexOnAnalysis(String branchUuid, Set unchangedComponentUuids) { doIndex(Size.REGULAR, branchUuid); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/AnalysisIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/AnalysisIndexerTest.java new file mode 100644 index 00000000000..480625bd657 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/AnalysisIndexerTest.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Set; +import org.junit.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatCode; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +public class AnalysisIndexerTest { + + @Test + public void indexOnAnalysis_whenDiffIndexingNotSupported_shouldThrowISE() { + AnalysisIndexer analysisIndexer = new IndexerNotSupportingDiffs(); + + assertThatThrownBy(() -> analysisIndexer.indexOnAnalysis("branchUuid", Set.of("diffToIndex"))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Diff indexing is not supported by this indexer org.sonar.server.es.AnalysisIndexerTest$IndexerNotSupportingDiffs"); + + } + + @Test + public void indexOnAnalysis_whenDiffIndexingSupported_shouldNotThrowISE() { + AnalysisIndexer analysisIndexer = new IndexerSupportingDiffs(); + + assertThatCode(() -> analysisIndexer.indexOnAnalysis("branchUuid", Set.of("diffToIndex"))) + .doesNotThrowAnyException(); + } + + private static class IndexerNotSupportingDiffs implements AnalysisIndexer { + @Override + public void indexOnAnalysis(String branchUuid) { + // no-op + } + + } + + private static class IndexerSupportingDiffs implements AnalysisIndexer { + @Override + public void indexOnAnalysis(String branchUuid) { + // no-op + } + + @Override + public boolean supportDiffIndexing() { + return true; + } + } + +} diff --git a/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java index d3b78b0e5f1..f8a09d3d175 100644 --- a/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-webserver-es/src/testFixtures/java/org/sonar/server/permission/index/FooIndexer.java @@ -20,7 +20,6 @@ package org.sonar.server.permission.index; import java.util.Optional; -import java.util.Set; import org.elasticsearch.action.index.IndexRequest; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -50,24 +49,17 @@ public class FooIndexer implements AnalysisIndexer, NeedAuthorizationIndexer { @Override public void indexOnAnalysis(String branchUuid) { - indexOnAnalysis(branchUuid, Set.of()); - } - - @Override - public void indexOnAnalysis(String branchUuid, Set unchangedComponentUuids) { - try(DbSession dbSession = dbClient.openSession(true)){ + try (DbSession dbSession = dbClient.openSession(true)) { Optional branchDto = dbClient.branchDao().selectByUuid(dbSession, branchUuid); if (branchDto.isEmpty()) { //For portfolio, adding branchUuid directly addToIndex(branchUuid, "bar"); addToIndex(branchUuid, "baz"); - }else{ + } else { addToIndex(branchDto.get().getProjectUuid(), "bar"); addToIndex(branchDto.get().getProjectUuid(), "baz"); } } - - } private void addToIndex(String projectUuid, String name) { -- 2.39.5