diff options
60 files changed, 1172 insertions, 178 deletions
diff --git a/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java b/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java index f0f2a205930..1c155e145e6 100644 --- a/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java +++ b/server/sonar-ce-common/src/test/java/org/sonar/ce/queue/CeQueueImplTest.java @@ -35,7 +35,6 @@ import org.sonar.ce.task.CeTask; import org.sonar.core.util.SequenceUuidFactory; import org.sonar.core.util.UuidFactory; import org.sonar.core.util.UuidFactoryFast; -import org.sonar.core.util.UuidFactoryImpl; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.ce.CeActivityDto; @@ -397,7 +396,7 @@ public class CeQueueImplTest { @Test public void fail_to_cancel_if_in_progress() { submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(11))); - CeQueueDto ceQueueDto = db.getDbClient().ceQueueDao().peek(session, WORKER_UUID).get(); + CeQueueDto ceQueueDto = db.getDbClient().ceQueueDao().peek(session, WORKER_UUID, false).get(); expectedException.expect(IllegalStateException.class); expectedException.expectMessage(startsWith("Task is in progress and can't be canceled")); @@ -411,7 +410,7 @@ public class CeQueueImplTest { CeTask pendingTask1 = submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(12))); CeTask pendingTask2 = submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(12))); - db.getDbClient().ceQueueDao().peek(session, WORKER_UUID); + db.getDbClient().ceQueueDao().peek(session, WORKER_UUID, false); int canceledCount = underTest.cancelAll(); assertThat(canceledCount).isEqualTo(2); @@ -438,7 +437,7 @@ public class CeQueueImplTest { @Test public void pauseWorkers_marks_workers_as_pausing_if_some_tasks_in_progress() { submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(12))); - db.getDbClient().ceQueueDao().peek(session, WORKER_UUID); + db.getDbClient().ceQueueDao().peek(session, WORKER_UUID, false); // task is in-progress assertThat(underTest.getWorkersPauseStatus()).isEqualTo(CeQueue.WorkersPauseStatus.RESUMED); @@ -459,7 +458,7 @@ public class CeQueueImplTest { @Test public void resumeWorkers_resumes_pausing_workers() { submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(12))); - db.getDbClient().ceQueueDao().peek(session, WORKER_UUID); + db.getDbClient().ceQueueDao().peek(session, WORKER_UUID, false); // task is in-progress underTest.pauseWorkers(); @@ -481,7 +480,7 @@ public class CeQueueImplTest { @Test public void fail_in_progress_task() { CeTask task = submit(CeTaskTypes.REPORT, newComponent(randomAlphabetic(12))); - CeQueueDto queueDto = db.getDbClient().ceQueueDao().peek(db.getSession(), WORKER_UUID).get(); + CeQueueDto queueDto = db.getDbClient().ceQueueDao().peek(db.getSession(), WORKER_UUID, false).get(); underTest.fail(db.getSession(), queueDto, "TIMEOUT", "Failed on timeout"); diff --git a/server/sonar-ce-task-projectanalysis/build.gradle b/server/sonar-ce-task-projectanalysis/build.gradle index fdef5bf68b5..b6d0b390b0d 100644 --- a/server/sonar-ce-task-projectanalysis/build.gradle +++ b/server/sonar-ce-task-projectanalysis/build.gradle @@ -32,6 +32,7 @@ dependencies { compile 'org.picocontainer:picocontainer' compile project(':sonar-core') + compile project(':server:sonar-ce-common') compile project(':server:sonar-ce-task') compile project(':server:sonar-db-migration') compile project(':server:sonar-process') diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStep.java new file mode 100644 index 00000000000..a291b7006ea --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStep.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.server.issue.index.IssueIndexer; + +public final class IndexIssuesStep implements ComputationStep { + private static final Logger LOG = Loggers.get(IndexIssuesStep.class); + private final CeTask ceTask; + private final DbClient dbClient; + private final IssueIndexer issueIndexer; + + public IndexIssuesStep(CeTask ceTask, DbClient dbClient, IssueIndexer issueIndexer) { + this.ceTask = ceTask; + this.dbClient = dbClient; + this.issueIndexer = issueIndexer; + } + + @Override + public void execute(Context context) { + String branchUuid = ceTask.getComponent().orElseThrow(() -> new UnsupportedOperationException("component not found in task")).getUuid(); + + try (DbSession dbSession = dbClient.openSession(false)) { + dbClient.branchDao().selectByUuid(dbSession, branchUuid) + .ifPresent(branchDto -> { + + if (branchDto.isNeedIssueSync()) { + LOG.info("indexing issues of branch {}", branchUuid); + issueIndexer.indexOnAnalysis(branchUuid); + dbClient.branchDao().updateNeedIssueSync(dbSession, branchUuid, false); + dbSession.commit(); + } else { + // branch has been analyzed since task was created, do not index issues twice + LOG.debug("issues of branch {} are already in sync", branchUuid); + } + }); + + } + } + + @Override + public String getDescription() { + return "index issues"; + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskModule.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskModule.java new file mode 100644 index 00000000000..23eb54fdf1e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskModule.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import org.sonar.core.platform.Module; + +public class IssueSyncTaskModule extends Module { + + @Override + protected void configureModule() { + add( + IssueSyncTaskProcessor.class); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessor.java new file mode 100644 index 00000000000..53f1826975b --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessor.java @@ -0,0 +1,86 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import javax.annotation.CheckForNull; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.CeTaskResult; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.container.TaskContainerImpl; +import org.sonar.ce.task.projectanalysis.step.AbstractComputationSteps; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.ce.task.step.ComputationStepExecutor; +import org.sonar.ce.task.taskprocessor.CeTaskProcessor; +import org.sonar.core.platform.ComponentContainer; +import org.sonar.core.platform.ContainerPopulator; + +import static org.sonar.db.ce.CeTaskTypes.BRANCH_ISSUE_SYNC; + +public class IssueSyncTaskProcessor implements CeTaskProcessor { + private static final Set<String> HANDLED_TYPES = ImmutableSet.of(BRANCH_ISSUE_SYNC); + + private final ComponentContainer ceEngineContainer; + + public IssueSyncTaskProcessor(ComponentContainer ceEngineContainer) { + this.ceEngineContainer = ceEngineContainer; + } + + @Override + public Set<String> getHandledCeTaskTypes() { + return HANDLED_TYPES; + } + + @CheckForNull + @Override + public CeTaskResult process(CeTask task) { + try (TaskContainer container = new TaskContainerImpl(ceEngineContainer, newContainerPopulator(task))) { + container.bootup(); + container.getComponentByType(ComputationStepExecutor.class).execute(); + } + return null; + } + + static ContainerPopulator<TaskContainer> newContainerPopulator(CeTask task) { + return taskContainer -> { + taskContainer.add(task); + taskContainer.add(IndexIssuesStep.class); + taskContainer.add(new SyncComputationSteps(taskContainer)); + taskContainer.add(ComputationStepExecutor.class); + }; + } + + public static final class SyncComputationSteps extends AbstractComputationSteps { + + public SyncComputationSteps(ContainerPopulator.Container container) { + super(container); + } + + @Override + public List<Class<? extends ComputationStep>> orderedStepClasses() { + return Collections.singletonList(IndexIssuesStep.class); + } + + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStepTest.java new file mode 100644 index 00000000000..8f004404c38 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStepTest.java @@ -0,0 +1,123 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.util.Optional; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.utils.System2; +import org.sonar.ce.task.CeTask; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; +import org.sonar.server.es.EsTester; +import org.sonar.server.issue.index.IssueIndexer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.sonar.db.component.BranchType.BRANCH; + +public class IndexIssuesStepTest { + + private String BRANCH_UUID = "branch_uuid"; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + + private DbClient dbClient = dbTester.getDbClient(); + + private CeTask.Component component = new CeTask.Component(BRANCH_UUID, "component key", "component name"); + private CeTask ceTask = new CeTask.Builder() + .setOrganizationUuid("organizationUuid") + .setType("type") + .setUuid("uuid") + .setComponent(component) + .setMainComponent(component) + .build(); + + @Rule + public EsTester es = EsTester.create(); + private System2 system2 = new System2(); + @Rule + public DbTester db = DbTester.create(system2); + + private IssueIndexer issueIndexer = mock(IssueIndexer.class); + + private IndexIssuesStep underTest = new IndexIssuesStep(ceTask, dbClient, issueIndexer); + + @Test + public void execute() { + BranchDto branchDto = new BranchDto() + .setBranchType(BRANCH) + .setKey("branchName") + .setUuid(BRANCH_UUID) + .setProjectUuid("project_uuid") + .setNeedIssueSync(true); + dbClient.branchDao().insert(dbTester.getSession(), branchDto); + dbTester.commit(); + + underTest.execute(() -> null); + + verify(issueIndexer, times(1)).indexOnAnalysis(eq(BRANCH_UUID)); + Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID); + assertThat(branch.get().isNeedIssueSync()).isFalse(); + } + + @Test + public void execute_on_already_indexed_branch() { + BranchDto branchDto = new BranchDto() + .setBranchType(BRANCH) + .setKey("branchName") + .setUuid(BRANCH_UUID) + .setProjectUuid("project_uuid") + .setNeedIssueSync(false); + dbClient.branchDao().insert(dbTester.getSession(), branchDto); + dbTester.commit(); + + underTest.execute(() -> null); + + verify(issueIndexer, times(0)).indexOnAnalysis(eq(BRANCH_UUID)); + } + + @Test + public void fail_if_missing_component_in_task() { + CeTask ceTask = new CeTask.Builder() + .setOrganizationUuid("organizationUuid") + .setType("type") + .setUuid("uuid") + .setComponent(null) + .setMainComponent(null) + .build(); + IndexIssuesStep underTest = new IndexIssuesStep(ceTask, dbClient, issueIndexer); + + expectedException.expect(UnsupportedOperationException.class); + expectedException.expectMessage("component not found in task"); + + underTest.execute(() -> null); + } + +} diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessorTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessorTest.java new file mode 100644 index 00000000000..cbc4bfb530e --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/taskprocessor/IssueSyncTaskProcessorTest.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.taskprocessor; + +import java.util.List; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.mockito.Mockito; +import org.sonar.ce.task.CeTask; +import org.sonar.ce.task.container.TaskContainer; +import org.sonar.ce.task.step.ComputationStep; +import org.sonar.core.platform.ComponentContainer; + +import static org.mockito.ArgumentMatchers.any; +import static org.sonar.ce.task.projectanalysis.taskprocessor.IssueSyncTaskProcessor.*; +import static org.sonar.db.ce.CeTaskTypes.BRANCH_ISSUE_SYNC; + +public class IssueSyncTaskProcessorTest { + + private ComponentContainer ceEngineContainer = Mockito.mock(ComponentContainer.class); + + private IssueSyncTaskProcessor underTest = new IssueSyncTaskProcessor(ceEngineContainer); + private TaskContainer container = Mockito.spy(TaskContainer.class); + + @Test + public void getHandledCeTaskTypes() { + Assertions.assertThat(underTest.getHandledCeTaskTypes()).containsExactly(BRANCH_ISSUE_SYNC); + } + + @Test + public void newContainerPopulator() { + CeTask task = new CeTask.Builder() + .setOrganizationUuid("ORGANIZATION_UUID") + .setUuid("TASK_UUID") + .setType("Type") + .build(); + + IssueSyncTaskProcessor.newContainerPopulator(task).populateContainer(container); + Mockito.verify(container, Mockito.times(4)).add(any()); + } + + @Test + public void orderedStepClasses(){ + SyncComputationSteps syncComputationSteps = new SyncComputationSteps(null); + + List<Class<? extends ComputationStep>> steps = syncComputationSteps.orderedStepClasses(); + + Assertions.assertThat(steps).containsExactly(IndexIssuesStep.class); + } + +} diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java index a455b71f39c..f7e3793a818 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java @@ -53,6 +53,7 @@ import org.sonar.ce.async.SynchronousAsyncExecution; import org.sonar.ce.cleaning.CeCleaningModule; import org.sonar.ce.cleaning.NoopCeCleaningSchedulerImpl; import org.sonar.ce.db.ReadOnlyPropertiesDao; +import org.sonar.ce.issue.index.NoAsyncIssueIndexing; import org.sonar.ce.logging.CeProcessLogging; import org.sonar.ce.monitoring.CEQueueStatusImpl; import org.sonar.ce.monitoring.DistributedCEQueueStatusImpl; @@ -64,6 +65,7 @@ import org.sonar.ce.task.projectanalysis.ProjectAnalysisTaskModule; import org.sonar.ce.task.projectanalysis.analysis.ProjectConfigurationFactory; import org.sonar.ce.task.projectanalysis.issue.AdHocRuleCreator; import org.sonar.ce.task.projectanalysis.notification.ReportAnalysisFailureNotificationModule; +import org.sonar.ce.task.projectanalysis.taskprocessor.IssueSyncTaskModule; import org.sonar.ce.taskprocessor.CeProcessingScheduler; import org.sonar.ce.taskprocessor.CeTaskProcessorModule; import org.sonar.core.component.DefaultResourceTypes; @@ -402,6 +404,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { // issues IssueStorage.class, + NoAsyncIssueIndexing.class, IssueIndexer.class, IssueIteratorFactory.class, IssueFieldsSetter.class, // used in Web Services and CE's DebtCalculator @@ -439,6 +442,7 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer { CeHttpModule.class, CeTaskCommonsModule.class, ProjectAnalysisTaskModule.class, + IssueSyncTaskModule.class, CeTaskProcessorModule.class, OfficialDistribution.class, diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/issue/index/NoAsyncIssueIndexing.java b/server/sonar-ce/src/main/java/org/sonar/ce/issue/index/NoAsyncIssueIndexing.java new file mode 100644 index 00000000000..ba4d525c4dd --- /dev/null +++ b/server/sonar-ce/src/main/java/org/sonar/ce/issue/index/NoAsyncIssueIndexing.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.issue.index; + +import org.sonar.api.ce.ComputeEngineSide; +import org.sonar.server.issue.index.AsyncIssueIndexing; + +@ComputeEngineSide +public class NoAsyncIssueIndexing implements AsyncIssueIndexing { + @Override + public void triggerOnIndexCreation() { + throw new IllegalStateException("Async issue indexing should not be triggered in Compute Engine"); + } +} diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java index 2058d7a19c5..30466805efc 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueue.java @@ -40,6 +40,8 @@ public interface InternalCeQueue extends CeQueue { * The task status is changed to {@link org.sonar.db.ce.CeQueueDto.Status#IN_PROGRESS}. * Does not return anything if workers are paused or being paused (see {@link #getWorkersPauseStatus()}. * + * @param excludeIndexationJob change the underlying request to exclude indexation tasks. + * * <p>Only a single task can be peeked by project.</p> * * <p>An unchecked exception may be thrown on technical errors (db connection, ...).</p> @@ -47,7 +49,7 @@ public interface InternalCeQueue extends CeQueue { * <p>Tasks which have been executed twice already but are still {@link org.sonar.db.ce.CeQueueDto.Status#PENDING} * are ignored</p> */ - Optional<CeTask> peek(String workerUuid); + Optional<CeTask> peek(String workerUuid, boolean excludeIndexationJob); /** * Removes a task from the queue and registers it to past activities. This method diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java index 5bdab435748..711961b1f95 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/queue/InternalCeQueueImpl.java @@ -73,7 +73,7 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue } @Override - public Optional<CeTask> peek(String workerUuid) { + public Optional<CeTask> peek(String workerUuid, boolean excludeIndexationJob) { requireNonNull(workerUuid, "workerUuid can't be null"); if (computeEngineStatus.getStatus() != ComputeEngineStatus.Status.STARTED || getWorkersPauseStatus() != WorkersPauseStatus.RESUMED) { @@ -86,20 +86,20 @@ public class InternalCeQueueImpl extends CeQueueImpl implements InternalCeQueue dbSession.commit(); LOG.debug("{} in progress tasks reset for worker uuid {}", i, workerUuid); } - Optional<CeQueueDto> opt = ceQueueDao.peek(dbSession, workerUuid); - if (opt.isPresent()) { - CeQueueDto taskDto = opt.get(); - Map<String, ComponentDto> componentsByUuid = loadComponentDtos(dbSession, taskDto); - Map<String, String> characteristics = dbClient.ceTaskCharacteristicsDao().selectByTaskUuids(dbSession, singletonList(taskDto.getUuid())).stream() - .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); - - CeTask task = convertToTask(dbSession, taskDto, characteristics, - ofNullable(taskDto.getComponentUuid()).map(componentsByUuid::get).orElse(null), - ofNullable(taskDto.getMainComponentUuid()).map(componentsByUuid::get).orElse(null)); - queueStatus.addInProgress(); - return Optional.of(task); + Optional<CeQueueDto> opt = ceQueueDao.peek(dbSession, workerUuid, excludeIndexationJob); + if (!opt.isPresent()) { + return Optional.empty(); } - return Optional.empty(); + CeQueueDto taskDto = opt.get(); + Map<String, ComponentDto> componentsByUuid = loadComponentDtos(dbSession, taskDto); + Map<String, String> characteristics = dbClient.ceTaskCharacteristicsDao().selectByTaskUuids(dbSession, singletonList(taskDto.getUuid())).stream() + .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); + + CeTask task = convertToTask(dbSession, taskDto, characteristics, + ofNullable(taskDto.getComponentUuid()).map(componentsByUuid::get).orElse(null), + ofNullable(taskDto.getMainComponentUuid()).map(componentsByUuid::get).orElse(null)); + queueStatus.addInProgress(); + return Optional.of(task); } } diff --git a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java index ef6cad9a8ab..10c234639f5 100644 --- a/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java +++ b/server/sonar-ce/src/main/java/org/sonar/ce/taskprocessor/CeWorkerImpl.java @@ -61,17 +61,21 @@ public class CeWorkerImpl implements CeWorker { private final CeWorkerController ceWorkerController; private final List<ExecutionListener> listeners; private final AtomicReference<RunningState> runningState = new AtomicReference<>(); + private boolean indexationTaskLookupEnabled; + private boolean excludeIndexationJob; public CeWorkerImpl(int ordinal, String uuid, - InternalCeQueue queue, CeTaskProcessorRepository taskProcessorRepository, - CeWorkerController ceWorkerController, - ExecutionListener... listeners) { + InternalCeQueue queue, CeTaskProcessorRepository taskProcessorRepository, + CeWorkerController ceWorkerController, + ExecutionListener... listeners) { this.ordinal = checkOrdinal(ordinal); this.uuid = uuid; this.queue = queue; this.taskProcessorRepository = taskProcessorRepository; this.ceWorkerController = ceWorkerController; this.listeners = Arrays.asList(listeners); + indexationTaskLookupEnabled = true; + excludeIndexationJob = false; } private static int checkOrdinal(int ordinal) { @@ -119,7 +123,7 @@ public class CeWorkerImpl implements CeWorker { localRunningState = new RunningState(currentThread); if (!runningState.compareAndSet(null, localRunningState)) { LOG.warn("Worker {} (UUID=%s) starts executing with new Thread {} while running state isn't null. " + - "Forcefully updating Workers's running state to new Thread.", + "Forcefully updating Workers's running state to new Thread.", getOrdinal(), getUUID(), currentThread); runningState.set(localRunningState); } @@ -138,7 +142,7 @@ public class CeWorkerImpl implements CeWorker { localRunningState.runningThread.setName(oldName); if (!runningState.compareAndSet(localRunningState, null)) { LOG.warn("Worker {} (UUID=%s) ending execution in Thread {} while running state has already changed." + - " Keeping this new state.", + " Keeping this new state.", getOrdinal(), getUUID(), localRunningState.runningThread); } } @@ -154,7 +158,7 @@ public class CeWorkerImpl implements CeWorker { } try (CeWorkerController.ProcessingRecorderHook processing = ceWorkerController.registerProcessingFor(this); - ExecuteTask executeTask = new ExecuteTask(localRunningState, ceTask.get())) { + ExecuteTask executeTask = new ExecuteTask(localRunningState, ceTask.get())) { executeTask.run(); } catch (Exception e) { LOG.error(format("An error occurred while executing task with uuid '%s'", ceTask.get().getUuid()), e); @@ -164,13 +168,35 @@ public class CeWorkerImpl implements CeWorker { private Optional<CeTask> tryAndFindTaskToExecute() { try { - return queue.peek(uuid); + if (indexationTaskLookupEnabled) { + return tryAndFindTaskToExecuteIncludingIndexation(); + } else { + return queue.peek(uuid, true); + } } catch (Exception e) { LOG.error("Failed to pop the queue of analysis reports", e); } return Optional.empty(); } + private Optional<CeTask> tryAndFindTaskToExecuteIncludingIndexation() { + excludeIndexationJob = !excludeIndexationJob; + Optional<CeTask> peek = queue.peek(uuid, excludeIndexationJob); + if (peek.isPresent()) { + return peek; + } + if (excludeIndexationJob) { + peek = queue.peek(uuid, false); + if (peek.isPresent()) { + return peek; + } + // do not lookup for indexation tasks anymore + indexationTaskLookupEnabled = false; + LOG.info(String.format("worker %s found no pending task (including indexation task). Disabling indexation task lookup for this worker until next SonarQube restart.", uuid)); + } + return Optional.empty(); + } + private final class ExecuteTask implements Runnable, AutoCloseable { private final CeTask task; private final RunningState localRunningState; @@ -237,7 +263,7 @@ public class CeWorkerImpl implements CeWorker { } private void finalizeTask(CeTask task, Profiler ceProfiler, CeActivityDto.Status status, - @Nullable CeTaskResult taskResult, @Nullable Throwable error) { + @Nullable CeTaskResult taskResult, @Nullable Throwable error) { try { queue.remove(task, status, taskResult, error); } catch (Exception e) { diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java index d31c514d45a..922da41cdc3 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/queue/InternalCeQueueImplTest.java @@ -159,18 +159,18 @@ public class InternalCeQueueImplTest { expectedException.expect(NullPointerException.class); expectedException.expectMessage("workerUuid can't be null"); - underTest.peek(null); + underTest.peek(null, false); } @Test public void test_remove() { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); underTest.remove(peek.get(), CeActivityDto.Status.SUCCESS, null, null); // queue is empty assertThat(db.getDbClient().ceQueueDao().selectByUuid(db.getSession(), task.getUuid()).isPresent()).isFalse(); - assertThat(underTest.peek(WORKER_UUID_2).isPresent()).isFalse(); + assertThat(underTest.peek(WORKER_UUID_2, false).isPresent()).isFalse(); // available in history Optional<CeActivityDto> history = db.getDbClient().ceActivityDao().selectByUuid(db.getSession(), task.getUuid()); @@ -199,7 +199,7 @@ public class InternalCeQueueImplTest { @Test public void remove_does_not_set_analysisUuid_in_CeActivity_when_CeTaskResult_has_no_analysis_uuid() { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); underTest.remove(peek.get(), CeActivityDto.Status.SUCCESS, newTaskResult(null), null); // available in history @@ -212,7 +212,7 @@ public class InternalCeQueueImplTest { public void remove_sets_analysisUuid_in_CeActivity_when_CeTaskResult_has_analysis_uuid() { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_2); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_2, false); underTest.remove(peek.get(), CeActivityDto.Status.SUCCESS, newTaskResult(AN_ANALYSIS_UUID), null); // available in history @@ -226,7 +226,7 @@ public class InternalCeQueueImplTest { Throwable error = new NullPointerException("Fake NPE to test persistence to DB"); CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); underTest.remove(peek.get(), CeActivityDto.Status.FAILED, null, error); Optional<CeActivityDto> activityDto = db.getDbClient().ceActivityDao().selectByUuid(session, task.getUuid()); @@ -242,7 +242,7 @@ public class InternalCeQueueImplTest { Throwable error = new TypedExceptionImpl("aType", "aMessage"); CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); underTest.remove(peek.get(), CeActivityDto.Status.FAILED, null, error); CeActivityDto activityDto = db.getDbClient().ceActivityDao().selectByUuid(session, task.getUuid()).get(); @@ -348,7 +348,7 @@ public class InternalCeQueueImplTest { public void test_peek() { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); assertThat(peek.isPresent()).isTrue(); assertThat(peek.get().getUuid()).isEqualTo(task.getUuid()); assertThat(peek.get().getType()).isEqualTo(CeTaskTypes.REPORT); @@ -356,7 +356,7 @@ public class InternalCeQueueImplTest { assertThat(peek.get().getMainComponent()).contains(peek.get().getComponent().get()); // no more pending tasks - peek = underTest.peek(WORKER_UUID_2); + peek = underTest.peek(WORKER_UUID_2, false); assertThat(peek.isPresent()).isFalse(); } @@ -366,7 +366,7 @@ public class InternalCeQueueImplTest { ComponentDto branch = db.components().insertProjectBranch(project); CeTask task = submit(CeTaskTypes.REPORT, branch); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); assertThat(peek.isPresent()).isTrue(); assertThat(peek.get().getUuid()).isEqualTo(task.getUuid()); assertThat(peek.get().getType()).isEqualTo(CeTaskTypes.REPORT); @@ -374,7 +374,7 @@ public class InternalCeQueueImplTest { assertThat(peek.get().getMainComponent()).contains(new CeTask.Component(project.uuid(), project.getDbKey(), project.name())); // no more pending tasks - peek = underTest.peek(WORKER_UUID_2); + peek = underTest.peek(WORKER_UUID_2, false); assertThat(peek.isPresent()).isFalse(); } @@ -383,11 +383,11 @@ public class InternalCeQueueImplTest { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); underTest.pauseWorkers(); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); assertThat(peek).isEmpty(); underTest.resumeWorkers(); - peek = underTest.peek(WORKER_UUID_1); + peek = underTest.peek(WORKER_UUID_1, false); assertThat(peek).isPresent(); assertThat(peek.get().getUuid()).isEqualTo(task.getUuid()); } @@ -401,7 +401,7 @@ public class InternalCeQueueImplTest { makeInProgress(dto, "foo"); db.commit(); - assertThat(underTest.peek(WORKER_UUID_1)).isEmpty(); + assertThat(underTest.peek(WORKER_UUID_1, false)).isEmpty(); } @Test @@ -409,7 +409,7 @@ public class InternalCeQueueImplTest { submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); when(computeEngineStatus.getStatus()).thenReturn(STOPPING); - Optional<CeTask> peek = underTest.peek(WORKER_UUID_1); + Optional<CeTask> peek = underTest.peek(WORKER_UUID_1, false); assertThat(peek.isPresent()).isFalse(); } @@ -421,7 +421,7 @@ public class InternalCeQueueImplTest { .setStatus(CeQueueDto.Status.PENDING)); db.commit(); - assertThat(underTest.peek(WORKER_UUID_1).get().getUuid()).isEqualTo("uuid"); + assertThat(underTest.peek(WORKER_UUID_1, false).get().getUuid()).isEqualTo("uuid"); } @Test @@ -430,7 +430,7 @@ public class InternalCeQueueImplTest { CeQueueDto u1 = insertPending("u1");// will be picked-because older than any of the reset ones CeQueueDto u2 = insertInProgress("u2", WORKER_UUID_1);// will be reset - assertThat(underTest.peek(WORKER_UUID_1).get().getUuid()).isEqualTo("u0"); + assertThat(underTest.peek(WORKER_UUID_1, false).get().getUuid()).isEqualTo("u0"); verifyUnmodifiedTask(u1); verifyResetTask(u2); @@ -444,7 +444,7 @@ public class InternalCeQueueImplTest { CeQueueDto u3 = insertInProgress("u3", WORKER_UUID_1); CeQueueDto u4 = insertInProgress("u4", WORKER_UUID_2); - assertThat(underTest.peek(WORKER_UUID_1).get().getUuid()).isEqualTo("u0"); + assertThat(underTest.peek(WORKER_UUID_1, false).get().getUuid()).isEqualTo("u0"); verifyResetTask(u1); verifyUnmodifiedTask(u2); @@ -502,7 +502,7 @@ public class InternalCeQueueImplTest { @Test public void fail_to_cancel_if_in_progress() { CeTask task = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); - underTest.peek(WORKER_UUID_2); + underTest.peek(WORKER_UUID_2, false); CeQueueDto queueDto = db.getDbClient().ceQueueDao().selectByUuid(db.getSession(), task.getUuid()).get(); expectedException.expect(IllegalStateException.class); @@ -516,7 +516,7 @@ public class InternalCeQueueImplTest { CeTask inProgressTask = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_1")); CeTask pendingTask1 = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_2")); CeTask pendingTask2 = submit(CeTaskTypes.REPORT, newProjectDto("PROJECT_3")); - underTest.peek(WORKER_UUID_2); + underTest.peek(WORKER_UUID_2, false); int canceledCount = underTest.cancelAll(); assertThat(canceledCount).isEqualTo(2); diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeWorkerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeWorkerImplTest.java index 5f67a7fdb32..84943ca6009 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeWorkerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/taskprocessor/CeWorkerImplTest.java @@ -36,7 +36,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; -import org.mockito.Mockito; import org.mockito.stubbing.Answer; import org.sonar.api.impl.utils.TestSystem2; import org.sonar.api.utils.MessageException; @@ -60,9 +59,11 @@ import org.sonar.server.organization.BillingValidations; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.*; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; @@ -99,7 +100,7 @@ public class CeWorkerImplTest { private CeWorker underTest = new CeWorkerImpl(randomOrdinal, workerUuid, queue, taskProcessorRepository, ceWorkerController, executionListener1, executionListener2); private CeWorker underTestNoListener = new CeWorkerImpl(randomOrdinal, workerUuid, queue, taskProcessorRepository, ceWorkerController); - private InOrder inOrder = Mockito.inOrder(taskProcessor, queue, executionListener1, executionListener2); + private InOrder inOrder = inOrder(taskProcessor, queue, executionListener1, executionListener2); private final CeTask.User submitter = new CeTask.User("UUID_USER_1", "LOGIN_1"); @Before @@ -144,7 +145,7 @@ public class CeWorkerImplTest { @Test public void no_pending_tasks_in_queue() throws Exception { - when(queue.peek(anyString())).thenReturn(Optional.empty()); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.empty()); assertThat(underTest.call()).isEqualTo(NO_TASK); @@ -153,7 +154,7 @@ public class CeWorkerImplTest { @Test public void no_pending_tasks_in_queue_without_listener() throws Exception { - when(queue.peek(anyString())).thenReturn(Optional.empty()); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.empty()); assertThat(underTestNoListener.call()).isEqualTo(NO_TASK); @@ -164,7 +165,7 @@ public class CeWorkerImplTest { public void fail_when_no_CeTaskProcessor_is_found_in_repository() throws Exception { CeTask task = createCeTask(null); taskProcessorRepository.setNoProcessorForTask(CeTaskTypes.REPORT); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); assertThat(underTest.call()).isEqualTo(TASK_PROCESSED); @@ -180,7 +181,7 @@ public class CeWorkerImplTest { public void fail_when_no_CeTaskProcessor_is_found_in_repository_without_listener() throws Exception { CeTask task = createCeTask(null); taskProcessorRepository.setNoProcessorForTask(CeTaskTypes.REPORT); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); assertThat(underTestNoListener.call()).isEqualTo(TASK_PROCESSED); @@ -193,7 +194,7 @@ public class CeWorkerImplTest { public void peek_and_process_task() throws Exception { CeTask task = createCeTask(null); taskProcessorRepository.setProcessorForTask(task.getType(), taskProcessor); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); assertThat(underTest.call()).isEqualTo(TASK_PROCESSED); @@ -210,7 +211,7 @@ public class CeWorkerImplTest { public void peek_and_process_task_without_listeners() throws Exception { CeTask task = createCeTask(null); taskProcessorRepository.setProcessorForTask(task.getType(), taskProcessor); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); assertThat(underTestNoListener.call()).isEqualTo(TASK_PROCESSED); @@ -223,7 +224,7 @@ public class CeWorkerImplTest { @Test public void fail_to_process_task() throws Exception { CeTask task = createCeTask(null); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); taskProcessorRepository.setProcessorForTask(task.getType(), taskProcessor); Throwable error = makeTaskProcessorFail(task); @@ -241,7 +242,7 @@ public class CeWorkerImplTest { @Test public void fail_to_process_task_without_listeners() throws Exception { CeTask task = createCeTask(null); - when(queue.peek(anyString())).thenReturn(Optional.of(task)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(task)); taskProcessorRepository.setProcessorForTask(task.getType(), taskProcessor); Throwable error = makeTaskProcessorFail(task); @@ -255,7 +256,7 @@ public class CeWorkerImplTest { @Test public void log_task_characteristics() throws Exception { - when(queue.peek(anyString())).thenReturn(Optional.of(createCeTask(null, "pullRequest", "123", "branch", "foo"))); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(createCeTask(null, "pullRequest", "123", "branch", "foo"))); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); underTest.call(); @@ -270,7 +271,7 @@ public class CeWorkerImplTest { @Test public void do_not_log_submitter_param_if_anonymous_and_success() throws Exception { - when(queue.peek(anyString())).thenReturn(Optional.of(createCeTask(null))); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(createCeTask(null))); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); underTest.call(); @@ -286,7 +287,7 @@ public class CeWorkerImplTest { @Test public void do_not_log_submitter_param_if_anonymous_and_error() throws Exception { CeTask ceTask = createCeTask(null); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(ceTask.getType(), taskProcessor); makeTaskProcessorFail(ceTask); @@ -306,7 +307,7 @@ public class CeWorkerImplTest { @Test public void log_submitter_login_if_authenticated_and_success() throws Exception { UserDto userDto = insertRandomUser(); - when(queue.peek(anyString())).thenReturn(Optional.of(createCeTask(toTaskSubmitter(userDto)))); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(createCeTask(toTaskSubmitter(userDto)))); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); underTest.call(); @@ -322,7 +323,7 @@ public class CeWorkerImplTest { @Test public void log_submitterUuid_if_user_matching_submitterUuid_can_not_be_found() throws Exception { - when(queue.peek(anyString())).thenReturn(Optional.of(createCeTask(new CeTask.User("UUID_USER", null)))); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(createCeTask(new CeTask.User("UUID_USER", null)))); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); underTest.call(); @@ -340,7 +341,7 @@ public class CeWorkerImplTest { public void display_submitterLogin_in_logs_when_set_in_case_of_error() throws Exception { UserDto userDto = insertRandomUser(); CeTask ceTask = createCeTask(toTaskSubmitter(userDto)); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(ceTask.getType(), taskProcessor); makeTaskProcessorFail(ceTask); @@ -360,7 +361,7 @@ public class CeWorkerImplTest { public void display_start_stop_at_debug_level_for_console_if_DEBUG_is_enabled_and_task_successful() throws Exception { logTester.setLevel(LoggerLevel.DEBUG); - when(queue.peek(anyString())).thenReturn(Optional.of(createCeTask(submitter))); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(createCeTask(submitter))); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); underTest.call(); @@ -379,7 +380,7 @@ public class CeWorkerImplTest { logTester.setLevel(LoggerLevel.DEBUG); CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); makeTaskProcessorFail(ceTask); @@ -399,7 +400,7 @@ public class CeWorkerImplTest { @Test public void call_sets_and_restores_thread_name_with_information_of_worker_when_there_is_no_task_to_process() throws Exception { String threadName = randomAlphabetic(3); - when(queue.peek(anyString())).thenAnswer(invocation -> { + when(queue.peek(anyString(), anyBoolean())).thenAnswer(invocation -> { assertThat(Thread.currentThread().getName()) .isEqualTo("Worker " + randomOrdinal + " (UUID=" + workerUuid + ") on " + threadName); return Optional.empty(); @@ -413,7 +414,7 @@ public class CeWorkerImplTest { @Test public void call_sets_and_restores_thread_name_with_information_of_worker_when_a_task_is_processed() throws Exception { String threadName = randomAlphabetic(3); - when(queue.peek(anyString())).thenAnswer(invocation -> { + when(queue.peek(anyString(), anyBoolean())).thenAnswer(invocation -> { assertThat(Thread.currentThread().getName()) .isEqualTo("Worker " + randomOrdinal + " (UUID=" + workerUuid + ") on " + threadName); return Optional.of(createCeTask(submitter)); @@ -429,7 +430,7 @@ public class CeWorkerImplTest { public void call_sets_and_restores_thread_name_with_information_of_worker_when_an_error_occurs() throws Exception { String threadName = randomAlphabetic(3); CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenAnswer(invocation -> { + when(queue.peek(anyString(), anyBoolean())).thenAnswer(invocation -> { assertThat(Thread.currentThread().getName()) .isEqualTo("Worker " + randomOrdinal + " (UUID=" + workerUuid + ") on " + threadName); return Optional.of(ceTask); @@ -457,7 +458,7 @@ public class CeWorkerImplTest { @Test public void log_error_when_task_fails_with_not_MessageException() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); makeTaskProcessorFail(ceTask); @@ -475,7 +476,7 @@ public class CeWorkerImplTest { @Test public void do_no_log_error_when_task_fails_with_MessageException() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); makeTaskProcessorFail(ceTask, MessageException.of("simulate MessageException thrown by TaskProcessor#process")); @@ -491,7 +492,7 @@ public class CeWorkerImplTest { @Test public void do_no_log_error_when_task_fails_with_BillingValidationsException() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); makeTaskProcessorFail(ceTask, new BillingValidations.BillingValidationsException("simulate MessageException thrown by TaskProcessor#process")); @@ -507,7 +508,7 @@ public class CeWorkerImplTest { @Test public void log_error_when_task_was_successful_but_ending_state_can_not_be_persisted_to_db() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); doThrow(new RuntimeException("Simulate queue#remove failing")).when(queue).remove(ceTask, CeActivityDto.Status.SUCCESS, null, null); @@ -519,7 +520,7 @@ public class CeWorkerImplTest { @Test public void log_error_when_task_failed_and_ending_state_can_not_be_persisted_to_db() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); IllegalStateException ex = makeTaskProcessorFail(ceTask); RuntimeException runtimeException = new RuntimeException("Simulate queue#remove failing"); @@ -548,7 +549,7 @@ public class CeWorkerImplTest { @Test public void log_error_as_suppressed_when_task_failed_with_MessageException_and_ending_state_can_not_be_persisted_to_db() throws Exception { CeTask ceTask = createCeTask(submitter); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(CeTaskTypes.REPORT, taskProcessor); MessageException ex = makeTaskProcessorFail(ceTask, MessageException.of("simulate MessageException thrown by TaskProcessor#process")); RuntimeException runtimeException = new RuntimeException("Simulate queue#remove failing"); @@ -579,7 +580,7 @@ public class CeWorkerImplTest { CountDownLatch inCallLatch = new CountDownLatch(1); CountDownLatch assertionsDoneLatch = new CountDownLatch(1); // mock long running peek(String) call => Thread is executing call() but not running a task - when(queue.peek(anyString())).thenAnswer((Answer<Optional<CeTask>>) invocation -> { + when(queue.peek(anyString(), anyBoolean())).thenAnswer((Answer<Optional<CeTask>>) invocation -> { inCallLatch.countDown(); try { assertionsDoneLatch.await(10, TimeUnit.SECONDS); @@ -614,7 +615,7 @@ public class CeWorkerImplTest { String taskType = randomAlphabetic(12); CeTask ceTask = mock(CeTask.class); when(ceTask.getType()).thenReturn(taskType); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(taskType, new SimpleCeTaskProcessor() { @CheckForNull @Override @@ -657,7 +658,7 @@ public class CeWorkerImplTest { CountDownLatch inCallLatch = new CountDownLatch(1); CountDownLatch assertionsDoneLatch = new CountDownLatch(1); // mock long running peek(String) call => Thread is executing call() but not running a task - when(queue.peek(anyString())).thenAnswer((Answer<Optional<CeTask>>) invocation -> { + when(queue.peek(anyString(), anyBoolean())).thenAnswer((Answer<Optional<CeTask>>) invocation -> { inCallLatch.countDown(); try { assertionsDoneLatch.await(10, TimeUnit.SECONDS); @@ -688,7 +689,7 @@ public class CeWorkerImplTest { String taskType = randomAlphabetic(12); CeTask ceTask = mock(CeTask.class); when(ceTask.getType()).thenReturn(taskType); - when(queue.peek(anyString())).thenReturn(Optional.of(ceTask)); + when(queue.peek(anyString(), anyBoolean())).thenReturn(Optional.of(ceTask)); taskProcessorRepository.setProcessorForTask(taskType, new SimpleCeTaskProcessor() { @CheckForNull @@ -745,7 +746,7 @@ public class CeWorkerImplTest { } private void verifyWorkerUuid() { - verify(queue).peek(workerUuidCaptor.capture()); + verify(queue, atLeastOnce()).peek(workerUuidCaptor.capture(), anyBoolean()); assertThat(workerUuidCaptor.getValue()).isEqualTo(workerUuid); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java index ab065fe07e4..b5dd36852c0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueDao.java @@ -166,8 +166,8 @@ public class CeQueueDao implements Dao { return builder.build(); } - public Optional<CeQueueDto> peek(DbSession session, String workerUuid) { - List<String> eligibles = mapper(session).selectEligibleForPeek(ONE_RESULT_PAGINATION); + public Optional<CeQueueDto> peek(DbSession session, String workerUuid, boolean excludeIndexationJob) { + List<String> eligibles = mapper(session).selectEligibleForPeek(ONE_RESULT_PAGINATION, excludeIndexationJob); if (eligibles.isEmpty()) { return Optional.empty(); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java index c24c8fe477e..e7c3843d640 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeQueueMapper.java @@ -36,7 +36,7 @@ public interface CeQueueMapper { int countByQuery(@Param("query") CeTaskQuery query); - List<String> selectEligibleForPeek(@Param("pagination") Pagination pagination); + List<String> selectEligibleForPeek(@Param("pagination") Pagination pagination, @Param("excludeIndexationJob") boolean excludeIndexationJob); @CheckForNull CeQueueDto selectByUuid(@Param("uuid") String uuid); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java index 38401ff9708..ed90ea8b79c 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/ce/CeTaskTypes.java @@ -22,6 +22,7 @@ package org.sonar.db.ce; public final class CeTaskTypes { public static final String REPORT = "REPORT"; + public static final String BRANCH_ISSUE_SYNC = "ISSUE_SYNC"; private CeTaskTypes() { // only statics diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java index f527e249a6c..70e89109b08 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDao.java @@ -142,4 +142,17 @@ public class BranchDao implements Dao { private static BranchMapper mapper(DbSession dbSession) { return dbSession.getMapper(BranchMapper.class); } + + public List<BranchDto> selectBranchNeedingIssueSync(DbSession dbSession) { + return mapper(dbSession).selectBranchNeedingIssueSync(); + } + + public long updateAllNeedIssueSync(DbSession dbSession) { + return mapper(dbSession).updateAllNeedIssueSync(system2.now()); + } + + public long updateNeedIssueSync(DbSession dbSession, String branchUuid, boolean needIssueSync) { + long now = system2.now(); + return mapper(dbSession).updateNeedIssueSync(branchUuid, needIssueSync, now); + } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDto.java index 202d9b2b9dd..cfdb4dfbdac 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchDto.java @@ -206,8 +206,9 @@ public class BranchDto { return needIssueSync; } - public void setNeedIssueSync(boolean needIssueSync) { + public BranchDto setNeedIssueSync(boolean needIssueSync) { this.needIssueSync = needIssueSync; + return this; } @Override diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java index 78a85813999..31d867c5d17 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/component/BranchMapper.java @@ -60,4 +60,10 @@ public interface BranchMapper { int countAll(); + List<BranchDto> selectBranchNeedingIssueSync(); + + long updateAllNeedIssueSync(@Param("now") long now); + + long updateNeedIssueSync(@Param("uuid") String uuid, @Param("needIssueSync")boolean needIssueSync,@Param("now") long now); + } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml index fbf2d11dbee..0d2a49d25c5 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/ce/CeQueueMapper.xml @@ -139,7 +139,7 @@ </where> </sql> - <select id="selectEligibleForPeek" resultType="String"> + <select id="selectEligibleForPeek" parameterType="map" resultType="String"> select cq.uuid <include refid="sqlSelectEligibleForPeek"/> <include refid="orderBySelectEligibleForPeek"/> @@ -182,6 +182,9 @@ where cq.status='PENDING' and cq.started_at is null + <if test="excludeIndexationJob"> + and cq.task_type <> 'ISSUE_SYNC' + </if> and not exists ( select 1 diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml index e082c19a173..b4be8752d57 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/component/BranchMapper.xml @@ -71,7 +71,7 @@ </update> <select id="selectByKey" resultType="org.sonar.db.component.BranchDto"> - select <include refid="columns" /> + select <include refid="columns"/> from project_branches pb where pb.project_uuid = #{projectUuid, jdbcType=VARCHAR} and @@ -81,7 +81,7 @@ <select id="selectByBranchKeys" resultType="org.sonar.db.component.BranchDto"> select - <include refid="columns" /> + <include refid="columns"/> from project_branches pb where <foreach collection="branchKeyByProjectUuid" index="key" item="value" open="" separator=" or " close=""> @@ -90,14 +90,14 @@ </select> <select id="selectByProjectUuid" parameterType="string" resultType="org.sonar.db.component.BranchDto"> - select <include refid="columns" /> + select <include refid="columns"/> from project_branches pb where pb.project_uuid = #{projectUuid, jdbcType=VARCHAR} </select> <select id="selectByUuids" resultType="org.sonar.db.component.BranchDto"> - select <include refid="columns" /> + select <include refid="columns"/> from project_branches pb where pb.uuid in @@ -117,7 +117,7 @@ <select id="selectProjectUuidsWithIssuesNeedSync" resultType="String"> select distinct pb.project_uuid from project_branches pb - where pb.need_issue_sync = true + where pb.need_issue_sync = ${_true} and pb.project_uuid in <foreach collection="projectUuids" open="(" close=")" item="uuid" separator=","> #{uuid,jdbcType=VARCHAR} @@ -164,4 +164,28 @@ from project_branches pb </select> + <select id="selectBranchNeedingIssueSync" resultType="org.sonar.db.component.BranchDto"> + select + <include refid="columns"/> + from project_branches pb + where need_issue_sync = ${_true} + order by pb.updated_at, uuid + </select> + + <update id="updateAllNeedIssueSync"> + update project_branches + set + need_issue_sync = ${_true}, + updated_at = #{now, jdbcType=BIGINT} + </update> + + <update id="updateNeedIssueSync"> + update project_branches + set + need_issue_sync = #{needIssueSync, jdbcType=BOOLEAN}, + updated_at = #{now, jdbcType=BIGINT} + where + uuid = #{uuid, jdbcType=VARCHAR} + </update> + </mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java index ab4648f4f9a..f58956ee4b8 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeQueueDaoTest.java @@ -392,11 +392,11 @@ public class CeQueueDaoTest { @Test public void peek_none_if_no_pendings() { - assertThat(underTest.peek(db.getSession(), WORKER_UUID_1).isPresent()).isFalse(); + assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false).isPresent()).isFalse(); // not pending, but in progress makeInProgress(WORKER_UUID_1, 2_232_222L, insertPending(TASK_UUID_1, MAIN_COMPONENT_UUID_1)); - assertThat(underTest.peek(db.getSession(), WORKER_UUID_1).isPresent()).isFalse(); + assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false).isPresent()).isFalse(); } @Test @@ -409,7 +409,7 @@ public class CeQueueDaoTest { verifyCeQueueStatuses(TASK_UUID_1, PENDING, TASK_UUID_2, PENDING); // peek first one - Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1); + Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false); assertThat(peek).isPresent(); assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1); assertThat(peek.get().getStatus()).isEqualTo(IN_PROGRESS); @@ -417,7 +417,7 @@ public class CeQueueDaoTest { verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, PENDING); // peek second one - peek = underTest.peek(db.getSession(), WORKER_UUID_2); + peek = underTest.peek(db.getSession(), WORKER_UUID_2, false); assertThat(peek).isPresent(); assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_2); assertThat(peek.get().getStatus()).isEqualTo(IN_PROGRESS); @@ -425,7 +425,7 @@ public class CeQueueDaoTest { verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, IN_PROGRESS); // no more pendings - assertThat(underTest.peek(db.getSession(), WORKER_UUID_1).isPresent()).isFalse(); + assertThat(underTest.peek(db.getSession(), WORKER_UUID_1, false).isPresent()).isFalse(); } @Test @@ -435,7 +435,7 @@ public class CeQueueDaoTest { system2.setNow(INIT_TIME + 3_000_000); insertPending(TASK_UUID_2, MAIN_COMPONENT_UUID_1); - Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1); + Optional<CeQueueDto> peek = underTest.peek(db.getSession(), WORKER_UUID_1, false); assertThat(peek).isPresent(); assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_1); assertThat(peek.get().getMainComponentUuid()).isEqualTo(MAIN_COMPONENT_UUID_1); @@ -443,12 +443,12 @@ public class CeQueueDaoTest { verifyCeQueueStatuses(TASK_UUID_1, IN_PROGRESS, TASK_UUID_2, PENDING); // do not peek second task as long as the first one is in progress - peek = underTest.peek(db.getSession(), WORKER_UUID_1); + peek = underTest.peek(db.getSession(), WORKER_UUID_1, false); assertThat(peek.isPresent()).isFalse(); // first one is finished underTest.deleteByUuid(db.getSession(), TASK_UUID_1); - peek = underTest.peek(db.getSession(), WORKER_UUID_2); + peek = underTest.peek(db.getSession(), WORKER_UUID_2, false); assertThat(peek.get().getUuid()).isEqualTo(TASK_UUID_2); assertThat(peek.get().getWorkerUuid()).isEqualTo(WORKER_UUID_2); } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java index cf20f627244..0ffa0b010d9 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/BranchDaoTest.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -605,4 +606,50 @@ public class BranchDaoTest { assertThat(underTest.countAll(dbSession)).isEqualTo(8); } + + @Test + public void selectBranchNeedingIssueSync(){ + ComponentDto project = db.components().insertPrivateProject(); + String uuid = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(true)).uuid(); + db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(false)); + + assertThat(underTest.selectBranchNeedingIssueSync(dbSession)) + .extracting(BranchDto::getUuid) + .containsExactly(uuid); + } + + @Test + public void updateAllNeedIssueSync(){ + ComponentDto project = db.components().insertPrivateProject(); + String uuid1 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(true)).uuid(); + String uuid2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(false)).uuid(); + + underTest.updateAllNeedIssueSync(dbSession); + + Optional<BranchDto> project1 = underTest.selectByUuid(dbSession, uuid1); + assertThat(project1).isPresent(); + assertThat(project1.get().isNeedIssueSync()).isTrue(); + + Optional<BranchDto> project2 = underTest.selectByUuid(dbSession, uuid2); + assertThat(project2).isPresent(); + assertThat(project2.get().isNeedIssueSync()).isTrue(); + } + + @Test + public void updateNeedIssueSync(){ + ComponentDto project = db.components().insertPrivateProject(); + String uuid1 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(false)).uuid(); + String uuid2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH).setNeedIssueSync(true)).uuid(); + + underTest.updateNeedIssueSync(dbSession, uuid1, true); + underTest.updateNeedIssueSync(dbSession, uuid2, false); + + Optional<BranchDto> project1 = underTest.selectByUuid(dbSession, uuid1); + assertThat(project1).isPresent(); + assertThat(project1.get().isNeedIssueSync()).isTrue(); + + Optional<BranchDto> project2 = underTest.selectByUuid(dbSession, uuid2); + assertThat(project2).isPresent(); + assertThat(project2.get().isNeedIssueSync()).isFalse(); + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/StartupIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/StartupIndexer.java index e5a1cf8868a..7b5a33fcbed 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/StartupIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/StartupIndexer.java @@ -25,12 +25,25 @@ import java.util.Set; * This kind of indexers get initialized during web server startup. */ public interface StartupIndexer { + enum Type { + SYNCHRONOUS, ASYNCHRONOUS + } + + default Type getType() { + return Type.SYNCHRONOUS; + } + + default void triggerAsyncIndexOnStartup(Set<IndexType> uninitializedIndexTypes) { + throw new IllegalStateException("ASYNCHRONE StartupIndexer must implement initAsyncIndexOnStartup"); + } /** * This reindexing method will only be called on startup, and only, * if there is at least one uninitialized type. */ - void indexOnStartup(Set<IndexType> uninitializedIndexTypes); + default void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { + throw new IllegalStateException("SYNCHRONE StartupIndexer must implement indexOnStartup"); + } Set<IndexType> getIndexTypes(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexing.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexing.java new file mode 100644 index 00000000000..0261fb721cd --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexing.java @@ -0,0 +1,24 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +public interface AsyncIssueIndexing { + void triggerOnIndexCreation(); +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 41dcce54518..3e78f12b19f 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -73,11 +73,13 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { private final EsClient esClient; private final DbClient dbClient; private final IssueIteratorFactory issueIteratorFactory; + private final AsyncIssueIndexing asyncIssueIndexing; - public IssueIndexer(EsClient esClient, DbClient dbClient, IssueIteratorFactory issueIteratorFactory) { + public IssueIndexer(EsClient esClient, DbClient dbClient, IssueIteratorFactory issueIteratorFactory, AsyncIssueIndexing asyncIssueIndexing) { this.esClient = esClient; this.dbClient = dbClient; this.issueIteratorFactory = issueIteratorFactory; + this.asyncIssueIndexing = asyncIssueIndexing; } @Override @@ -91,7 +93,17 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { + public Type getType() { + return Type.ASYNCHRONOUS; + } + + @Override + public void triggerAsyncIndexOnStartup(Set<IndexType> uninitializedIndexTypes) { + asyncIssueIndexing.triggerOnIndexCreation(); + } + + @VisibleForTesting + public void indexAllIssues() { try (IssueIterator issues = issueIteratorFactory.createForAll()) { doIndex(issues, Size.LARGE, IndexingListener.FAIL_ON_ERROR); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/StartupIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/StartupIndexerTest.java new file mode 100644 index 00000000000..8701855278c --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/StartupIndexerTest.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Collections; +import org.assertj.core.api.Assertions; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.sonar.server.es.StartupIndexer.Type.SYNCHRONOUS; + +public class StartupIndexerTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private StartupIndexer underTest = () -> null; + + @Test + public void getType() { + Assertions.assertThat(underTest.getType()).isEqualTo(SYNCHRONOUS); + } + + @Test + public void triggerAsyncIndexOnStartup() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("ASYNCHRONE StartupIndexer must implement initAsyncIndexOnStartup"); + + underTest.triggerAsyncIndexOnStartup(Collections.emptySet()); + } + + @Test + public void indexOnStartup() { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("SYNCHRONE StartupIndexer must implement indexOnStartup"); + + underTest.indexOnStartup(Collections.emptySet()); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index 1c01e4c4b7d..07da774c810 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; +import org.assertj.core.api.Assertions; import org.elasticsearch.search.SearchHit; import org.junit.Before; import org.junit.Rule; @@ -46,6 +47,7 @@ import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.server.es.EsTester; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ProjectIndexer; +import org.sonar.server.es.StartupIndexer; import org.sonar.server.permission.index.AuthorizationScope; import org.sonar.server.permission.index.IndexPermissions; import org.sonar.server.security.SecurityStandards; @@ -76,7 +78,7 @@ public class IssueIndexerTest { public LogTester logTester = new LogTester(); private OrganizationDto organization; - private IssueIndexer underTest = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer underTest = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); @Before public void setUp() { @@ -106,7 +108,7 @@ public class IssueIndexerTest { IssueDto issue1 = db.issues().insert(organization); IssueDto issue2 = db.issues().insert(organization); - underTest.indexOnStartup(emptySet()); + underTest.indexAllIssues(); assertThatIndexHasOnly(issue1, issue2); } @@ -119,7 +121,7 @@ public class IssueIndexerTest { ComponentDto file = db.components().insertComponent(newFileDto(project, dir, "F1")); IssueDto issue = db.issues().insert(rule, project, file); - underTest.indexOnStartup(emptySet()); + underTest.indexAllIssues(); IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getId()).isEqualTo(issue.getKey()); @@ -153,7 +155,7 @@ public class IssueIndexerTest { ComponentDto file = db.components().insertComponent(newFileDto(project, dir, "F1")); IssueDto issue = db.issues().insert(rule, project, file); - underTest.indexOnStartup(emptySet()); + underTest.indexAllIssues(); IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getCwe()).containsExactlyInAnyOrder("123", "863"); @@ -481,7 +483,7 @@ public class IssueIndexerTest { IssueDoc issueDoc = new IssueDoc(); issueDoc.setKey("key"); issueDoc.setProjectUuid("parent-does-not-exist"); - new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())) + new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null) .index(asList(issueDoc).iterator()); assertThat(es.countDocuments(TYPE_ISSUE)).isEqualTo(1L); @@ -496,7 +498,7 @@ public class IssueIndexerTest { ComponentDto file = db.components().insertComponent(newFileDto(branch, dir, "F1")); IssueDto issue = db.issues().insert(rule, branch, file); - underTest.indexOnStartup(emptySet()); + underTest.indexAllIssues(); IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getId()).isEqualTo(issue.getKey()); @@ -507,6 +509,11 @@ public class IssueIndexerTest { assertThat(doc.isMainBranch()).isFalse(); } + @Test + public void getType(){ + Assertions.assertThat(underTest.getType()).isEqualTo(StartupIndexer.Type.ASYNCHRONOUS); + } + private void addIssueToIndex(String projectUuid, String issueKey) { es.putDocuments(TYPE_ISSUE, newDoc().setKey(issueKey).setProjectUuid(projectUuid)); diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java index 95a13193aff..8896b4f6869 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java @@ -64,13 +64,38 @@ public class IndexerStartupTask { Set<IndexType> uninitializedTypes = getUninitializedTypes(indexer); if (!uninitializedTypes.isEmpty()) { Profiler profiler = Profiler.create(LOG); - profiler.startInfo(getLogMessage(uninitializedTypes, "...")); - indexer.indexOnStartup(uninitializedTypes); - uninitializedTypes.forEach(this::setInitialized); - profiler.stopInfo(getLogMessage(uninitializedTypes, "done")); + StartupIndexer.Type type = indexer.getType(); + switch (type) { + case SYNCHRONOUS: + synchronousIndexing(indexer, uninitializedTypes, profiler); + break; + case ASYNCHRONOUS: + asynchronousIndexing(indexer, uninitializedTypes, profiler); + break; + default: + throw new IllegalArgumentException("Unsupported StartupIndexer type:" + type); + } } } + private void synchronousIndexing(StartupIndexer indexer, Set<IndexType> uninitializedTypes, Profiler profiler) { + String logMessage = getSynchronousIndexingLogMessage(uninitializedTypes); + + profiler.startInfo(logMessage + "..."); + indexer.indexOnStartup(uninitializedTypes); + uninitializedTypes.forEach(this::setInitialized); + profiler.stopInfo(logMessage + " done"); + } + + private void asynchronousIndexing(StartupIndexer indexer, Set<IndexType> uninitializedTypes, Profiler profiler) { + String logMessage = getAsynchronousIndexingLogMessage(uninitializedTypes); + + profiler.startInfo(logMessage + "..."); + indexer.triggerAsyncIndexOnStartup(uninitializedTypes); + uninitializedTypes.forEach(this::setInitialized); + profiler.stopInfo(logMessage + " done"); + } + private Set<IndexType> getUninitializedTypes(StartupIndexer indexer) { return indexer.getIndexTypes().stream() .filter(indexType -> !metadataIndex.getInitialized(indexType)) @@ -87,9 +112,15 @@ public class IndexerStartupTask { ClusterHealthAction.INSTANCE.newRequestBuilder(nativeClient).setIndices(index).setWaitForYellowStatus().get(TimeValue.timeValueMinutes(10)); } - private String getLogMessage(Set<IndexType> emptyTypes, String suffix) { + private static String getSynchronousIndexingLogMessage(Set<IndexType> emptyTypes) { + String s = emptyTypes.size() == 1 ? "" : "s"; + String typeList = emptyTypes.stream().map(Object::toString).collect(Collectors.joining(",")); + return String.format("Indexing of type%s %s", s, typeList); + } + + private static String getAsynchronousIndexingLogMessage(Set<IndexType> emptyTypes) { String s = emptyTypes.size() == 1 ? "" : "s"; String typeList = emptyTypes.stream().map(Object::toString).collect(Collectors.joining(",")); - return String.format("Indexing of type%s %s %s", s, typeList, suffix); + return String.format("Trigger asynchronous indexing of type%s %s", s, typeList); } } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexingImpl.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexingImpl.java new file mode 100644 index 00000000000..ea1415756be --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/AsyncIssueIndexingImpl.java @@ -0,0 +1,84 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.List; +import java.util.stream.Collectors; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.queue.CeQueue; +import org.sonar.ce.queue.CeTaskSubmit; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.BranchDto; + +import static java.util.Collections.emptyMap; +import static org.sonar.db.ce.CeTaskTypes.BRANCH_ISSUE_SYNC; + +@ServerSide +public class AsyncIssueIndexingImpl implements AsyncIssueIndexing { + + private static final Logger LOG = Loggers.get(AsyncIssueIndexingImpl.class); + + private final CeQueue ceQueue; + private final DbClient dbClient; + + public AsyncIssueIndexingImpl(CeQueue ceQueue, DbClient dbClient) { + this.ceQueue = ceQueue; + this.dbClient = dbClient; + } + + @Override + public void triggerOnIndexCreation() { + + try (DbSession dbSession = dbClient.openSession(false)) { + + dbClient.branchDao().updateAllNeedIssueSync(dbSession); + + // TODO check the queue for any BRANCH_ISSUE_SYNC existing task pending + + List<BranchDto> branchInNeedOfIssueSync = dbClient.branchDao().selectBranchNeedingIssueSync(dbSession); + + if (branchInNeedOfIssueSync.isEmpty()) { + LOG.info("No branch found in need of issue sync"); + return; + } + + String branchListForDisplay = branchInNeedOfIssueSync.stream().map(BranchDto::toString).collect(Collectors.joining(", ")); + LOG.info("{} branch found in need of issue sync : {}", branchInNeedOfIssueSync.size(), branchListForDisplay); + + List<CeTaskSubmit> tasks = branchInNeedOfIssueSync.stream() + .map(this::buildTaskSubmit) + .collect(Collectors.toList()); + ceQueue.massSubmit(tasks); + + dbSession.commit(); + + } + } + + private CeTaskSubmit buildTaskSubmit(BranchDto branch) { + return ceQueue.prepareSubmit() + .setType(BRANCH_ISSUE_SYNC) + .setComponent(new CeTaskSubmit.Component(branch.getUuid(), branch.getProjectUuid())) + .setCharacteristics(emptyMap()).build(); + } +} diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/package-info.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/package-info.java new file mode 100644 index 00000000000..93c5f924c3f --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/issue/index/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.issue.index; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskAsyncTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskAsyncTest.java new file mode 100644 index 00000000000..15fbdd494e1 --- /dev/null +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskAsyncTest.java @@ -0,0 +1,96 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.metadata.MetadataIndex; +import org.sonar.server.es.metadata.MetadataIndexImpl; +import org.sonar.server.es.newindex.FakeIndexDefinition; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; + +public class IndexerStartupTaskAsyncTest { + + @Rule + public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); + + private final MapSettings settings = new MapSettings(); + private final MetadataIndex metadataIndex = mock(MetadataIndexImpl.class); + private final StartupIndexer indexer = mock(StartupIndexer.class); + private final IndexerStartupTask underTest = new IndexerStartupTask(es.client(), settings.asConfig(), metadataIndex, indexer); + + @Before + public void setUp() { + when(indexer.getType()).thenReturn(StartupIndexer.Type.ASYNCHRONOUS); + doReturn(ImmutableSet.of(TYPE_FAKE)).when(indexer).getIndexTypes(); + } + + @Test + public void test(){ + doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE); + + underTest.execute(); + + verify(indexer, times(1)).triggerAsyncIndexOnStartup(anySet()); + } + + @Test + public void set_initialized_after_indexation() { + doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE); + + underTest.execute(); + + verify(metadataIndex).setInitialized(eq(TYPE_FAKE), eq(true)); + } + + @Test + public void do_not_index_if_already_initialized() { + doReturn(true).when(metadataIndex).getInitialized(TYPE_FAKE); + + underTest.execute(); + + verify(indexer).getIndexTypes(); + verifyNoMoreInteractions(indexer); + } + + @Test + public void do_not_index_if_indexes_are_disabled() { + settings.setProperty("sonar.internal.es.disableIndexes", "true"); + es.putDocuments(TYPE_FAKE, new FakeDoc()); + + underTest.execute(); + + // do not index + verifyNoMoreInteractions(indexer); + } +} diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java index e1ea9151a08..03e1f33c636 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java @@ -34,6 +34,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; public class IndexerStartupTaskTest { @@ -48,6 +49,7 @@ public class IndexerStartupTaskTest { @Before public void setUp() { + when(indexer.getType()).thenReturn(StartupIndexer.Type.SYNCHRONOUS); doReturn(ImmutableSet.of(TYPE_FAKE)).when(indexer).getIndexTypes(); } diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/issue/index/AsyncIssueIndexingImplTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/issue/index/AsyncIssueIndexingImplTest.java new file mode 100644 index 00000000000..996c544e77e --- /dev/null +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/issue/index/AsyncIssueIndexingImplTest.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.Optional; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.ce.queue.CeQueue; +import org.sonar.ce.queue.CeTaskSubmit; +import org.sonar.core.util.SequenceUuidFactory; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.DbClient; +import org.sonar.db.DbTester; +import org.sonar.db.component.BranchDto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.db.component.BranchType.BRANCH; + +public class AsyncIssueIndexingImplTest { + + @Rule + public DbTester dbTester = DbTester.create(System2.INSTANCE); + @Rule + public LogTester logTester = new LogTester(); + + private DbClient dbClient = dbTester.getDbClient(); + private CeQueue ceQueue = mock(CeQueue.class); + private UuidFactory uuidFactory = new SequenceUuidFactory(); + + private final AsyncIssueIndexingImpl underTest = new AsyncIssueIndexingImpl(ceQueue, dbClient); + + @Before + public void before() { + when(ceQueue.prepareSubmit()).thenReturn(new CeTaskSubmit.Builder(uuidFactory.create())); + } + + @Test + public void triggerOnIndexCreation() { + BranchDto dto = new BranchDto() + .setBranchType(BRANCH) + .setKey("branchName") + .setUuid("branch_uuid") + .setProjectUuid("project_uuid"); + dbClient.branchDao().insert(dbTester.getSession(), dto); + dbTester.commit(); + + underTest.triggerOnIndexCreation(); + + Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), "branch_uuid"); + assertThat(branch).isPresent(); + assertThat(branch.get().isNeedIssueSync()).isTrue(); + verify(ceQueue, times(1)).prepareSubmit(); + verify(ceQueue, times(1)).massSubmit(anyCollection()); + assertThat(logTester.logs(LoggerLevel.INFO)) + .contains("1 branch found in need of issue sync : BranchDto{uuid='branch_uuid', projectUuid='project_uuid'," + + " kee='branchName', keyType=BRANCH, branchType=BRANCH, mergeBranchUuid='null', excludeFromPurge=false, needIssueSync=true}"); + } + + @Test + public void triggerOnIndexCreation_no_branch() { + underTest.triggerOnIndexCreation(); + + assertThat(logTester.logs(LoggerLevel.INFO)).contains("No branch found in need of issue sync"); + } + +} diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java index db51ec14578..76ff3471d0c 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java @@ -66,7 +66,7 @@ public class IssueIndexDebtTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java index a4ee52291ec..b735f379a4f 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java @@ -82,7 +82,7 @@ public class IssueIndexFacetsTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java index 579aa05a5cd..3359730b847 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java @@ -78,7 +78,7 @@ public class IssueIndexFiltersTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java index 2ccf3703050..3dbf32d727c 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java @@ -57,7 +57,7 @@ public class IssueIndexProjectStatisticsTest { @Rule public UserSessionRule userSessionRule = UserSessionRule.standalone(); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), null, new IssueIteratorFactory(null)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), null, new IssueIteratorFactory(null), null); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java index a45c8dbd2f2..23164942c65 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java @@ -73,7 +73,7 @@ public class IssueIndexSecurityHotspotsTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java index b101d6e4a15..c398be4e9c7 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java @@ -71,7 +71,7 @@ public class IssueIndexSecurityReportsTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java index fed5abe890d..619b105dc54 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java @@ -66,7 +66,7 @@ public class IssueIndexSortTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index 9cff9a010d5..2f078e10610 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -84,7 +84,7 @@ public class IssueIndexTest { @Rule public DbTester db = DbTester.create(system2); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/pr/ws/ListActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/pr/ws/ListActionTest.java index 1654fa4cf65..0f81c264826 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/pr/ws/ListActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/pr/ws/ListActionTest.java @@ -89,7 +89,7 @@ public class ListActionTest { private MetricDto qualityGateStatus; private ResourceTypes resourceTypes = new ResourceTypesRule().setRootQualifiers(PROJECT); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(es, issueIndexer); @@ -259,7 +259,7 @@ public class ListActionTest { db.issues().insert(rule, pullRequest, pullRequest, i -> i.setType(CODE_SMELL).setResolution(null)); db.issues().insert(rule, pullRequest, pullRequest, i -> i.setType(CODE_SMELL).setResolution(null)); db.issues().insert(rule, pullRequest, pullRequest, i -> i.setType(CODE_SMELL).setResolution(RESOLUTION_FALSE_POSITIVE)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexerTester.allowOnlyAnyone(project); ListWsResponse response = ws.newRequest() @@ -282,7 +282,7 @@ public class ListActionTest { .setBranchType(PULL_REQUEST) .setMergeBranchUuid(nonMainBranch.uuid()) .setPullRequestData(DbProjectBranches.PullRequestData.newBuilder().setBranch("feature/bar").build())); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexerTester.allowOnlyAnyone(project); ListWsResponse response = ws.newRequest() @@ -324,7 +324,7 @@ public class ListActionTest { db.getDbClient().snapshotDao().insert(db.getSession(), newAnalysis(pullRequest2).setCreatedAt(lastAnalysisPullRequest)); db.commit(); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexerTester.allowOnlyAnyone(project); ListWsResponse response = ws.newRequest() @@ -338,6 +338,10 @@ public class ListActionTest { tuple(true, lastAnalysisPullRequest)); } + private void indexIssues() { + issueIndexer.indexAllIssues(); + } + @Test public void does_not_fail_when_only_browse_permission_on_project() { ComponentDto project = db.components().insertPrivateProject(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/ws/ListActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/ws/ListActionTest.java index fff49e9d67f..33c45ca1e61 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/ws/ListActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/branch/ws/ListActionTest.java @@ -20,7 +20,6 @@ package org.sonar.server.branch.ws; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -74,7 +73,7 @@ public class ListActionTest { public UserSessionRule userSession = UserSessionRule.standalone(); private ResourceTypes resourceTypes = new ResourceTypesRule().setRootQualifiers(PROJECT); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(es, issueIndexer); private MetricDto qualityGateStatus; @@ -112,7 +111,7 @@ public class ListActionTest { RuleDefinitionDto rule = db.rules().insert(); db.issues().insert(rule, branch, branch, i -> i.setType(BUG).setResolution(null)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addProjectPermission(USER, project); @@ -140,7 +139,7 @@ public class ListActionTest { RuleDefinitionDto rule = db.rules().insert(); db.issues().insert(rule, branch, branch, i -> i.setType(BUG).setResolution(null)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addProjectPermission(SCAN_EXECUTION, project); @@ -228,7 +227,7 @@ public class ListActionTest { db.getDbClient().snapshotDao().insert(db.getSession(), newAnalysis(branch2).setCreatedAt(lastAnalysisBranch)); db.commit(); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexerTester.allowOnlyAnyone(project); ListWsResponse response = ws.newRequest() @@ -243,6 +242,10 @@ public class ListActionTest { tuple(BranchType.BRANCH, true, lastAnalysisBranch)); } + private void indexIssues() { + issueIndexer.indexAllIssues(); + } + @Test public void application_branches() { ComponentDto application = db.components().insertPrivateApplication(db.getDefaultOrganization()); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java index a8fe30dda30..db792d87a90 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/hotspot/ws/SearchActionTest.java @@ -109,7 +109,7 @@ public class SearchActionTest { private TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private ViewIndexer viewIndexer = new ViewIndexer(dbClient, es.client()); private PermissionIndexer permissionIndexer = new PermissionIndexer(dbClient, es.client(), issueIndexer); private HotspotWsResponseFormatter responseFormatter = new HotspotWsResponseFormatter(defaultOrganizationProvider); @@ -1520,7 +1520,7 @@ public class SearchActionTest { } private void indexIssues() { - issueIndexer.indexOnStartup(issueIndexer.getIndexTypes()); + issueIndexer.indexAllIssues(); } private void indexViews() { diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java index 35a94aea042..2a13b1572e3 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AddCommentActionTest.java @@ -94,7 +94,7 @@ public class AddCommentActionTest { private IssueDbTester issueDbTester = new IssueDbTester(dbTester); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private WebIssueStorage serverIssueStorage = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(dbClient, defaultOrganizationProvider), issueIndexer, new SequenceUuidFactory()); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AssignActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AssignActionTest.java index 507945d95c7..15d0099fbc0 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AssignActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AssignActionTest.java @@ -86,7 +86,7 @@ public class AssignActionTest { private NotificationManager notificationManager = mock(NotificationManager.class); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AuthorsActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AuthorsActionTest.java index 1574954e816..7129f4976f7 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AuthorsActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/AuthorsActionTest.java @@ -70,7 +70,7 @@ public class AuthorsActionTest { public ExpectedException expectedException = ExpectedException.none(); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private PermissionIndexerTester permissionIndexer = new PermissionIndexerTester(es, issueIndexer); private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); @@ -88,7 +88,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); AuthorsResponse result = ws.newRequest().executeProtobuf(AuthorsResponse.class); @@ -105,7 +105,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); AuthorsResponse result = ws.newRequest() @@ -129,7 +129,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project1, project1, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project2, project2, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(organization1); assertThat(ws.newRequest() @@ -159,7 +159,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project1, project1, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project2, project2, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(organization); assertThat(ws.newRequest() @@ -191,7 +191,7 @@ public class AuthorsActionTest { permissionIndexer.allowOnlyAnyone(project); RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(organization); @@ -211,7 +211,7 @@ public class AuthorsActionTest { permissionIndexer.allowOnlyAnyone(project); RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(defaultOrganization); @@ -232,7 +232,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project1, project1, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project2, project2, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); AuthorsResponse result = ws.newRequest().executeProtobuf(AuthorsResponse.class); @@ -253,7 +253,7 @@ public class AuthorsActionTest { db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(han)); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); AuthorsResponse result = ws.newRequest() @@ -274,7 +274,7 @@ public class AuthorsActionTest { UserDto user = db.users().insertUser(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(leia)); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn(user).addMembership(db.getDefaultOrganization()); // User has no permission on project @@ -292,7 +292,7 @@ public class AuthorsActionTest { permissionIndexer.allowOnlyAnyone(project); db.issues().insertHotspot(project, project, issue -> issue .setAuthorLogin(luke)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); AuthorsResponse result = ws.newRequest().executeProtobuf(AuthorsResponse.class); @@ -300,6 +300,10 @@ public class AuthorsActionTest { assertThat(result.getAuthorsList()).isEmpty(); } + private void indexIssues() { + issueIndexer.indexAllIssues(); + } + @Test public void fail_when_user_is_not_logged() { userSession.anonymous(); @@ -391,7 +395,7 @@ public class AuthorsActionTest { RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin("luke.skywalker")); db.issues().insertIssue(rule, project, project, issue -> issue.setAuthorLogin("leia.organa")); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); userSession.logIn().addMembership(db.getDefaultOrganization()); String result = ws.newRequest().execute().getInput(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java index 7b344d9a872..6517d62db71 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/BulkChangeActionTest.java @@ -120,7 +120,7 @@ public class BulkChangeActionTest { private IssueWorkflow issueWorkflow = new IssueWorkflow(new FunctionExecutor(issueFieldsSetter), issueFieldsSetter); private WebIssueStorage issueStorage = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(dbClient, TestDefaultOrganizationProvider.from(db)), - new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)), new SequenceUuidFactory()); + new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null), new SequenceUuidFactory()); private NotificationManager notificationManager = mock(NotificationManager.class); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java index 965c9639f77..0cb1839f63d 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/DoTransitionActionTest.java @@ -98,7 +98,7 @@ public class DoTransitionActionTest { private IssueWorkflow workflow = new IssueWorkflow(new FunctionExecutor(updater), updater); private TransitionService transitionService = new TransitionService(userSession, workflow); private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); private IssueUpdater issueUpdater = new IssueUpdater(dbClient, diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/IssueUpdaterTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/IssueUpdaterTest.java index 3eb6c835bae..3b7a1b32384 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/IssueUpdaterTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/IssueUpdaterTest.java @@ -86,7 +86,7 @@ public class IssueUpdaterTest { private NotificationManager notificationManager = mock(NotificationManager.class); private ArgumentCaptor<IssuesChangesNotification> notificationArgumentCaptor = ArgumentCaptor.forClass(IssuesChangesNotification.class); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); private IssueUpdater underTest = new IssueUpdater(dbClient, diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsTest.java index 1d6314dcb93..9092fbac4c8 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsTest.java @@ -97,7 +97,7 @@ public class SearchActionComponentsTest { private DbClient dbClient = db.getDbClient(); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private ViewIndexer viewIndexer = new ViewIndexer(dbClient, es.client()); private IssueQueryFactory issueQueryFactory = new IssueQueryFactory(dbClient, Clock.systemUTC(), userSession); private IssueFieldsSetter issueFieldsSetter = new IssueFieldsSetter(); @@ -809,7 +809,7 @@ public class SearchActionComponentsTest { } private void indexIssues() { - issueIndexer.indexOnStartup(null); + issueIndexer.indexAllIssues(); } private void indexIssuesAndViews() { diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionFacetsTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionFacetsTest.java index 537840c3cc3..4bb32521612 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionFacetsTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionFacetsTest.java @@ -87,7 +87,7 @@ public class SearchActionFacetsTest { public ExpectedException expectedException = ExpectedException.none(); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private StartupIndexer permissionIndexer = new PermissionIndexer(db.getDbClient(), es.client(), issueIndexer); private IssueQueryFactory issueQueryFactory = new IssueQueryFactory(db.getDbClient(), Clock.systemUTC(), userSession); private SearchResponseLoader searchResponseLoader = new SearchResponseLoader(userSession, db.getDbClient(), new TransitionService(userSession, null)); @@ -595,7 +595,7 @@ public class SearchActionFacetsTest { } private void indexIssues() { - issueIndexer.indexOnStartup(issueIndexer.getIndexTypes()); + issueIndexer.indexAllIssues(); } } diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionTest.java index 8a2242c98e2..2d739cf8853 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SearchActionTest.java @@ -128,7 +128,7 @@ public class SearchActionTest { private DbClient dbClient = db.getDbClient(); private DbSession session = db.getSession(); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private IssueQueryFactory issueQueryFactory = new IssueQueryFactory(dbClient, Clock.systemUTC(), userSession); private IssueFieldsSetter issueFieldsSetter = new IssueFieldsSetter(); private IssueWorkflow issueWorkflow = new IssueWorkflow(new FunctionExecutor(issueFieldsSetter), issueFieldsSetter); @@ -1149,7 +1149,7 @@ public class SearchActionTest { } private void indexIssues() { - issueIndexer.indexOnStartup(issueIndexer.getIndexTypes()); + issueIndexer.indexAllIssues(); } private void grantPermissionToAnyone(ComponentDto project, String permission) { diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java index 02ce8ef49d0..bfea798a895 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetSeverityActionTest.java @@ -89,7 +89,7 @@ public class SetSeverityActionTest { private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); private ArgumentCaptor<SearchResponseData> preloadedSearchResponseDataCaptor = ArgumentCaptor.forClass(SearchResponseData.class); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); private WsActionTester tester = new WsActionTester(new SetSeverityAction(userSession, dbClient, new IssueFinder(dbClient, userSession), new IssueFieldsSetter(), diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTagsActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTagsActionTest.java index 4b03a24eba7..0b4248eb5e7 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTagsActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTagsActionTest.java @@ -88,7 +88,7 @@ public class SetTagsActionTest { private DbClient dbClient = db.getDbClient(); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private ArgumentCaptor<SearchResponseData> preloadedSearchResponseDataCaptor = ArgumentCaptor.forClass(SearchResponseData.class); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java index 57a4d4edb7c..0831387d7f5 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/SetTypeActionTest.java @@ -102,7 +102,7 @@ public class SetTypeActionTest { private OperationResponseWriter responseWriter = mock(OperationResponseWriter.class); private ArgumentCaptor<SearchResponseData> preloadedSearchResponseDataCaptor = ArgumentCaptor.forClass(SearchResponseData.class); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient)); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), dbClient, new IssueIteratorFactory(dbClient), null); private TestIssueChangePostProcessor issueChangePostProcessor = new TestIssueChangePostProcessor(); private IssuesChangesNotificationSerializer issuesChangesSerializer = new IssuesChangesNotificationSerializer(); private WsActionTester tester = new WsActionTester(new SetTypeAction(userSession, dbClient, new IssueFinder(dbClient, userSession), new IssueFieldsSetter(), diff --git a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java index b386900b54b..3e68a498cf7 100644 --- a/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java +++ b/server/sonar-webserver-webapi/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java @@ -69,7 +69,7 @@ public class TagsActionTest { public ExpectedException expectedException = ExpectedException.none(); private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new WebAuthorizationTypeSupport(userSession)); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); + private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()), null); private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); private PermissionIndexerTester permissionIndexer = new PermissionIndexerTester(es, issueIndexer); private ResourceTypesRule resourceTypes = new ResourceTypesRule().setRootQualifiers(PROJECT); @@ -82,7 +82,7 @@ public class TagsActionTest { ComponentDto project = db.components().insertPrivateProject(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag1", "tag2"))); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag3", "tag4", "tag5"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); TagsResponse result = ws.newRequest().executeProtobuf(TagsResponse.class); @@ -98,7 +98,7 @@ public class TagsActionTest { Consumer<IssueDto> setTags = issue -> issue.setTags(asList("tag1", "tag2")); db.issues().insertIssue(issueRule, project, project, setTags); db.issues().insertHotspot(hotspotRule, project, project, setTags); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); TestRequest testRequest = ws.newRequest(); @@ -111,7 +111,7 @@ public class TagsActionTest { ComponentDto project = db.components().insertPrivateProject(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag1", "tag2"))); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag12", "tag4", "tag5"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); assertThat(tagListOf(ws.newRequest().setParam("q", "ag1"))).containsExactly("tag1", "tag12"); @@ -124,7 +124,7 @@ public class TagsActionTest { ComponentDto project = db.components().insertPrivateProject(); db.issues().insertIssue(issueRule, project, project, issue -> issue.setTags(asList("tag1", "tag2"))); db.issues().insertHotspot(hotspotRule, project, project, issue -> issue.setTags(asList("tag1", "tag12", "tag4", "tag5"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); TestRequest testRequest = ws.newRequest(); @@ -147,7 +147,7 @@ public class TagsActionTest { OrganizationDto organization2 = db.organizations().insert(); ComponentDto project2 = db.components().insertPrivateProject(organization2); db.issues().insertIssue(rule, project2, project2, issue -> issue.setTags(singletonList("tag3"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project1, project2); assertThat(tagListOf(ws.newRequest().setParam("organization", organization1.getKey()))).containsExactly("tag1", "tag2"); @@ -167,7 +167,7 @@ public class TagsActionTest { ComponentDto project2 = db.components().insertPrivateProject(organization2); db.issues().insertIssue(issueRule, project2, project2, issue -> issue.setTags(singletonList("tag5"))); db.issues().insertHotspot(hotspotRule, project2, project2, issue -> issue.setTags(singletonList("tag6"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project1, project2); assertThat(tagListOf(ws.newRequest().setParam("organization", organization1.getKey()))).containsExactly("tag1", "tag2"); @@ -181,7 +181,7 @@ public class TagsActionTest { ComponentDto project2 = db.components().insertPrivateProject(organization); db.issues().insertIssue(rule, project1, project1, issue -> issue.setTags(singletonList("tag1"))); db.issues().insertIssue(rule, project2, project2, issue -> issue.setTags(singletonList("tag2"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project1, project2); assertThat(tagListOf(ws.newRequest() @@ -200,7 +200,7 @@ public class TagsActionTest { db.issues().insertIssue(issueRule, project1, project1, issue -> issue.setTags(singletonList("tag2"))); db.issues().insertHotspot(hotspotRule, project2, project2, issue -> issue.setTags(singletonList("tag3"))); db.issues().insertIssue(issueRule, project2, project2, issue -> issue.setTags(singletonList("tag4"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project1, project2); assertThat(tagListOf(ws.newRequest() @@ -217,7 +217,7 @@ public class TagsActionTest { permissionIndexer.allowOnlyAnyone(project); RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(singletonList("cwe"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(organization); @@ -235,7 +235,7 @@ public class TagsActionTest { RuleDefinitionDto hotspotRule = db.rules().insertHotspotRule(); db.issues().insertHotspot(hotspotRule, project, project, issue -> issue.setTags(singletonList("cwe"))); db.issues().insertIssue(issueRule, project, project, issue -> issue.setTags(singletonList("foo"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(organization); @@ -251,7 +251,7 @@ public class TagsActionTest { permissionIndexer.allowOnlyAnyone(project); RuleDefinitionDto rule = db.rules().insertIssueRule(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(singletonList("cwe"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(organization); @@ -269,7 +269,7 @@ public class TagsActionTest { RuleDefinitionDto hotspotRule = db.rules().insertHotspotRule(); db.issues().insertIssue(issueRule, project, project, issue -> issue.setTags(singletonList("cwe"))); db.issues().insertHotspot(hotspotRule, project, project, issue -> issue.setTags(singletonList("foo"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); viewIndexer.indexOnStartup(emptySet()); userSession.logIn().addMembership(organization); @@ -282,7 +282,7 @@ public class TagsActionTest { ComponentDto project = db.components().insertPrivateProject(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag1", "tag2"))); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("tag3", "tag4", "tag5"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); TagsResponse result = ws.newRequest() @@ -299,7 +299,7 @@ public class TagsActionTest { ComponentDto project2 = db.components().insertPrivateProject(); db.issues().insertIssue(rule, project1, project1, issue -> issue.setTags(asList("tag1", "tag2"))); db.issues().insertIssue(rule, project2, project2, issue -> issue.setTags(asList("tag3", "tag4", "tag5"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); // Project 2 is not visible to current user permissionIndexer.allowOnlyAnyone(project1); @@ -319,7 +319,7 @@ public class TagsActionTest { OrganizationDto organization2 = db.organizations().insert(); ComponentDto project2 = db.components().insertPrivateProject(organization2); db.issues().insertIssue(rule, project2, project2, issue -> issue.setTags(singletonList("tag3"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project1, project2); TagsResponse result = ws.newRequest().executeProtobuf(TagsResponse.class); @@ -339,7 +339,7 @@ public class TagsActionTest { OrganizationDto organization = db.organizations().insert(); OrganizationDto otherOrganization = db.organizations().insert(); ComponentDto project = db.components().insertPrivateProject(otherOrganization); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project, project); expectedException.expect(IllegalArgumentException.class); @@ -351,12 +351,16 @@ public class TagsActionTest { .execute(); } + private void indexIssues() { + issueIndexer.indexAllIssues(); + } + @Test public void fail_when_project_parameter_does_not_match_a_project() { OrganizationDto organization = db.organizations().insert(); ComponentDto project = db.components().insertPrivateProject(organization); ComponentDto file = db.components().insertComponent(newFileDto(project)); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project, project); expectedException.expect(IllegalArgumentException.class); @@ -374,7 +378,7 @@ public class TagsActionTest { ComponentDto project = db.components().insertPrivateProject(); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(asList("convention", "security"))); db.issues().insertIssue(rule, project, project, issue -> issue.setTags(singletonList("cwe"))); - issueIndexer.indexOnStartup(emptySet()); + indexIssues(); permissionIndexer.allowOnlyAnyone(project); String result = ws.newRequest().execute().getInput(); diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index fbada5ddc07..e9077216b0f 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -83,6 +83,7 @@ import org.sonar.server.issue.RemoveTagsAction; import org.sonar.server.issue.SetSeverityAction; import org.sonar.server.issue.SetTypeAction; import org.sonar.server.issue.TransitionAction; +import org.sonar.server.issue.index.AsyncIssueIndexingImpl; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.issue.index.IssueIndexer; import org.sonar.server.issue.index.IssueIteratorFactory; @@ -404,6 +405,7 @@ public class PlatformLevel4 extends PlatformLevel { // issues IssueIndexDefinition.class, + AsyncIssueIndexingImpl.class, IssueIndexer.class, IssueIteratorFactory.class, PermissionIndexer.class, |