From 49ef17cd091692c3831bdea6759a673d0c9cad64 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Tue, 8 Sep 2015 16:25:52 +0200 Subject: [PATCH] Optimize batch project container --- .../batch/analysis/DefaultAnalysisMode.java | 2 +- .../batch/bootstrap/GlobalContainer.java | 5 +- .../batch/cache/ProjectSyncContainer.java | 9 +- .../batch/issue/tracking/IssueTransition.java | 133 ++++++++++++++++++ .../issue/tracking/LocalIssueTracking.java | 63 +-------- .../org/sonar/batch/phases/PhaseExecutor.java | 8 +- .../batch/scan/ProjectScanContainer.java | 29 ++-- 7 files changed, 175 insertions(+), 74 deletions(-) create mode 100644 sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java diff --git a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java index a01928f3529..b4d2e3f322e 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java +++ b/sonar-batch/src/main/java/org/sonar/batch/analysis/DefaultAnalysisMode.java @@ -84,7 +84,7 @@ public class DefaultAnalysisMode extends AbstractAnalysisMode implements Analysi LOG.info("Medium test mode"); } if (notAssociated) { - LOG.info("Project is not associated with the server"); + LOG.info("Local analysis"); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java index 5469343c55d..c4f09fcbac8 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.bootstrap; -import org.sonar.batch.analysis.DefaultAnalysisMode; +import javax.annotation.CheckForNull; +import org.sonar.batch.analysis.DefaultAnalysisMode; import org.sonar.batch.cache.PersistentCacheProvider; import org.sonar.batch.cache.WSLoader.LoadStrategy; import org.sonar.batch.analysis.AnalysisProperties; @@ -32,6 +33,7 @@ import org.sonar.batch.rule.RulesProvider; import java.util.List; import java.util.Map; + import org.sonar.api.CoreProperties; import org.sonar.api.SonarPlugin; import org.sonar.api.utils.System2; @@ -129,6 +131,7 @@ public class GlobalContainer extends ComponentContainer { new ProjectScanContainer(this, props, components).execute(); } + @CheckForNull private static String getProjectKeyWithBranch(AnalysisProperties props) { String projectKey = props.property(CoreProperties.PROJECT_KEY_PROPERTY); if (projectKey != null && props.property(CoreProperties.PROJECT_BRANCH_PROPERTY) != null) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectSyncContainer.java b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectSyncContainer.java index 0bd67f66261..600a6791c3a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectSyncContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectSyncContainer.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.cache; -import org.sonar.batch.repository.ProjectRepositoriesFactoryProvider; +import javax.annotation.Nullable; +import org.sonar.batch.repository.ProjectRepositoriesFactoryProvider; import org.sonar.batch.analysis.DefaultAnalysisMode; import org.sonar.api.CoreProperties; import com.google.common.collect.ImmutableMap; @@ -47,7 +48,7 @@ public class ProjectSyncContainer extends ComponentContainer { private final boolean force; private final String projectKey; - public ProjectSyncContainer(ComponentContainer globalContainer, String projectKey, boolean force) { + public ProjectSyncContainer(ComponentContainer globalContainer, @Nullable String projectKey, boolean force) { super(globalContainer); this.projectKey = projectKey; this.force = force; @@ -67,7 +68,7 @@ public class ProjectSyncContainer extends ComponentContainer { } } - private static DefaultAnalysisMode createIssuesAnalisysMode() { + private static DefaultAnalysisMode createIssuesAnalysisMode() { Map props = ImmutableMap.of(CoreProperties.ANALYSIS_MODE, CoreProperties.ANALYSIS_MODE_ISSUES); GlobalProperties globalProps = new GlobalProperties(props); AnalysisProperties analysisProps = new AnalysisProperties(props); @@ -79,7 +80,7 @@ public class ProjectSyncContainer extends ComponentContainer { projectKey != null ? ProjectCacheSynchronizer.class : NonAssociatedCacheSynchronizer.class, UserRepositoryLoader.class, new ProjectRepositoriesFactoryProvider(projectKey), - createIssuesAnalisysMode()); + createIssuesAnalysisMode()); addIfMissing(DefaultProjectCacheStatus.class, ProjectCacheStatus.class); addIfMissing(DefaultProjectRepositoriesLoader.class, ProjectRepositoriesLoader.class); diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java new file mode 100644 index 00000000000..279d5d93fce --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java @@ -0,0 +1,133 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.issue.tracking; + +import org.sonar.api.batch.BatchSide; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.sonar.api.resources.Project; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.utils.KeyValueFormat; +import org.sonar.batch.index.BatchComponent; +import org.sonar.batch.index.BatchComponentCache; +import org.sonar.batch.issue.IssueCache; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.batch.report.ReportPublisher; +import org.sonar.core.issue.DefaultIssue; +import org.sonar.core.issue.IssueChangeContext; +import org.sonar.core.issue.workflow.IssueWorkflow; +import org.sonar.core.util.CloseableIterator; + +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Set; + +@BatchSide +public class IssueTransition { + private final IssueCache issueCache; + private final IssueWorkflow workflow; + private final IssueChangeContext changeContext; + private final BatchComponentCache componentCache; + private final ReportPublisher reportPublisher; + private final Date analysisDate; + private final LocalIssueTracking localIssueTracking; + + public IssueTransition(BatchComponentCache componentCache, IssueCache issueCache, IssueWorkflow workflow, ReportPublisher reportPublisher, + LocalIssueTracking localIssueTracking) { + this.componentCache = componentCache; + this.issueCache = issueCache; + this.workflow = workflow; + this.reportPublisher = reportPublisher; + this.localIssueTracking = localIssueTracking; + this.analysisDate = ((Project) componentCache.getRoot().resource()).getAnalysisDate(); + this.changeContext = IssueChangeContext.createScan(analysisDate); + } + + public IssueTransition(BatchComponentCache componentCache, IssueCache issueCache, IssueWorkflow workflow, ReportPublisher reportPublisher) { + this(componentCache, issueCache, workflow, reportPublisher, null); + } + + public void execute() { + if (localIssueTracking != null) { + localIssueTracking.init(); + } + + BatchReportReader reader = new BatchReportReader(reportPublisher.getReportDir()); + + for (BatchComponent component : componentCache.all()) { + trackIssues(reader, component); + } + } + + public void trackIssues(BatchReportReader reader, BatchComponent component) { + // raw issues = all the issues created by rule engines during this module scan and not excluded by filters + Set rawIssues = Sets.newIdentityHashSet(); + try (CloseableIterator it = reader.readComponentIssues(component.batchId())) { + while (it.hasNext()) { + rawIssues.add(it.next()); + } + } catch (Exception e) { + throw new IllegalStateException("Can't read issues for " + component.key(), e); + } + + List trackedIssues; + if (localIssueTracking != null) { + trackedIssues = localIssueTracking.trackIssues(reader, component, rawIssues); + } else { + trackedIssues = Lists.newArrayList(); + } + + // Unmatched raw issues = new issues + addUnmatchedRawIssues(component, rawIssues, trackedIssues); + + for (DefaultIssue issue : trackedIssues) { + workflow.doAutomaticTransition(issue, changeContext); + issueCache.put(issue); + } + } + + private void addUnmatchedRawIssues(BatchComponent component, Set rawIssues, List trackedIssues) { + for (BatchReport.Issue rawIssue : rawIssues) { + + DefaultIssue tracked = toTracked(component, rawIssue); + tracked.setNew(true); + tracked.setCreationDate(analysisDate); + + trackedIssues.add(tracked); + } + } + + private DefaultIssue toTracked(BatchComponent component, BatchReport.Issue rawIssue) { + DefaultIssue trackedIssue = new org.sonar.core.issue.DefaultIssueBuilder() + .componentKey(component.key()) + .projectKey("unused") + .ruleKey(RuleKey.of(rawIssue.getRuleRepository(), rawIssue.getRuleKey())) + .effortToFix(rawIssue.hasEffortToFix() ? rawIssue.getEffortToFix() : null) + .line(rawIssue.hasLine() ? rawIssue.getLine() : null) + .message(rawIssue.hasMsg() ? rawIssue.getMsg() : null) + .severity(rawIssue.getSeverity().name()) + .build(); + trackedIssue.setAttributes(rawIssue.hasAttributes() ? KeyValueFormat.parse(rawIssue.getAttributes()) : Collections.emptyMap()); + return trackedIssue; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java index e0d5cc547c1..0acf38329c5 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java +++ b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java @@ -19,12 +19,9 @@ */ package org.sonar.batch.issue.tracking; -import org.sonar.batch.analysis.DefaultAnalysisMode; - import org.sonar.batch.repository.ProjectSettingsRepo; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.Collection; @@ -46,7 +43,6 @@ import org.sonar.api.rule.RuleKey; import org.sonar.api.utils.KeyValueFormat; import org.sonar.batch.index.BatchComponent; import org.sonar.batch.index.BatchComponentCache; -import org.sonar.batch.issue.IssueCache; import org.sonar.batch.protocol.output.BatchReport; import org.sonar.batch.protocol.output.BatchReportReader; import org.sonar.batch.report.ReportPublisher; @@ -54,69 +50,41 @@ import org.sonar.core.component.ComponentKeys; import org.sonar.core.issue.DefaultIssue; import org.sonar.core.issue.IssueChangeContext; import org.sonar.core.issue.IssueUpdater; -import org.sonar.core.issue.workflow.IssueWorkflow; -import org.sonar.core.util.CloseableIterator; @BatchSide public class LocalIssueTracking { - - private final IssueCache issueCache; private final IssueTracking tracking; private final ServerLineHashesLoader lastLineHashes; - private final IssueWorkflow workflow; private final IssueUpdater updater; private final IssueChangeContext changeContext; private final ActiveRules activeRules; - private final BatchComponentCache componentCache; private final ServerIssueRepository serverIssueRepository; - private final ReportPublisher reportPublisher; private final Date analysisDate; private boolean hasServerAnalysis; - public LocalIssueTracking(BatchComponentCache resourceCache, IssueCache issueCache, IssueTracking tracking, - ServerLineHashesLoader lastLineHashes, IssueWorkflow workflow, IssueUpdater updater, - ActiveRules activeRules, ServerIssueRepository serverIssueRepository, - ProjectSettingsRepo projectRepositories, ReportPublisher reportPublisher, DefaultAnalysisMode mode) { - this.componentCache = resourceCache; - this.issueCache = issueCache; + public LocalIssueTracking(BatchComponentCache resourceCache, IssueTracking tracking, ServerLineHashesLoader lastLineHashes, IssueUpdater updater, + ActiveRules activeRules, ServerIssueRepository serverIssueRepository, ProjectSettingsRepo projectRepositories, ReportPublisher reportPublisher) { this.tracking = tracking; this.lastLineHashes = lastLineHashes; - this.workflow = workflow; this.updater = updater; this.serverIssueRepository = serverIssueRepository; - this.reportPublisher = reportPublisher; this.analysisDate = ((Project) resourceCache.getRoot().resource()).getAnalysisDate(); this.changeContext = IssueChangeContext.createScan(analysisDate); this.activeRules = activeRules; - this.hasServerAnalysis = !mode.isNotAssociated() && projectRepositories.lastAnalysisDate() != null; + this.hasServerAnalysis = projectRepositories.lastAnalysisDate() != null; } - public void execute() { + public void init() { if (hasServerAnalysis) { serverIssueRepository.load(); } - - BatchReportReader reader = new BatchReportReader(reportPublisher.getReportDir()); - - for (BatchComponent component : componentCache.all()) { - trackIssues(reader, component); - } } - public void trackIssues(BatchReportReader reader, BatchComponent component) { - // raw issues = all the issues created by rule engines during this module scan and not excluded by filters - Set rawIssues = Sets.newIdentityHashSet(); - try (CloseableIterator it = reader.readComponentIssues(component.batchId())) { - while (it.hasNext()) { - rawIssues.add(it.next()); - } - } catch (Exception e) { - throw new IllegalStateException("Can't read issues for " + component.key(), e); - } - + public List trackIssues(BatchReportReader reader, BatchComponent component, Set rawIssues) { List trackedIssues = Lists.newArrayList(); if (hasServerAnalysis) { + // all the issues that are not closed in db before starting this module scan, including manual issues Collection serverIssues = loadServerIssues(component); @@ -130,29 +98,12 @@ public class LocalIssueTracking { mergeMatched(component, trackingResult, trackedIssues, rawIssues); } - // Unmatched raw issues = new issues - addUnmatchedRawIssues(component, rawIssues, trackedIssues); - if (hasServerAnalysis && ResourceUtils.isRootProject(component.resource())) { // issues that relate to deleted components addIssuesOnDeletedComponents(trackedIssues); } - for (DefaultIssue issue : trackedIssues) { - workflow.doAutomaticTransition(issue, changeContext); - issueCache.put(issue); - } - } - - private void addUnmatchedRawIssues(BatchComponent component, Set rawIssues, List trackedIssues) { - for (BatchReport.Issue rawIssue : rawIssues) { - - DefaultIssue tracked = toTracked(component, rawIssue); - tracked.setNew(true); - tracked.setCreationDate(analysisDate); - - trackedIssues.add(tracked); - } + return trackedIssues; } private DefaultIssue toTracked(BatchComponent component, BatchReport.Issue rawIssue) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java index 54ff9c59f68..d2889f323f3 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.phases; -import org.sonar.batch.analysis.DefaultAnalysisMode; +import org.sonar.batch.issue.tracking.IssueTransition; +import org.sonar.batch.analysis.DefaultAnalysisMode; import org.sonar.batch.issue.IssueCallback; import org.sonar.api.batch.SensorContext; import org.sonar.api.resources.Project; @@ -28,7 +29,6 @@ import org.sonar.batch.events.BatchStepEvent; import org.sonar.batch.events.EventBus; import org.sonar.batch.index.DefaultIndex; import org.sonar.batch.issue.ignore.scanner.IssueExclusionsLoader; -import org.sonar.batch.issue.tracking.LocalIssueTracking; import org.sonar.batch.report.ReportPublisher; import org.sonar.batch.rule.QProfileVerifier; import org.sonar.batch.scan.filesystem.DefaultModuleFileSystem; @@ -51,14 +51,14 @@ public final class PhaseExecutor { private final IssueExclusionsLoader issueExclusionsLoader; private final IssuesReports issuesReport; private final DefaultAnalysisMode analysisMode; - private final LocalIssueTracking localIssueTracking; + private final IssueTransition localIssueTracking; private final IssueCallback issueCallback; public PhaseExecutor(InitializersExecutor initializersExecutor, PostJobsExecutor postJobsExecutor, SensorsExecutor sensorsExecutor, SensorContext sensorContext, DefaultIndex index, EventBus eventBus, ReportPublisher reportPublisher, ProjectInitializer pi, FileSystemLogger fsLogger, IssuesReports jsonReport, DefaultModuleFileSystem fs, QProfileVerifier profileVerifier, - IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, LocalIssueTracking localIssueTracking, IssueCallback issueCallback) { + IssueExclusionsLoader issueExclusionsLoader, DefaultAnalysisMode analysisMode, IssueTransition localIssueTracking, IssueCallback issueCallback) { this.postJobsExecutor = postJobsExecutor; this.initializersExecutor = initializersExecutor; this.sensorsExecutor = sensorsExecutor; diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java index d6f58f0852a..7d65aae8ab5 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java @@ -19,8 +19,10 @@ */ package org.sonar.batch.scan; -import org.sonar.batch.repository.DefaultProjectRepositoriesFactory; +import org.sonar.batch.issue.tracking.LocalIssueTracking; +import org.sonar.batch.issue.tracking.IssueTransition; +import org.sonar.batch.repository.DefaultProjectRepositoriesFactory; import org.sonar.batch.repository.QualityProfileProvider; import org.sonar.batch.repository.DefaultQualityProfileLoader; import org.sonar.batch.repository.QualityProfileLoader; @@ -65,7 +67,6 @@ import org.sonar.batch.index.Caches; import org.sonar.batch.index.DefaultIndex; import org.sonar.batch.issue.DefaultProjectIssues; import org.sonar.batch.issue.IssueCache; -import org.sonar.batch.issue.tracking.LocalIssueTracking; import org.sonar.batch.issue.tracking.ServerIssueRepository; import org.sonar.batch.mediumtest.ScanTaskObservers; import org.sonar.batch.phases.PhasesTimeProfiler; @@ -118,6 +119,9 @@ public class ProjectScanContainer extends ComponentContainer { if (settings != null && settings.getBoolean(CoreProperties.PROFILING_LOG_PROPERTY)) { add(PhasesSumUpTimeProfiler.class); } + if (isTherePreviousAnalysis()) { + addIssueTrackingComponents(); + } } private void addBatchComponents() { @@ -164,8 +168,7 @@ public class ProjectScanContainer extends ComponentContainer { IssueWorkflow.class, IssueCache.class, DefaultProjectIssues.class, - ServerIssueRepository.class, - LocalIssueTracking.class, + IssueTransition.class, // metrics DefaultMetricFinder.class, @@ -201,16 +204,26 @@ public class ProjectScanContainer extends ComponentContainer { ScanTaskObservers.class, UserRepositoryLoader.class); - addIfMissing(DefaultServerIssuesLoader.class, ServerIssuesLoader.class); - addIfMissing(DefaultServerLineHashesLoader.class, ServerLineHashesLoader.class); addIfMissing(DefaultActiveRulesLoader.class, ActiveRulesLoader.class); addIfMissing(DefaultQualityProfileLoader.class, QualityProfileLoader.class); addIfMissing(DefaultProjectRepositoriesLoader.class, ProjectRepositoriesLoader.class); addIfMissing(DefaultProjectSettingsLoader.class, ProjectSettingsLoader.class); } - private boolean isProjectAssociated() { - return !getComponentByType(DefaultAnalysisMode.class).isNotAssociated(); + private void addIssueTrackingComponents() { + add( + LocalIssueTracking.class, + ServerIssueRepository.class); + addIfMissing(DefaultServerIssuesLoader.class, ServerIssuesLoader.class); + addIfMissing(DefaultServerLineHashesLoader.class, ServerLineHashesLoader.class); + } + + private boolean isTherePreviousAnalysis() { + if (getComponentByType(DefaultAnalysisMode.class).isNotAssociated()) { + return false; + } + + return getComponentByType(DefaultProjectRepositoriesFactory.class).create().lastAnalysisDate() != null; } private void addBatchExtensions() { -- 2.39.5