From 78fbdc2a8445e9131a10d2210b88d0e4d9927a14 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Tue, 23 Sep 2014 10:51:34 +0200 Subject: [PATCH] SONAR-5644, SONAR-5473 Create new SCM extension point and fetch SCM data using WS --- .../org/sonar/plugins/core/CorePlugin.java | 106 ++++++++++- .../org/sonar/plugins/cpd/JavaCpdEngine.java | 16 +- .../main/java/org/sonar/xoo/XooPlugin.java | 6 +- ...ctivitySensor.java => XooScmProvider.java} | 58 +++--- .../sonar/batch/protocol/input/FileData.java | 59 ++++++ .../protocol/input/ProjectReferentials.java | 12 ++ .../input/ProjectReferentialsTest.java | 19 +- .../org/sonar/batch/DefaultTimeMachine.java | 37 ++++ .../batch/bootstrap/BatchComponents.java | 8 +- .../org/sonar/batch/index/DefaultIndex.java | 4 +- .../DefaultProjectReferentialsLoader.java | 53 +++++- .../batch/scan/SensorContextAdapter.java | 15 +- .../filesystem/PreviousFileHashLoader.java | 4 +- .../scan/filesystem/StatusDetection.java | 17 +- .../filesystem/StatusDetectionFactory.java | 18 +- .../batch/scan2/DefaultSensorContext.java | 20 ++- .../batch/scm/ScmActivityConfiguration.java | 100 +++++++++++ .../sonar/batch/scm/ScmActivitySensor.java | 168 ++++++++++++++++++ .../measures/MeasuresMediumTest.java | 6 +- .../DefaultProjectReferentialsLoaderTest.java | 8 +- .../StatusDetectionFactoryTest.java | 3 +- .../scan/filesystem/StatusDetectionTest.java | 11 +- .../java/org/sonar/api/CoreProperties.java | 10 ++ .../sonar/api/batch/fs/InputFileFilter.java | 1 - .../org/sonar/api/batch/scm/BlameLine.java | 102 +++++++++++ .../org/sonar/api/batch/scm/ScmProvider.java | 61 +++++++ .../org/sonar/api/batch/scm/package-info.java | 24 +++ .../measure/internal/DefaultMeasure.java | 16 ++ .../java/org/sonar/api/measures/Measure.java | 15 ++ 29 files changed, 873 insertions(+), 104 deletions(-) rename plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/{ScmActivitySensor.java => XooScmProvider.java} (56%) create mode 100644 sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/FileData.java create mode 100644 sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivityConfiguration.java create mode 100644 sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivitySensor.java create mode 100644 sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/BlameLine.java create mode 100644 sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/ScmProvider.java create mode 100644 sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/package-info.java diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java index 1476e9aefe7..dbf7b8af7d1 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java +++ b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java @@ -20,7 +20,11 @@ package org.sonar.plugins.core; import com.google.common.collect.ImmutableList; -import org.sonar.api.*; +import org.sonar.api.CoreProperties; +import org.sonar.api.Properties; +import org.sonar.api.Property; +import org.sonar.api.PropertyType; +import org.sonar.api.SonarPlugin; import org.sonar.api.checks.NoSonarFilter; import org.sonar.core.timemachine.Periods; import org.sonar.plugins.core.batch.IndexProjectPostJob; @@ -32,17 +36,81 @@ import org.sonar.plugins.core.dashboards.GlobalDefaultDashboard; import org.sonar.plugins.core.dashboards.ProjectDefaultDashboard; import org.sonar.plugins.core.dashboards.ProjectIssuesDashboard; import org.sonar.plugins.core.dashboards.ProjectTimeMachineDashboard; -import org.sonar.plugins.core.issue.*; -import org.sonar.plugins.core.issue.notification.*; +import org.sonar.plugins.core.issue.CountFalsePositivesDecorator; +import org.sonar.plugins.core.issue.CountUnresolvedIssuesDecorator; +import org.sonar.plugins.core.issue.InitialOpenIssuesSensor; +import org.sonar.plugins.core.issue.InitialOpenIssuesStack; +import org.sonar.plugins.core.issue.IssueHandlers; +import org.sonar.plugins.core.issue.IssueTracking; +import org.sonar.plugins.core.issue.IssueTrackingDecorator; +import org.sonar.plugins.core.issue.notification.ChangesOnMyIssueNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.IssueChangesEmailTemplate; +import org.sonar.plugins.core.issue.notification.NewFalsePositiveNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.NewIssuesEmailTemplate; +import org.sonar.plugins.core.issue.notification.NewIssuesNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.SendIssueNotificationsPostJob; import org.sonar.plugins.core.measurefilters.MyFavouritesFilter; import org.sonar.plugins.core.measurefilters.ProjectFilter; import org.sonar.plugins.core.notifications.alerts.NewAlerts; import org.sonar.plugins.core.security.ApplyProjectRolesDecorator; -import org.sonar.plugins.core.sensors.*; -import org.sonar.plugins.core.timemachine.*; -import org.sonar.plugins.core.widgets.*; -import org.sonar.plugins.core.widgets.issues.*; -import org.sonar.plugins.core.widgets.measures.*; +import org.sonar.plugins.core.sensors.BranchCoverageDecorator; +import org.sonar.plugins.core.sensors.CommentDensityDecorator; +import org.sonar.plugins.core.sensors.CoverageDecorator; +import org.sonar.plugins.core.sensors.CoverageMeasurementFilter; +import org.sonar.plugins.core.sensors.DirectoriesDecorator; +import org.sonar.plugins.core.sensors.FileHashSensor; +import org.sonar.plugins.core.sensors.FilesDecorator; +import org.sonar.plugins.core.sensors.ItBranchCoverageDecorator; +import org.sonar.plugins.core.sensors.ItCoverageDecorator; +import org.sonar.plugins.core.sensors.ItLineCoverageDecorator; +import org.sonar.plugins.core.sensors.LineCoverageDecorator; +import org.sonar.plugins.core.sensors.ManualMeasureDecorator; +import org.sonar.plugins.core.sensors.OverallBranchCoverageDecorator; +import org.sonar.plugins.core.sensors.OverallCoverageDecorator; +import org.sonar.plugins.core.sensors.OverallLineCoverageDecorator; +import org.sonar.plugins.core.sensors.ProjectLinksSensor; +import org.sonar.plugins.core.sensors.UnitTestDecorator; +import org.sonar.plugins.core.sensors.VersionEventsSensor; +import org.sonar.plugins.core.timemachine.NewCoverageAggregator; +import org.sonar.plugins.core.timemachine.NewCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.NewItCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.NewOverallCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.TendencyDecorator; +import org.sonar.plugins.core.timemachine.TimeMachineConfigurationPersister; +import org.sonar.plugins.core.timemachine.VariationDecorator; +import org.sonar.plugins.core.widgets.AlertsWidget; +import org.sonar.plugins.core.widgets.BubbleChartWidget; +import org.sonar.plugins.core.widgets.ComplexityWidget; +import org.sonar.plugins.core.widgets.CoverageWidget; +import org.sonar.plugins.core.widgets.CustomMeasuresWidget; +import org.sonar.plugins.core.widgets.DebtOverviewWidget; +import org.sonar.plugins.core.widgets.DescriptionWidget; +import org.sonar.plugins.core.widgets.DocumentationCommentsWidget; +import org.sonar.plugins.core.widgets.DuplicationsWidget; +import org.sonar.plugins.core.widgets.EventsWidget; +import org.sonar.plugins.core.widgets.HotspotMetricWidget; +import org.sonar.plugins.core.widgets.HotspotMostViolatedRulesWidget; +import org.sonar.plugins.core.widgets.ItCoverageWidget; +import org.sonar.plugins.core.widgets.ProjectFileCloudWidget; +import org.sonar.plugins.core.widgets.SizeWidget; +import org.sonar.plugins.core.widgets.TechnicalDebtPyramidWidget; +import org.sonar.plugins.core.widgets.TimeMachineWidget; +import org.sonar.plugins.core.widgets.TimelineWidget; +import org.sonar.plugins.core.widgets.TreemapWidget; +import org.sonar.plugins.core.widgets.WelcomeWidget; +import org.sonar.plugins.core.widgets.issues.ActionPlansWidget; +import org.sonar.plugins.core.widgets.issues.FalsePositiveIssuesWidget; +import org.sonar.plugins.core.widgets.issues.IssueFilterWidget; +import org.sonar.plugins.core.widgets.issues.IssuesWidget; +import org.sonar.plugins.core.widgets.issues.MyUnresolvedIssuesWidget; +import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesPerAssigneeWidget; +import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesStatusesWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterAsBubbleChartWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterAsCloudWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterAsHistogramWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterAsPieChartWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterAsTreemapWidget; +import org.sonar.plugins.core.widgets.measures.MeasureFilterListWidget; import java.util.List; @@ -186,7 +254,27 @@ import java.util.List; global = false, defaultValue = "admin", type = PropertyType.STRING, - multiValues = true) + multiValues = true), + @Property( + key = CoreProperties.SCM_ENABLED_KEY, + defaultValue = "true", + name = "Activation of the SCM Activity step", + description = "This property can be set to false in order to deactivate the SCM Activity step.", + module = false, + project = true, + global = true, + type = PropertyType.BOOLEAN + ), + @Property( + key = CoreProperties.SCM_PROVIDER_KEY, + defaultValue = "", + name = "Key of the SCM provider for this project", + description = "Force the provider to be used to get SCM information for this project. By default auto-detection is done. Exemple: svn, git.", + module = false, + project = true, + global = false, + type = PropertyType.BOOLEAN + ) }) public final class CorePlugin extends SonarPlugin { diff --git a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/JavaCpdEngine.java b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/JavaCpdEngine.java index 5dfff7e3dcd..f8baa2ce642 100644 --- a/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/JavaCpdEngine.java +++ b/plugins/sonar-cpd-plugin/src/main/java/org/sonar/plugins/cpd/JavaCpdEngine.java @@ -33,6 +33,7 @@ import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.duplication.DuplicationBuilder; import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder; +import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.config.Settings; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.FileLinesContext; @@ -206,20 +207,23 @@ public class JavaCpdEngine extends CpdEngine { } linesContext.save(); // Save - context.newMeasure() + ((DefaultMeasure) context.newMeasure() .forMetric(CoreMetrics.DUPLICATED_FILES) .onFile(inputFile) - .withValue(1) + .withValue(1)) + .setFromCore() .save(); - context.newMeasure() + ((DefaultMeasure) context.newMeasure() .forMetric(CoreMetrics.DUPLICATED_LINES) .onFile(inputFile) - .withValue(duplicatedLines.size()) + .withValue(duplicatedLines.size())) + .setFromCore() .save(); - context.newMeasure() + ((DefaultMeasure) context.newMeasure() .forMetric(CoreMetrics.DUPLICATED_BLOCKS) .onFile(inputFile) - .withValue(duplicatedBlocks) + .withValue(duplicatedBlocks)) + .setFromCore() .save(); DuplicationBuilder builder = context.duplicationBuilder(inputFile); diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java index c7e43d9e0ab..e1988ac37ab 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/XooPlugin.java @@ -22,10 +22,10 @@ package org.sonar.xoo; import org.sonar.api.SonarPlugin; import org.sonar.xoo.lang.CoveragePerTestSensor; import org.sonar.xoo.lang.MeasureSensor; -import org.sonar.xoo.lang.ScmActivitySensor; import org.sonar.xoo.lang.SymbolReferencesSensor; import org.sonar.xoo.lang.SyntaxHighlightingSensor; import org.sonar.xoo.lang.TestCaseSensor; +import org.sonar.xoo.lang.XooScmProvider; import org.sonar.xoo.lang.XooTokenizerSensor; import org.sonar.xoo.rule.CreateIssueByInternalKeySensor; import org.sonar.xoo.rule.OneIssueOnDirPerFileSensor; @@ -51,9 +51,11 @@ public class XooPlugin extends SonarPlugin { XooRulesDefinition.class, XooQualityProfile.class, + // SCM + XooScmProvider.class, + // sensors MeasureSensor.class, - ScmActivitySensor.class, SyntaxHighlightingSensor.class, SymbolReferencesSensor.class, XooTokenizerSensor.class, diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/ScmActivitySensor.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooScmProvider.java similarity index 56% rename from plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/ScmActivitySensor.java rename to plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooScmProvider.java index 12663b0b697..71582efc3ef 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/ScmActivitySensor.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/lang/XooScmProvider.java @@ -25,66 +25,58 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.sensor.Sensor; -import org.sonar.api.batch.sensor.SensorContext; -import org.sonar.api.batch.sensor.SensorDescriptor; -import org.sonar.api.measures.CoreMetrics; -import org.sonar.api.measures.FileLinesContext; -import org.sonar.api.measures.FileLinesContextFactory; +import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.config.Settings; import org.sonar.api.utils.DateUtils; -import org.sonar.xoo.Xoo; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Date; import java.util.List; -public class ScmActivitySensor implements Sensor { +public class XooScmProvider implements ScmProvider { - private static final Logger LOG = LoggerFactory.getLogger(ScmActivitySensor.class); + private static final Logger LOG = LoggerFactory.getLogger(XooScmProvider.class); private static final String SCM_EXTENSION = ".scm"; - private final FileSystem fs; - private final FileLinesContextFactory fileLinesContextFactory; + private final Settings settings; - public ScmActivitySensor(FileLinesContextFactory fileLinesContextFactory, FileSystem fileSystem) { - this.fs = fileSystem; - this.fileLinesContextFactory = fileLinesContextFactory; + public XooScmProvider(Settings settings) { + this.settings = settings; } @Override - public void describe(SensorDescriptor descriptor) { - descriptor - .name(this.getClass().getSimpleName()) - .provides(CoreMetrics.SCM_AUTHORS_BY_LINE, - CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE, - CoreMetrics.SCM_REVISIONS_BY_LINE) - .workOnLanguages(Xoo.KEY); + public String key() { + return "xoo"; } @Override - public void execute(SensorContext context) { - for (InputFile inputFile : fs.inputFiles(fs.predicates().hasLanguage(Xoo.KEY))) { - processFile(inputFile); - } + public boolean supports(File baseDir) { + return false; + } + @Override + public void blame(Iterable files, BlameResultHandler handler) { + for (InputFile inputFile : files) { + processFile(inputFile, handler); + } } @VisibleForTesting - protected void processFile(InputFile inputFile) { + protected void processFile(InputFile inputFile, BlameResultHandler handler) { File ioFile = inputFile.file(); File scmDataFile = new java.io.File(ioFile.getParentFile(), ioFile.getName() + SCM_EXTENSION); if (!scmDataFile.exists()) { - LOG.debug("Skipping SCM data injection for " + inputFile.relativePath()); - return; + throw new IllegalStateException("Missing file " + scmDataFile); } - FileLinesContext fileLinesContext = fileLinesContextFactory.createFor(inputFile); try { List lines = FileUtils.readLines(scmDataFile, Charsets.UTF_8.name()); + List blame = new ArrayList(lines.size()); int lineNumber = 0; for (String line : lines) { lineNumber++; @@ -99,14 +91,12 @@ public class ScmActivitySensor implements Sensor { // Will throw an exception, when date is not in format "yyyy-MM-dd" Date date = DateUtils.parseDate(fields[2]); - fileLinesContext.setStringValue(CoreMetrics.SCM_REVISIONS_BY_LINE_KEY, lineNumber, revision); - fileLinesContext.setStringValue(CoreMetrics.SCM_AUTHORS_BY_LINE_KEY, lineNumber, author); - fileLinesContext.setStringValue(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE_KEY, lineNumber, DateUtils.formatDateTime(date)); + blame.add(new BlameLine(date, revision, author)); } } + handler.handle(inputFile, blame); } catch (IOException e) { throw new IllegalStateException(e); } - fileLinesContext.save(); } } diff --git a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/FileData.java b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/FileData.java new file mode 100644 index 00000000000..fc7e7beca02 --- /dev/null +++ b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/FileData.java @@ -0,0 +1,59 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.protocol.input; + +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + +public class FileData { + + private final String hash; + private final String scmLastCommitDatetimesByLine; + private final String scmRevisionsByLine; + private final String scmAuthorsByLine; + + public FileData(@Nullable String hash, @Nullable String scmLastCommitDatetimesByLine, @Nullable String scmRevisionsByLine, @Nullable String scmAuthorsByLine) { + this.hash = hash; + this.scmLastCommitDatetimesByLine = scmLastCommitDatetimesByLine; + this.scmRevisionsByLine = scmRevisionsByLine; + this.scmAuthorsByLine = scmAuthorsByLine; + } + + @CheckForNull + public String hash() { + return hash; + } + + @CheckForNull + public String scmLastCommitDatetimesByLine() { + return scmLastCommitDatetimesByLine; + } + + @CheckForNull + public String scmRevisionsByLine() { + return scmRevisionsByLine; + } + + @CheckForNull + public String scmAuthorsByLine() { + return scmAuthorsByLine; + } + +} diff --git a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/ProjectReferentials.java b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/ProjectReferentials.java index 6a8d5c8691e..5e2bfafd59d 100644 --- a/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/ProjectReferentials.java +++ b/sonar-batch-protocol/src/main/java/org/sonar/batch/protocol/input/ProjectReferentials.java @@ -21,6 +21,8 @@ package org.sonar.batch.protocol.input; import com.google.gson.Gson; +import javax.annotation.CheckForNull; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -37,6 +39,7 @@ public class ProjectReferentials { private Map qprofilesByLanguage = new HashMap(); private Collection activeRules = new ArrayList(); private Map> settingsByModule = new HashMap>(); + private Map fileDataPerPath = new HashMap(); public Map settings(String projectKey) { return settingsByModule.containsKey(projectKey) ? settingsByModule.get(projectKey) : Collections.emptyMap(); @@ -70,6 +73,15 @@ public class ProjectReferentials { return this; } + public Map fileDataPerPath() { + return fileDataPerPath; + } + + @CheckForNull + public FileData fileDataPerPath(String path) { + return fileDataPerPath.get(path); + } + public long timestamp() { return timestamp; } diff --git a/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/input/ProjectReferentialsTest.java b/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/input/ProjectReferentialsTest.java index 9196a831ec9..ed98c51e46b 100644 --- a/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/input/ProjectReferentialsTest.java +++ b/sonar-batch-protocol/src/test/java/org/sonar/batch/protocol/input/ProjectReferentialsTest.java @@ -49,6 +49,7 @@ public class ProjectReferentialsTest { activeRule.addParam("param1", "value1"); ref.addActiveRule(activeRule); ref.setTimestamp(10); + ref.fileDataPerPath().put("src/main/java/Foo.java", new FileData("xyz", "1=12345,2=3456", "1=345,2=345", "1=henryju,2=gaudin")); System.out.println(ref.toJson()); JSONAssert @@ -56,16 +57,19 @@ public class ProjectReferentialsTest { "{timestamp:10," + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"Mar 14, 1984 12:00:00 AM\"}}," + "activeRules:[{repositoryKey:repo,ruleKey:rule,name:Rule,severity:MAJOR,internalKey:rule,language:java,params:{param1:value1}}]," - + "settingsByModule:{foo:{prop1:value1,prop2:value2,prop:value}}}", + + "settingsByModule:{foo:{prop1:value1,prop2:value2,prop:value}}," + + "fileDataPerPath:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}}", ref.toJson(), true); } @Test public void testFromJson() throws JSONException, ParseException { - ProjectReferentials ref = ProjectReferentials.fromJson("{timestamp:1," - + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"Mar 14, 1984 12:00:00 AM\"}}," - + "activeRules:[{repositoryKey:repo,ruleKey:rule,name:Rule,severity:MAJOR,internalKey:rule1,language:java,params:{param1:value1}}]," - + "settingsByModule:{foo:{prop:value}}}"); + ProjectReferentials ref = ProjectReferentials + .fromJson("{timestamp:1," + + "qprofilesByLanguage:{java:{key:\"squid-java\",name:Java,language:java,rulesUpdatedAt:\"Mar 14, 1984 12:00:00 AM\"}}," + + "activeRules:[{repositoryKey:repo,ruleKey:rule,name:Rule,severity:MAJOR,internalKey:rule1,language:java,params:{param1:value1}}]," + + "settingsByModule:{foo:{prop:value}}," + + "fileDataPerPath:{\"src/main/java/Foo.java\":{hash:xyz,scmLastCommitDatetimesByLine:\"1\u003d12345,2\u003d3456\",scmRevisionsByLine:\"1\u003d345,2\u003d345\",scmAuthorsByLine:\"1\u003dhenryju,2\u003dgaudin\"}}}"); assertThat(ref.timestamp()).isEqualTo(1); @@ -83,5 +87,10 @@ public class ProjectReferentialsTest { assertThat(qProfile.name()).isEqualTo("Java"); assertThat(qProfile.rulesUpdatedAt()).isEqualTo(new SimpleDateFormat("dd/MM/yyyy").parse("14/03/1984")); assertThat(ref.settings("foo")).includes(MapAssert.entry("prop", "value")); + + assertThat(ref.fileDataPerPath("src/main/java/Foo.java").hash()).isEqualTo("xyz"); + assertThat(ref.fileDataPerPath("src/main/java/Foo.java").scmAuthorsByLine()).isEqualTo("1=henryju,2=gaudin"); + assertThat(ref.fileDataPerPath("src/main/java/Foo.java").scmLastCommitDatetimesByLine()).isEqualTo("1=12345,2=3456"); + assertThat(ref.fileDataPerPath("src/main/java/Foo.java").scmRevisionsByLine()).isEqualTo("1=345,2=345"); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java b/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java index 565417efad0..40df91b4ab4 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java @@ -25,6 +25,7 @@ import org.sonar.api.batch.TimeMachine; import org.sonar.api.batch.TimeMachineQuery; import org.sonar.api.database.DatabaseSession; import org.sonar.api.database.model.MeasureModel; +import org.sonar.api.database.model.ResourceModel; import org.sonar.api.database.model.Snapshot; import org.sonar.api.measures.Measure; import org.sonar.api.measures.Metric; @@ -38,6 +39,7 @@ import org.sonar.batch.index.DefaultIndex; import javax.annotation.Nullable; import javax.persistence.Query; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -148,6 +150,41 @@ public class DefaultTimeMachine implements TimeMachine { return jpaQuery.getResultList(); } + // Temporary implementation for SONAR-5473 + public List query(String resourceKey, Integer... metricIds) { + StringBuilder sb = new StringBuilder(); + Map params = Maps.newHashMap(); + + sb.append("SELECT m"); + sb.append(" FROM ") + .append(MeasureModel.class.getSimpleName()) + .append(" m, ") + .append(ResourceModel.class.getSimpleName()) + .append(" r, ") + .append(Snapshot.class.getSimpleName()) + .append(" s WHERE m.snapshotId=s.id AND s.resourceId=r.id AND r.kee=:kee AND s.status=:status AND s.qualifier<>:lib"); + params.put("kee", resourceKey); + params.put("status", Snapshot.STATUS_PROCESSED); + params.put("lib", Qualifiers.LIBRARY); + + sb.append(" AND m.characteristicId IS NULL"); + sb.append(" AND m.personId IS NULL"); + sb.append(" AND m.ruleId IS NULL AND m.rulePriority IS NULL"); + if (metricIds.length > 0) { + sb.append(" AND m.metricId IN (:metricIds) "); + params.put("metricIds", Arrays.asList(metricIds)); + } + sb.append(" AND s.last=true "); + sb.append(" ORDER BY s.createdAt "); + + Query jpaQuery = session.createQuery(sb.toString()); + + for (Map.Entry entry : params.entrySet()) { + jpaQuery.setParameter(entry.getKey(), entry.getValue()); + } + return jpaQuery.getResultList(); + } + public Map getMetricsById(TimeMachineQuery query) { Collection metrics = metricFinder.findAll(query.getMetricKeys()); Map result = Maps.newHashMap(); diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java index 8fcf38baafc..c600e01f7e9 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java @@ -30,6 +30,8 @@ import org.sonar.batch.maven.DefaultMavenPluginExecutor; import org.sonar.batch.maven.MavenProjectBootstrapper; import org.sonar.batch.maven.MavenProjectBuilder; import org.sonar.batch.maven.MavenProjectConverter; +import org.sonar.batch.scm.ScmActivityConfiguration; +import org.sonar.batch.scm.ScmActivitySensor; import org.sonar.core.config.CorePropertyDefinitions; import java.util.Collection; @@ -51,7 +53,11 @@ public class BatchComponents { SubProjectDsmDecorator.class, DirectoryDsmDecorator.class, DirectoryTangleIndexDecorator.class, - FileTangleIndexDecorator.class + FileTangleIndexDecorator.class, + + // SCM + ScmActivityConfiguration.class, + ScmActivitySensor.class ); components.addAll(CorePropertyDefinitions.all()); return components; diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java index 70393580638..4b72027132c 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java @@ -215,8 +215,8 @@ public class DefaultIndex extends SonarIndex { if (metric == null) { throw new SonarException("Unknown metric: " + measure.getMetricKey()); } - if (DefaultSensorContext.INTERNAL_METRICS.contains(metric)) { - LOG.warn("Metric " + metric.key() + " is an internal metric computed by SonarQube. Please update your plugin."); + if (!measure.isFromCore() && DefaultSensorContext.INTERNAL_METRICS.contains(metric)) { + LOG.warn("Metric " + metric.key() + " is an internal metric computed by SonarQube. Please update or uninstall offending plugin."); return measure; } measure.setMetric(metric); diff --git a/sonar-batch/src/main/java/org/sonar/batch/referential/DefaultProjectReferentialsLoader.java b/sonar-batch/src/main/java/org/sonar/batch/referential/DefaultProjectReferentialsLoader.java index bcde2f88a05..98c46ef1172 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/referential/DefaultProjectReferentialsLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/referential/DefaultProjectReferentialsLoader.java @@ -19,31 +19,54 @@ */ package org.sonar.batch.referential; +import com.google.common.collect.ImmutableList; import org.sonar.api.batch.bootstrap.ProjectReactor; +import org.sonar.api.database.model.MeasureModel; +import org.sonar.api.measures.CoreMetrics; +import org.sonar.api.measures.Metric; +import org.sonar.api.measures.MetricFinder; +import org.sonar.batch.DefaultTimeMachine; import org.sonar.batch.bootstrap.AnalysisMode; import org.sonar.batch.bootstrap.ServerClient; import org.sonar.batch.bootstrap.TaskProperties; +import org.sonar.batch.protocol.input.FileData; import org.sonar.batch.protocol.input.ProjectReferentials; import org.sonar.batch.rule.ModuleQProfiles; +import org.sonar.batch.scan.filesystem.PreviousFileHashLoader; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.List; +import java.util.Map; public class DefaultProjectReferentialsLoader implements ProjectReferentialsLoader { private static final String BATCH_PROJECT_URL = "/batch/project"; + private static final List METRICS = ImmutableList.of( + CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE, + CoreMetrics.SCM_REVISIONS_BY_LINE, + CoreMetrics.SCM_AUTHORS_BY_LINE); + private final ServerClient serverClient; private final AnalysisMode analysisMode; + private final PreviousFileHashLoader fileHashLoader; + private final MetricFinder metricFinder; + private final DefaultTimeMachine defaultTimeMachine; - public DefaultProjectReferentialsLoader(ServerClient serverClient, AnalysisMode analysisMode) { + public DefaultProjectReferentialsLoader(ServerClient serverClient, AnalysisMode analysisMode, PreviousFileHashLoader fileHashLoader, MetricFinder finder, + DefaultTimeMachine defaultTimeMachine) { this.serverClient = serverClient; this.analysisMode = analysisMode; + this.fileHashLoader = fileHashLoader; + this.metricFinder = finder; + this.defaultTimeMachine = defaultTimeMachine; } @Override public ProjectReferentials load(ProjectReactor reactor, TaskProperties taskProperties) { - String url = BATCH_PROJECT_URL + "?key=" + reactor.getRoot().getKeyWithBranch(); + String projectKey = reactor.getRoot().getKeyWithBranch(); + String url = BATCH_PROJECT_URL + "?key=" + projectKey; if (taskProperties.properties().containsKey(ModuleQProfiles.SONAR_PROFILE_PROP)) { try { url += "&profile=" + URLEncoder.encode(taskProperties.properties().get(ModuleQProfiles.SONAR_PROFILE_PROP), "UTF-8"); @@ -52,6 +75,30 @@ public class DefaultProjectReferentialsLoader implements ProjectReferentialsLoad } } url += "&preview=" + analysisMode.isPreview(); - return ProjectReferentials.fromJson(serverClient.request(url)); + ProjectReferentials ref = ProjectReferentials.fromJson(serverClient.request(url)); + + Integer lastCommitsId = metricFinder.findByKey(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE.key()).getId(); + Integer revisionsId = metricFinder.findByKey(CoreMetrics.SCM_REVISIONS_BY_LINE.key()).getId(); + Integer authorsId = metricFinder.findByKey(CoreMetrics.SCM_AUTHORS_BY_LINE.key()).getId(); + for (Map.Entry hashByPaths : fileHashLoader.hashByRelativePath().entrySet()) { + String path = hashByPaths.getKey(); + String hash = hashByPaths.getValue(); + String lastCommits = null; + String revisions = null; + String authors = null; + List measures = defaultTimeMachine.query(projectKey + ":" + path, lastCommitsId, revisionsId, authorsId); + for (MeasureModel m : measures) { + if (m.getMetricId() == lastCommitsId) { + lastCommits = m.getData(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE); + } else if (m.getMetricId() == revisionsId) { + revisions = m.getData(CoreMetrics.SCM_REVISIONS_BY_LINE); + } + if (m.getMetricId() == authorsId) { + authors = m.getData(CoreMetrics.SCM_AUTHORS_BY_LINE); + } + } + ref.fileDataPerPath().put(path, new FileData(hash, lastCommits, revisions, authors)); + } + return ref; } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/SensorContextAdapter.java b/sonar-batch/src/main/java/org/sonar/batch/scan/SensorContextAdapter.java index 6c9b212e3f6..90c00db0ef5 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/SensorContextAdapter.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/SensorContextAdapter.java @@ -32,6 +32,7 @@ import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.issue.Issue; import org.sonar.api.batch.sensor.issue.Issue.Severity; import org.sonar.api.batch.sensor.measure.Measure; +import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.batch.sensor.test.TestCase; import org.sonar.api.batch.sensor.test.internal.DefaultTestCase; import org.sonar.api.component.ResourcePerspectives; @@ -95,18 +96,20 @@ public class SensorContextAdapter extends BaseSensorContext { } @Override - public void store(Measure measure) { + public void store(Measure newMeasure) { + DefaultMeasure measure = (DefaultMeasure) newMeasure; org.sonar.api.measures.Metric m = findMetricOrFail(measure.metric().key()); org.sonar.api.measures.Measure measureToSave = new org.sonar.api.measures.Measure(m); - setValueAccordingToMetricType(measure, m, measureToSave); - if (measure.inputFile() != null) { - Formula formula = measure.metric() instanceof org.sonar.api.measures.Metric ? - ((org.sonar.api.measures.Metric) measure.metric()).getFormula() : null; + setValueAccordingToMetricType(newMeasure, m, measureToSave); + measureToSave.setFromCore(measure.isFromCore()); + if (newMeasure.inputFile() != null) { + Formula formula = newMeasure.metric() instanceof org.sonar.api.measures.Metric ? + ((org.sonar.api.measures.Metric) newMeasure.metric()).getFormula() : null; if (formula instanceof SumChildDistributionFormula && !Scopes.isHigherThanOrEquals(Scopes.FILE, ((SumChildDistributionFormula) formula).getMinimumScopeToPersist())) { measureToSave.setPersistenceMode(PersistenceMode.MEMORY); } - sensorContext.saveMeasure(measure.inputFile(), measureToSave); + sensorContext.saveMeasure(newMeasure.inputFile(), measureToSave); } else { sensorContext.saveMeasure(measureToSave); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PreviousFileHashLoader.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PreviousFileHashLoader.java index 0fa91ea3c93..b86b501225e 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PreviousFileHashLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/PreviousFileHashLoader.java @@ -48,14 +48,14 @@ public class PreviousFileHashLoader implements BatchComponent { /** * Extract hash of the files parsed during the previous analysis */ - Map hashByRelativePath() { + public Map hashByRelativePath() { Map map = Maps.newHashMap(); PastSnapshot pastSnapshot = pastSnapshotFinder.findPreviousAnalysis(snapshot); if (pastSnapshot.isRelatedToSnapshot()) { Collection selectSnapshotData = dao.selectSnapshotData( pastSnapshot.getProjectSnapshot().getId().longValue(), Arrays.asList(SnapshotDataTypes.FILE_HASHES) - ); + ); if (!selectSnapshotData.isEmpty()) { SnapshotDataDto snapshotDataDto = selectSnapshotData.iterator().next(); String data = snapshotDataDto.getData(); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetection.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetection.java index 159a1f7f48e..96840779d22 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetection.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetection.java @@ -21,20 +21,23 @@ package org.sonar.batch.scan.filesystem; import org.apache.commons.lang.StringUtils; import org.sonar.api.batch.fs.InputFile; - -import java.util.Map; +import org.sonar.batch.protocol.input.FileData; +import org.sonar.batch.protocol.input.ProjectReferentials; class StatusDetection { - private final Map previousHashByRelativePath; + private final ProjectReferentials projectReferentials; - StatusDetection(Map previousHashByRelativePath) { - this.previousHashByRelativePath = previousHashByRelativePath; + StatusDetection(ProjectReferentials projectReferentials) { + this.projectReferentials = projectReferentials; } InputFile.Status status(String relativePath, String hash) { - String previousHash = previousHashByRelativePath.get(relativePath); - + FileData fileDataPerPath = projectReferentials.fileDataPerPath(relativePath); + if (fileDataPerPath == null) { + return InputFile.Status.ADDED; + } + String previousHash = fileDataPerPath.hash(); if (StringUtils.equals(hash, previousHash)) { return InputFile.Status.SAME; } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetectionFactory.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetectionFactory.java index 53729dde592..830cb346c09 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetectionFactory.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/StatusDetectionFactory.java @@ -20,25 +20,17 @@ package org.sonar.batch.scan.filesystem; import org.sonar.api.BatchComponent; - -import java.util.Collections; +import org.sonar.batch.protocol.input.ProjectReferentials; public class StatusDetectionFactory implements BatchComponent { - private final PreviousFileHashLoader previousFileHashLoader; - - public StatusDetectionFactory(PreviousFileHashLoader l) { - this.previousFileHashLoader = l; - } + private final ProjectReferentials projectReferentials; - /** - * Used by scan2 - */ - public StatusDetectionFactory() { - this.previousFileHashLoader = null; + public StatusDetectionFactory(ProjectReferentials projectReferentials) { + this.projectReferentials = projectReferentials; } StatusDetection create() { - return new StatusDetection(previousFileHashLoader != null ? previousFileHashLoader.hashByRelativePath() : Collections.emptyMap()); + return new StatusDetection(projectReferentials); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan2/DefaultSensorContext.java b/sonar-batch/src/main/java/org/sonar/batch/scan2/DefaultSensorContext.java index e8982ab4da7..0438d44c763 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan2/DefaultSensorContext.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan2/DefaultSensorContext.java @@ -59,7 +59,9 @@ public class DefaultSensorContext extends BaseSensorContext { private static final Logger LOG = LoggerFactory.getLogger(DefaultSensorContext.class); - public static final List INTERNAL_METRICS = Arrays.asList(CoreMetrics.DEPENDENCY_MATRIX, + public static final List INTERNAL_METRICS = Arrays.asList( + // Computed by DsmDecorator + CoreMetrics.DEPENDENCY_MATRIX, CoreMetrics.DIRECTORY_CYCLES, CoreMetrics.DIRECTORY_EDGES_WEIGHT, CoreMetrics.DIRECTORY_FEEDBACK_EDGES, @@ -69,7 +71,17 @@ public class DefaultSensorContext extends BaseSensorContext { CoreMetrics.FILE_EDGES_WEIGHT, CoreMetrics.FILE_FEEDBACK_EDGES, CoreMetrics.FILE_TANGLE_INDEX, - CoreMetrics.FILE_TANGLES + CoreMetrics.FILE_TANGLES, + // Computed by ScmActivitySensor + CoreMetrics.SCM_AUTHORS_BY_LINE, + CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE, + CoreMetrics.SCM_REVISIONS_BY_LINE, + // Computed by core duplication plugin + CoreMetrics.DUPLICATIONS_DATA, + CoreMetrics.DUPLICATION_LINES_DATA, + CoreMetrics.DUPLICATED_FILES, + CoreMetrics.DUPLICATED_LINES, + CoreMetrics.DUPLICATED_BLOCKS ); private final MeasureCache measureCache; private final IssueCache issueCache; @@ -97,8 +109,8 @@ public class DefaultSensorContext extends BaseSensorContext { @Override public void store(Measure newMeasure) { DefaultMeasure measure = (DefaultMeasure) newMeasure; - if (INTERNAL_METRICS.contains(measure.metric())) { - LOG.warn("Metric " + measure.metric().key() + " is an internal metric computed by SonarQube. Please update your plugin."); + if (!measure.isFromCore() && INTERNAL_METRICS.contains(measure.metric())) { + LOG.warn("Metric " + measure.metric().key() + " is an internal metric computed by SonarQube. Please remove or update offending plugin."); return; } InputFile inputFile = measure.inputFile(); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivityConfiguration.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivityConfiguration.java new file mode 100644 index 00000000000..36526063b6f --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivityConfiguration.java @@ -0,0 +1,100 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.scm; + +import com.google.common.base.Joiner; +import org.picocontainer.Startable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.BatchComponent; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.InstantiationStrategy; +import org.sonar.api.batch.bootstrap.ProjectReactor; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.config.Settings; + +import java.util.LinkedHashMap; +import java.util.Map; + +@InstantiationStrategy(InstantiationStrategy.PER_BATCH) +public final class ScmActivityConfiguration implements BatchComponent, Startable { + private static final Logger LOG = LoggerFactory.getLogger(ScmActivityConfiguration.class); + + private final ProjectReactor projectReactor; + private final Settings settings; + private final Map providerPerKey = new LinkedHashMap(); + + private ScmProvider provider; + + public ScmActivityConfiguration(ProjectReactor projectReactor, Settings settings, ScmProvider... providers) { + this.projectReactor = projectReactor; + this.settings = settings; + for (ScmProvider scmProvider : providers) { + providerPerKey.put(scmProvider.key(), scmProvider); + } + } + + public ScmActivityConfiguration(ProjectReactor projectReactor, Settings settings) { + this(projectReactor, settings, new ScmProvider[0]); + } + + @Override + public void start() { + if (!settings.getBoolean(CoreProperties.SCM_ENABLED_KEY)) { + LOG.debug("SCM Step is disabled by configuration"); + return; + } + if (settings.hasKey(CoreProperties.SCM_PROVIDER_KEY)) { + String forcedProviderKey = settings.getString(CoreProperties.SCM_PROVIDER_KEY); + if (providerPerKey.containsKey(forcedProviderKey)) { + this.provider = providerPerKey.get(forcedProviderKey); + } else { + throw new IllegalArgumentException("SCM provider was set to \"" + forcedProviderKey + "\" but no provider found for this key. Supported providers are " + + Joiner.on(",").join(providerPerKey.keySet())); + } + } else { + // Autodetection + for (ScmProvider provider : providerPerKey.values()) { + if (provider.supports(projectReactor.getRoot().getBaseDir())) { + if (this.provider == null) { + this.provider = provider; + } else { + throw new IllegalStateException("SCM provider autodetection failed. Both " + this.provider.key() + " and " + provider.key() + + " claim to support this project. Please use " + CoreProperties.SCM_PROVIDER_KEY + " to define SCM of your project."); + } + } + } + if (this.provider == null) { + throw new IllegalStateException("SCM provider autodetection failed. No provider claim to support this project. Please use " + CoreProperties.SCM_PROVIDER_KEY + + " to define SCM of your project."); + } + } + } + + public ScmProvider provider() { + return provider; + } + + @Override + public void stop() { + + } + +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivitySensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivitySensor.java new file mode 100644 index 00000000000..4bdf5163431 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmActivitySensor.java @@ -0,0 +1,168 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.scm; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.FileSystem; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.InputFile.Status; +import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.batch.scm.ScmProvider.BlameResultHandler; +import org.sonar.api.batch.sensor.Sensor; +import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.SensorDescriptor; +import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; +import org.sonar.api.measures.CoreMetrics; +import org.sonar.api.measures.Metric; +import org.sonar.api.measures.PropertiesBuilder; +import org.sonar.api.utils.DateUtils; +import org.sonar.api.utils.TimeProfiler; +import org.sonar.batch.protocol.input.FileData; +import org.sonar.batch.protocol.input.ProjectReferentials; + +import java.nio.charset.Charset; +import java.text.Normalizer; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Pattern; + +public final class ScmActivitySensor implements Sensor { + + private static final Logger LOG = LoggerFactory.getLogger(ScmActivitySensor.class); + + private static final Pattern NON_ASCII_CHARS = Pattern.compile("[^\\x00-\\x7F]"); + private static final Pattern ACCENT_CODES = Pattern.compile("\\p{InCombiningDiacriticalMarks}+"); + + private final ScmActivityConfiguration configuration; + private final FileSystem fs; + private final ProjectReferentials projectReferentials; + + public ScmActivitySensor(ScmActivityConfiguration configuration, ProjectReferentials projectReferentials, FileSystem fs) { + this.configuration = configuration; + this.projectReferentials = projectReferentials; + this.fs = fs; + } + + @Override + public void describe(SensorDescriptor descriptor) { + descriptor + .name("SCM Activity Sensor") + .provides(CoreMetrics.SCM_AUTHORS_BY_LINE, + CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE, + CoreMetrics.SCM_REVISIONS_BY_LINE); + } + + @Override + public void execute(final SensorContext context) { + if (configuration.provider() == null) { + LOG.info("No SCM provider"); + return; + } + + TimeProfiler profiler = new TimeProfiler().start("Retrieve SCM blame information with encoding " + Charset.defaultCharset()); + + List filesToBlame = new LinkedList(); + for (InputFile f : fs.inputFiles(fs.predicates().all())) { + FileData fileData = projectReferentials.fileDataPerPath(f.relativePath()); + if (f.status() == Status.SAME + && fileData != null + && fileData.scmAuthorsByLine() != null + && fileData.scmLastCommitDatetimesByLine() != null + && fileData.scmRevisionsByLine() != null) { + saveMeasures(context, f, fileData.scmAuthorsByLine(), fileData.scmLastCommitDatetimesByLine(), fileData.scmRevisionsByLine()); + } else { + filesToBlame.add(f); + } + } + configuration.provider().blame(filesToBlame, new BlameResultHandler() { + + @Override + public void handle(InputFile file, List lines) { + + PropertiesBuilder authors = propertiesBuilder(CoreMetrics.SCM_AUTHORS_BY_LINE); + PropertiesBuilder dates = propertiesBuilder(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE); + PropertiesBuilder revisions = propertiesBuilder(CoreMetrics.SCM_REVISIONS_BY_LINE); + + int lineNumber = 1; + for (BlameLine line : lines) { + authors.add(lineNumber, normalizeString(line.getAuthor())); + dates.add(lineNumber, DateUtils.formatDateTime(line.getDate())); + revisions.add(lineNumber, line.getRevision()); + + lineNumber++; + // SONARPLUGINS-3097 For some SCM blame is missing on last empty line + if (lineNumber > lines.size() && lineNumber == file.lines()) { + authors.add(lineNumber, normalizeString(line.getAuthor())); + dates.add(lineNumber, DateUtils.formatDateTime(line.getDate())); + revisions.add(lineNumber, line.getRevision()); + } + } + + saveMeasures(context, file, authors.buildData(), dates.buildData(), revisions.buildData()); + + } + }); + profiler.stop(); + } + + private String normalizeString(String inputString) { + String lowerCasedString = inputString.toLowerCase(); + String stringWithoutAccents = removeAccents(lowerCasedString); + return removeNonAsciiCharacters(stringWithoutAccents); + } + + private String removeAccents(String inputString) { + String unicodeDecomposedString = Normalizer.normalize(inputString, Normalizer.Form.NFD); + return ACCENT_CODES.matcher(unicodeDecomposedString).replaceAll(""); + } + + private String removeNonAsciiCharacters(String inputString) { + return NON_ASCII_CHARS.matcher(inputString).replaceAll("_"); + } + + private static PropertiesBuilder propertiesBuilder(Metric metric) { + return new PropertiesBuilder(metric); + } + + /** + * This method is synchronized since it is allowed for plugins to compute blame in parallel. + */ + private synchronized void saveMeasures(SensorContext context, InputFile f, String scmAuthorsByLine, String scmLastCommitDatetimesByLine, String scmRevisionsByLine) { + ((DefaultMeasure) context.newMeasure() + .onFile(f) + .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE) + .withValue(scmAuthorsByLine)) + .setFromCore() + .save(); + ((DefaultMeasure) context.newMeasure() + .onFile(f) + .forMetric(CoreMetrics.SCM_LAST_COMMIT_DATETIMES_BY_LINE) + .withValue(scmLastCommitDatetimesByLine)) + .setFromCore() + .save(); + ((DefaultMeasure) context.newMeasure() + .onFile(f) + .forMetric(CoreMetrics.SCM_REVISIONS_BY_LINE) + .withValue(scmRevisionsByLine)) + .setFromCore() + .save(); + } +} diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java index 2044ee9910a..d0f60bb4858 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/measures/MeasuresMediumTest.java @@ -66,7 +66,7 @@ public class MeasuresMediumTest { .newScanTask(new File(projectDir, "sonar-project.properties")) .start(); - assertThat(result.measures()).hasSize(19); + assertThat(result.measures()).hasSize(13); } @Test @@ -103,7 +103,7 @@ public class MeasuresMediumTest { } @Test - public void testDistributionMeasure() throws IOException { + public void testScmMeasure() throws IOException { File baseDir = temp.newFolder(); File srcDir = new File(baseDir, "src"); @@ -132,6 +132,8 @@ public class MeasuresMediumTest { .put("sonar.projectVersion", "1.0-SNAPSHOT") .put("sonar.projectDescription", "Description of Foo Project") .put("sonar.sources", "src") + .put("sonar.scm.enabled", "true") + .put("sonar.scm.provider", "xoo") .build()) .start(); diff --git a/sonar-batch/src/test/java/org/sonar/batch/referential/DefaultProjectReferentialsLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/referential/DefaultProjectReferentialsLoaderTest.java index d93f9aecbe1..95821910b74 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/referential/DefaultProjectReferentialsLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/referential/DefaultProjectReferentialsLoaderTest.java @@ -24,10 +24,14 @@ import org.junit.Before; import org.junit.Test; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.bootstrap.ProjectReactor; +import org.sonar.api.measures.Metric; +import org.sonar.api.measures.MetricFinder; +import org.sonar.batch.DefaultTimeMachine; import org.sonar.batch.bootstrap.AnalysisMode; import org.sonar.batch.bootstrap.ServerClient; import org.sonar.batch.bootstrap.TaskProperties; import org.sonar.batch.rule.ModuleQProfiles; +import org.sonar.batch.scan.filesystem.PreviousFileHashLoader; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; @@ -46,7 +50,9 @@ public class DefaultProjectReferentialsLoaderTest { public void prepare() { serverClient = mock(ServerClient.class); analysisMode = mock(AnalysisMode.class); - loader = new DefaultProjectReferentialsLoader(serverClient, analysisMode); + MetricFinder metricFinder = mock(MetricFinder.class); + when(metricFinder.findByKey(anyString())).thenReturn(new Metric().setId(1)); + loader = new DefaultProjectReferentialsLoader(serverClient, analysisMode, mock(PreviousFileHashLoader.class), metricFinder, mock(DefaultTimeMachine.class)); when(serverClient.request(anyString())).thenReturn(""); reactor = new ProjectReactor(ProjectDefinition.create().setKey("foo")); taskProperties = new TaskProperties(Maps.newHashMap(), ""); diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionFactoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionFactoryTest.java index ee24cbaec0e..46bf71ecc8b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionFactoryTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionFactoryTest.java @@ -20,6 +20,7 @@ package org.sonar.batch.scan.filesystem; import org.junit.Test; +import org.sonar.batch.protocol.input.ProjectReferentials; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -27,7 +28,7 @@ import static org.mockito.Mockito.mock; public class StatusDetectionFactoryTest { @Test public void testCreate() throws Exception { - StatusDetectionFactory factory = new StatusDetectionFactory(mock(PreviousFileHashLoader.class)); + StatusDetectionFactory factory = new StatusDetectionFactory(mock(ProjectReferentials.class)); StatusDetection detection = factory.create(); assertThat(detection).isNotNull(); } diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionTest.java index 1c4ad73733b..ecadbe3b1be 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/StatusDetectionTest.java @@ -19,19 +19,20 @@ */ package org.sonar.batch.scan.filesystem; -import com.google.common.collect.ImmutableMap; import org.junit.Test; import org.sonar.api.batch.fs.InputFile; +import org.sonar.batch.protocol.input.FileData; +import org.sonar.batch.protocol.input.ProjectReferentials; import static org.fest.assertions.Assertions.assertThat; public class StatusDetectionTest { @Test public void detect_status() throws Exception { - StatusDetection statusDetection = new StatusDetection(ImmutableMap.of( - "src/Foo.java", "ABCDE", - "src/Bar.java", "FGHIJ" - )); + ProjectReferentials ref = new ProjectReferentials(); + ref.fileDataPerPath().put("src/Foo.java", new FileData("ABCDE", null, null, null)); + ref.fileDataPerPath().put("src/Bar.java", new FileData("FGHIJ", null, null, null)); + StatusDetection statusDetection = new StatusDetection(ref); assertThat(statusDetection.status("src/Foo.java", "ABCDE")).isEqualTo(InputFile.Status.SAME); assertThat(statusDetection.status("src/Foo.java", "XXXXX")).isEqualTo(InputFile.Status.CHANGED); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java index 7016d777d8a..b0d48001837 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java @@ -615,4 +615,14 @@ public interface CoreProperties { * @since 4.5 */ String LANGUAGE_SPECIFIC_PARAMETERS_SIZE_METRIC_KEY = "size_metric"; + + /** + * @since 5.0 + */ + String SCM_ENABLED_KEY = "sonar.scm.enabled"; + + /** + * @since 5.0 + */ + String SCM_PROVIDER_KEY = "sonar.scm.provider"; } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFileFilter.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFileFilter.java index 1ece5081290..91836823e43 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFileFilter.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/InputFileFilter.java @@ -27,7 +27,6 @@ import org.sonar.api.BatchExtension; */ public interface InputFileFilter extends BatchExtension { - // TODO requires a context (FileSystem) ? boolean accept(InputFile f); } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/BlameLine.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/BlameLine.java new file mode 100644 index 00000000000..2303b90f794 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/BlameLine.java @@ -0,0 +1,102 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.api.batch.scm; + +import java.util.Date; + +/** + * @since 5.0 + */ +public class BlameLine { + + private Date date; + private String revision; + private String author; + private String committer; + + /** + * @param date of the commit + * @param revision of the commit + * @param author will also be used as committer identification + */ + public BlameLine(Date date, String revision, String author) { + this(date, revision, author, author); + } + + /** + * + * @param date of the commit + * @param revision of the commit + * @param author the person who wrote the line + * @param committer the person who committed the change + */ + public BlameLine(Date date, String revision, String author, String committer) { + setDate(date); + setRevision(revision); + setAuthor(author); + setCommitter(committer); + } + + public String getRevision() { + return revision; + } + + public void setRevision(String revision) { + this.revision = revision; + } + + public String getAuthor() { + return author; + } + + public void setAuthor(String author) { + this.author = author; + } + + public String getCommitter() { + return committer; + } + + public void setCommitter(String committer) { + this.committer = committer; + } + + /** + * @return the commit date + */ + public Date getDate() { + if (date != null) + { + return (Date) date.clone(); + } + return null; + } + + public void setDate(Date date) { + if (date != null) + { + this.date = new Date(date.getTime()); + } + else + { + this.date = null; + } + } +} diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/ScmProvider.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/ScmProvider.java new file mode 100644 index 00000000000..1fd3327d7c3 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/ScmProvider.java @@ -0,0 +1,61 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.api.batch.scm; + +import org.sonar.api.BatchExtension; +import org.sonar.api.batch.InstantiationStrategy; +import org.sonar.api.batch.fs.InputFile; + +import java.io.File; +import java.util.List; + +/** + * @since 5.0 + */ +@InstantiationStrategy(InstantiationStrategy.PER_BATCH) +public interface ScmProvider extends BatchExtension { + + /** + * Unique identifier of the provider. Can be used in SCM URL to define the provider to use. + */ + String key(); + + /** + * Does this provider able to manage files located in this directory. + * Used by autodetection. + */ + boolean supports(File baseDir); + + /** + * Compute blame of the provided files. Computation can be done in parallel. + * If there is an error that prevent to blame a file then an exception should be raised. + */ + void blame(Iterable files, BlameResultHandler handler); + + /** + * Callback for the provider to return results of blame per file. + */ + public static interface BlameResultHandler { + + void handle(InputFile file, List lines); + + } + +} diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/package-info.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/package-info.java new file mode 100644 index 00000000000..03eafe5f6e9 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/scm/package-info.java @@ -0,0 +1,24 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.api.batch.scm; + +import javax.annotation.ParametersAreNonnullByDefault; + diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/measure/internal/DefaultMeasure.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/measure/internal/DefaultMeasure.java index fee7172f9ea..29b5c92bc2c 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/measure/internal/DefaultMeasure.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/sensor/measure/internal/DefaultMeasure.java @@ -42,6 +42,7 @@ public class DefaultMeasure implements Measure { private Metric metric; private G value; private boolean saved = false; + private boolean fromCore = false; public DefaultMeasure() { this.storage = null; @@ -84,6 +85,21 @@ public class DefaultMeasure implements Measure { return this; } + /** + * For internal use. + */ + public boolean isFromCore() { + return fromCore; + } + + /** + * For internal use. Used by core components to bypass check that prevent a plugin to store core measures. + */ + public DefaultMeasure setFromCore() { + this.fromCore = true; + return this; + } + @Override public void save() { Preconditions.checkNotNull(this.storage, "No persister on this object"); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/measures/Measure.java b/sonar-plugin-api/src/main/java/org/sonar/api/measures/Measure.java index af202228252..9a75236a501 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/measures/Measure.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/measures/Measure.java @@ -63,6 +63,7 @@ public class Measure implements Serializable { protected Requirement requirement; protected Integer personId; protected PersistenceMode persistenceMode = PersistenceMode.FULL; + private boolean fromCore; public Measure(String metricKey) { this.metricKey = metricKey; @@ -687,6 +688,20 @@ public class Measure implements Serializable { && isZeroVariation(variation1, variation2, variation3, variation4, variation5); } + /** + * For internal use + */ + public boolean isFromCore() { + return fromCore; + } + + /** + * For internal use + */ + public void setFromCore(boolean fromCore) { + this.fromCore = fromCore; + } + private static boolean isZeroVariation(Double... variations) { for (Double variation : variations) { if (!((variation == null) || NumberUtils.compare(variation, 0.0) == 0)) { -- 2.39.5