From c64c9cf6a5fb914118bff87c626ba1045f5d33e9 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Thu, 1 Oct 2015 10:28:07 +0200 Subject: [PATCH] SONAR-6397 Execute blame only on updated files --- .../sonar/batch/scm/DefaultBlameOutput.java | 6 +- .../org/sonar/batch/scm/ScmConfiguration.java | 9 +- .../java/org/sonar/batch/scm/ScmSensor.java | 27 ++-- .../org/sonar/batch/util/ProgressReport.java | 6 +- .../batch/mediumtest/scm/ScmMediumTest.java | 123 +++++++++++++----- 5 files changed, 117 insertions(+), 54 deletions(-) diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java index f7ccc0b3e51..7ed72faabad 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java @@ -31,11 +31,11 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.scm.BlameCommand.BlameOutput; import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.batch.index.BatchComponent; import org.sonar.batch.index.BatchComponentCache; import org.sonar.batch.protocol.output.BatchReport; @@ -45,7 +45,7 @@ import org.sonar.batch.util.ProgressReport; class DefaultBlameOutput implements BlameOutput { - private static final Logger LOG = LoggerFactory.getLogger(DefaultBlameOutput.class); + private static final Logger LOG = Loggers.get(DefaultBlameOutput.class); private static final Pattern NON_ASCII_CHARS = Pattern.compile("[^\\x00-\\x7F]"); private static final Pattern ACCENT_CODES = Pattern.compile("\\p{InCombiningDiacriticalMarks}+"); diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java index d9e44531827..92566ddc971 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java @@ -23,8 +23,6 @@ import com.google.common.base.Joiner; import java.util.LinkedHashMap; import java.util.Map; import org.picocontainer.Startable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; import org.sonar.api.Properties; import org.sonar.api.Property; @@ -34,6 +32,8 @@ import org.sonar.api.batch.BatchSide; import org.sonar.api.batch.InstantiationStrategy; import org.sonar.api.batch.scm.ScmProvider; import org.sonar.api.config.Settings; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.batch.scan.ImmutableProjectReactor; @Properties({ @@ -46,13 +46,12 @@ import org.sonar.batch.scan.ImmutableProjectReactor; module = false, project = false, global = false, - category = CoreProperties.CATEGORY_SCM - ) + category = CoreProperties.CATEGORY_SCM) }) @InstantiationStrategy(InstantiationStrategy.PER_BATCH) @BatchSide public final class ScmConfiguration implements Startable { - private static final Logger LOG = LoggerFactory.getLogger(ScmConfiguration.class); + private static final Logger LOG = Loggers.get(ScmConfiguration.class); public static final String FORCE_RELOAD_KEY = "sonar.scm.forceReloadAll"; diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java index 8ff079170e8..d0111c2c91a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java @@ -19,15 +19,9 @@ */ package org.sonar.batch.scm; -import org.sonar.batch.repository.FileData; - -import org.sonar.batch.repository.ProjectRepositories; - import java.util.LinkedList; import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.lang.StringUtils; import org.sonar.api.CoreProperties; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.FileSystem; @@ -36,27 +30,31 @@ import org.sonar.api.batch.fs.InputFile.Status; import org.sonar.api.batch.sensor.Sensor; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.SensorDescriptor; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.batch.index.BatchComponentCache; import org.sonar.batch.report.ReportPublisher; +import org.sonar.batch.repository.FileData; +import org.sonar.batch.repository.ProjectRepositories; import org.sonar.batch.scan.filesystem.InputPathCache; public final class ScmSensor implements Sensor { - private static final Logger LOG = LoggerFactory.getLogger(ScmSensor.class); + private static final Logger LOG = Loggers.get(ScmSensor.class); private final ProjectDefinition projectDefinition; private final ScmConfiguration configuration; private final FileSystem fs; - private final ProjectRepositories projectSettings; + private final ProjectRepositories projectRepositories; private final BatchComponentCache resourceCache; private final ReportPublisher publishReportJob; public ScmSensor(ProjectDefinition projectDefinition, ScmConfiguration configuration, - ProjectRepositories projectSettings, FileSystem fs, InputPathCache inputPathCache, BatchComponentCache resourceCache, + ProjectRepositories projectRepositories, FileSystem fs, InputPathCache inputPathCache, BatchComponentCache resourceCache, ReportPublisher publishReportJob) { this.projectDefinition = projectDefinition; this.configuration = configuration; - this.projectSettings = projectSettings; + this.projectRepositories = projectRepositories; this.fs = fs; this.resourceCache = resourceCache; this.publishReportJob = publishReportJob; @@ -95,11 +93,12 @@ public final class ScmSensor implements Sensor { } List filesToBlame = new LinkedList<>(); for (InputFile f : fs.inputFiles(fs.predicates().all())) { - if (configuration.forceReloadAll()) { + if (configuration.forceReloadAll() || f.status() != Status.SAME) { addIfNotEmpty(filesToBlame, f); } else { - FileData fileData = projectSettings.fileData(projectDefinition.getKeyWithBranch(), f.relativePath()); - if (f.status() != Status.SAME || fileData == null || fileData.revision() != null) { + // File status is SAME so that mean fileData exists + FileData fileData = projectRepositories.fileData(projectDefinition.getKeyWithBranch(), f.relativePath()); + if (StringUtils.isEmpty(fileData.revision())) { addIfNotEmpty(filesToBlame, f); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java b/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java index f152acca65e..406799da72a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java +++ b/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java @@ -19,12 +19,12 @@ */ package org.sonar.batch.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; public class ProgressReport implements Runnable { - private static final Logger LOG = LoggerFactory.getLogger(ProgressReport.class); + private static final Logger LOG = Loggers.get(ProgressReport.class); private final long period; private String message = ""; private final Thread thread; diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java index d7a4af92705..32d0486df5b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java @@ -19,9 +19,9 @@ */ package org.sonar.batch.mediumtest.scm; -import org.sonar.batch.repository.FileData; - import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.junit.After; @@ -30,21 +30,25 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.sonar.api.utils.PathUtils; +import org.sonar.api.utils.log.LogTester; import org.sonar.batch.mediumtest.BatchMediumTester; import org.sonar.batch.mediumtest.BatchMediumTester.TaskBuilder; import org.sonar.batch.protocol.output.BatchReport; import org.sonar.batch.protocol.output.BatchReport.Changesets.Changeset; import org.sonar.batch.protocol.output.BatchReport.Component; import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.batch.repository.FileData; import org.sonar.xoo.XooPlugin; -import java.io.File; -import java.io.IOException; - import static org.assertj.core.api.Assertions.assertThat; public class ScmMediumTest { + private static final String MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES = "Missing blame information for the following files:"; + private static final String CHANGED_CONTENT_SCM_ON_SERVER_XOO = "src/changed_content_scm_on_server.xoo"; + private static final String SAME_CONTENT_SCM_ON_SERVER_XOO = "src/same_content_scm_on_server.xoo"; + private static final String SAME_CONTENT_NO_SCM_ON_SERVER_XOO = "src/same_content_no_scm_on_server.xoo"; private static final String SAMPLE_XOO_CONTENT = "Sample xoo\ncontent"; @org.junit.Rule @@ -53,10 +57,15 @@ public class ScmMediumTest { @Rule public ExpectedException thrown = ExpectedException.none(); + @Rule + public LogTester logTester = new LogTester(); + public BatchMediumTester tester = BatchMediumTester.builder() .registerPlugin("xoo", new XooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") - .addFileData("com.foo.project", "src/sample2.xoo", new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null)) + .addFileData("com.foo.project", CHANGED_CONTENT_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null)) + .addFileData("com.foo.project", SAME_CONTENT_NO_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null)) + .addFileData("com.foo.project", SAME_CONTENT_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), "1.1")) .build(); @Before @@ -87,7 +96,7 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets fileScm = getChangesets(baseDir, 0); + BatchReport.Changesets fileScm = getChangesets(baseDir, "src/sample.xoo"); assertThat(fileScm.getChangesetIndexByLineList()).hasSize(5); @@ -107,15 +116,19 @@ public class ScmMediumTest { assertThat(changesetLine5.getAuthor()).isEqualTo("simon"); } - private BatchReport.Changesets getChangesets(File baseDir, int fileId) { + private BatchReport.Changesets getChangesets(File baseDir, String path) { File reportDir = new File(baseDir, ".sonar/batch-report"); BatchReportReader reader = new BatchReportReader(reportDir); Component project = reader.readComponent(reader.readMetadata().getRootComponentRef()); Component dir = reader.readComponent(project.getChildRef(0)); - Component file = reader.readComponent(dir.getChildRef(fileId)); - BatchReport.Changesets fileScm = reader.readChangesets(file.getRef()); - return fileScm; + for (Integer fileRef : dir.getChildRefList()) { + Component file = reader.readComponent(fileRef); + if (file.getPath().equals(path)) { + return reader.readChangesets(file.getRef()); + } + } + return null; } @Test @@ -139,17 +152,17 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets changesets = getChangesets(baseDir, 0); + BatchReport.Changesets changesets = getChangesets(baseDir, "src/sample.xoo"); assertThat(changesets).isNull(); } @Test - public void sample2_dont_need_blame() throws IOException { + public void log_files_with_missing_blame() throws IOException { File baseDir = prepareProject(); - File xooFile = new File(baseDir, "src/sample2.xoo"); - FileUtils.write(xooFile, "Sample xoo\ncontent\n3\n4\n5"); + File xooFileWithoutBlame = new File(baseDir, "src/sample_no_blame.xoo"); + FileUtils.write(xooFileWithoutBlame, "Sample xoo\ncontent\n3\n4\n5"); tester.newTask() .properties(ImmutableMap.builder() @@ -164,25 +177,78 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets file1Scm = getChangesets(baseDir, 0); + BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo"); assertThat(file1Scm).isNotNull(); - BatchReport.Changesets file2Scm = getChangesets(baseDir, 1); - assertThat(file2Scm).isNull(); + BatchReport.Changesets fileWithoutBlameScm = getChangesets(baseDir, "src/sample_no_blame.xoo"); + assertThat(fileWithoutBlameScm).isNull(); + + assertThat(logTester.logs()).containsSubsequence("2 files to be analyzed", "1/2 files analyzed", MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, + " * " + PathUtils.sanitize(xooFileWithoutBlame.toPath().toString())); + } + + // SONAR-6397 + @Test + public void optimize_blame() throws IOException { + + File baseDir = prepareProject(); + File changedContentScmOnServer = new File(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO); + FileUtils.write(changedContentScmOnServer, SAMPLE_XOO_CONTENT + "\nchanged"); + File xooScmFile = new File(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO + ".scm"); + FileUtils.write(xooScmFile, + // revision,author,dateTime + "1,foo,2013-01-04\n" + + "1,bar,2013-01-04\n" + + "2,biz,2014-01-04\n"); + + File sameContentScmOnServer = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO); + FileUtils.write(sameContentScmOnServer, SAMPLE_XOO_CONTENT); + // No need to write .scm file since this file should not be blamed + + File sameContentNoScmOnServer = new File(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO); + FileUtils.write(sameContentNoScmOnServer, SAMPLE_XOO_CONTENT); + xooScmFile = new File(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO + ".scm"); + FileUtils.write(xooScmFile, + // revision,author,dateTime + "1,foo,2013-01-04\n" + + "1,bar,2013-01-04\n"); + + tester.newTask() + .properties(ImmutableMap.builder() + .put("sonar.task", "scan") + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.projectName", "Foo Project") + .put("sonar.projectVersion", "1.0-SNAPSHOT") + .put("sonar.projectDescription", "Description of Foo Project") + .put("sonar.sources", "src") + .put("sonar.scm.provider", "xoo") + .build()) + .start(); + + assertThat(getChangesets(baseDir, "src/sample.xoo")).isNotNull(); + + assertThat(getChangesets(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO)).isNotNull(); + + assertThat(getChangesets(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO)).isNull(); + + assertThat(getChangesets(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO)).isNotNull(); + + assertThat(logTester.logs()).containsSubsequence("3 files to be analyzed", "3/3 files analyzed"); + assertThat(logTester.logs()).doesNotContain(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES); } @Test public void forceReload() throws IOException { File baseDir = prepareProject(); - File xooFileNoScm = new File(baseDir, "src/sample2.xoo"); + File xooFileNoScm = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO); FileUtils.write(xooFileNoScm, SAMPLE_XOO_CONTENT); - File xooScmFile = new File(baseDir, "src/sample2.xoo.scm"); + File xooScmFile = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO + ".scm"); FileUtils.write(xooScmFile, // revision,author,dateTime "1,foo,2013-01-04\n" + - "1,bar,2013-01-04\n" - ); + "1,bar,2013-01-04\n"); TaskBuilder taskBuilder = tester.newTask() .properties(ImmutableMap.builder() @@ -200,10 +266,10 @@ public class ScmMediumTest { taskBuilder.start(); - BatchReport.Changesets file1Scm = getChangesets(baseDir, 0); + BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo"); assertThat(file1Scm).isNotNull(); - BatchReport.Changesets file2Scm = getChangesets(baseDir, 1); + BatchReport.Changesets file2Scm = getChangesets(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO); assertThat(file2Scm).isNotNull(); } @@ -225,7 +291,7 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets file1Scm = getChangesets(baseDir, 0); + BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo"); assertThat(file1Scm).isNotNull(); } @@ -247,7 +313,7 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets file1Scm = getChangesets(baseDir, 0); + BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo"); assertThat(file1Scm).isNotNull(); } @@ -265,8 +331,7 @@ public class ScmMediumTest { "2,julien,2013-01-04\n" + "3,julien,2013-02-03\n" + "3,julien,2013-02-03\n" + - "4,simon,2013-03-04\n" - ); + "4,simon,2013-03-04\n"); return baseDir; } @@ -291,7 +356,7 @@ public class ScmMediumTest { .build()) .start(); - BatchReport.Changesets changesets = getChangesets(baseDir, 0); + BatchReport.Changesets changesets = getChangesets(baseDir, "src/sample.xoo"); assertThat(changesets).isNull(); } -- 2.39.5