SONAR-6397 Execute blame only on updated files

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
index f7ccc0b3e5184524117759effcfcd5d9bbf964bd..7ed72faabad3626dc1ddfc1ceafe615037be2e4d 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/DefaultBlameOutput.java
@@ -31,11 +31,11 @@ import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.scm.BlameCommand.BlameOutput;
 import org.sonar.api.batch.scm.BlameLine;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.protocol.output.BatchReport;
@@ -45,7 +45,7 @@ import org.sonar.batch.util.ProgressReport;
 
 class DefaultBlameOutput implements BlameOutput {
 
-  private static final Logger LOG = LoggerFactory.getLogger(DefaultBlameOutput.class);
+  private static final Logger LOG = Loggers.get(DefaultBlameOutput.class);
 
   private static final Pattern NON_ASCII_CHARS = Pattern.compile("[^\\x00-\\x7F]");
   private static final Pattern ACCENT_CODES = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
index d9e44531827312298d48b90a91fa919b38750cae..92566ddc971f865afdfc3dfd8d1bf9b5d3564f00 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
@@ -23,8 +23,6 @@ import com.google.common.base.Joiner;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import org.picocontainer.Startable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.Properties;
 import org.sonar.api.Property;
@@ -34,6 +32,8 @@ import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.InstantiationStrategy;
 import org.sonar.api.batch.scm.ScmProvider;
 import org.sonar.api.config.Settings;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.batch.scan.ImmutableProjectReactor;
 
 @Properties({
@@ -46,13 +46,12 @@ import org.sonar.batch.scan.ImmutableProjectReactor;
     module = false,
     project = false,
     global = false,
-    category = CoreProperties.CATEGORY_SCM
-  )
+    category = CoreProperties.CATEGORY_SCM)
 })
 @InstantiationStrategy(InstantiationStrategy.PER_BATCH)
 @BatchSide
 public final class ScmConfiguration implements Startable {
-  private static final Logger LOG = LoggerFactory.getLogger(ScmConfiguration.class);
+  private static final Logger LOG = Loggers.get(ScmConfiguration.class);
 
   public static final String FORCE_RELOAD_KEY = "sonar.scm.forceReloadAll";
 

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
index 8ff079170e807a0f5fd041f9b06d175cec4c3df0..d0111c2c91ad1cb5d040faab8a9f2db141f1df20 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
@@ -19,15 +19,9 @@
  */
 package org.sonar.batch.scm;
 
-import org.sonar.batch.repository.FileData;
-
-import org.sonar.batch.repository.ProjectRepositories;
-
 import java.util.LinkedList;
 import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.lang.StringUtils;
 import org.sonar.api.CoreProperties;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
 import org.sonar.api.batch.fs.FileSystem;
@@ -36,27 +30,31 @@ import org.sonar.api.batch.fs.InputFile.Status;
 import org.sonar.api.batch.sensor.Sensor;
 import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.SensorDescriptor;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.batch.report.ReportPublisher;
+import org.sonar.batch.repository.FileData;
+import org.sonar.batch.repository.ProjectRepositories;
 import org.sonar.batch.scan.filesystem.InputPathCache;
 
 public final class ScmSensor implements Sensor {
 
-  private static final Logger LOG = LoggerFactory.getLogger(ScmSensor.class);
+  private static final Logger LOG = Loggers.get(ScmSensor.class);
 
   private final ProjectDefinition projectDefinition;
   private final ScmConfiguration configuration;
   private final FileSystem fs;
-  private final ProjectRepositories projectSettings;
+  private final ProjectRepositories projectRepositories;
   private final BatchComponentCache resourceCache;
   private final ReportPublisher publishReportJob;
 
   public ScmSensor(ProjectDefinition projectDefinition, ScmConfiguration configuration,
-    ProjectRepositories projectSettings, FileSystem fs, InputPathCache inputPathCache, BatchComponentCache resourceCache,
+    ProjectRepositories projectRepositories, FileSystem fs, InputPathCache inputPathCache, BatchComponentCache resourceCache,
     ReportPublisher publishReportJob) {
     this.projectDefinition = projectDefinition;
     this.configuration = configuration;
-    this.projectSettings = projectSettings;
+    this.projectRepositories = projectRepositories;
     this.fs = fs;
     this.resourceCache = resourceCache;
     this.publishReportJob = publishReportJob;
@@ -95,11 +93,12 @@ public final class ScmSensor implements Sensor {
     }
     List<InputFile> filesToBlame = new LinkedList<>();
     for (InputFile f : fs.inputFiles(fs.predicates().all())) {
-      if (configuration.forceReloadAll()) {
+      if (configuration.forceReloadAll() || f.status() != Status.SAME) {
         addIfNotEmpty(filesToBlame, f);
       } else {
-        FileData fileData = projectSettings.fileData(projectDefinition.getKeyWithBranch(), f.relativePath());
-        if (f.status() != Status.SAME || fileData == null || fileData.revision() != null) {
+        // File status is SAME so that mean fileData exists
+        FileData fileData = projectRepositories.fileData(projectDefinition.getKeyWithBranch(), f.relativePath());
+        if (StringUtils.isEmpty(fileData.revision())) {
           addIfNotEmpty(filesToBlame, f);
         }
       }

diff --git a/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java b/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java
index f152acca65e8b7722c97520c416847ff9e19e214..406799da72a24826af4c436d819bfec08bb52934 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/util/ProgressReport.java
@@ -19,12 +19,12 @@
  */
 package org.sonar.batch.util;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 
 public class ProgressReport implements Runnable {
 
-  private static final Logger LOG = LoggerFactory.getLogger(ProgressReport.class);
+  private static final Logger LOG = Loggers.get(ProgressReport.class);
   private final long period;
   private String message = "";
   private final Thread thread;

diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
index d7a4af92705e90d76e10bfb45674e0385dd834a7..32d0486df5b58356f75d5211d576493a2660c730 100644 (file)
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
@@ -19,9 +19,9 @@
  */
 package org.sonar.batch.mediumtest.scm;
 
-import org.sonar.batch.repository.FileData;
-
 import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import java.io.IOException;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.junit.After;
@@ -30,21 +30,25 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
+import org.sonar.api.utils.PathUtils;
+import org.sonar.api.utils.log.LogTester;
 import org.sonar.batch.mediumtest.BatchMediumTester;
 import org.sonar.batch.mediumtest.BatchMediumTester.TaskBuilder;
 import org.sonar.batch.protocol.output.BatchReport;
 import org.sonar.batch.protocol.output.BatchReport.Changesets.Changeset;
 import org.sonar.batch.protocol.output.BatchReport.Component;
 import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.batch.repository.FileData;
 import org.sonar.xoo.XooPlugin;
 
-import java.io.File;
-import java.io.IOException;
-
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class ScmMediumTest {
 
+  private static final String MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES = "Missing blame information for the following files:";
+  private static final String CHANGED_CONTENT_SCM_ON_SERVER_XOO = "src/changed_content_scm_on_server.xoo";
+  private static final String SAME_CONTENT_SCM_ON_SERVER_XOO = "src/same_content_scm_on_server.xoo";
+  private static final String SAME_CONTENT_NO_SCM_ON_SERVER_XOO = "src/same_content_no_scm_on_server.xoo";
   private static final String SAMPLE_XOO_CONTENT = "Sample xoo\ncontent";
 
   @org.junit.Rule
@@ -53,10 +57,15 @@ public class ScmMediumTest {
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 
+  @Rule
+  public LogTester logTester = new LogTester();
+
   public BatchMediumTester tester = BatchMediumTester.builder()
     .registerPlugin("xoo", new XooPlugin())
     .addDefaultQProfile("xoo", "Sonar Way")
-    .addFileData("com.foo.project", "src/sample2.xoo", new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null))
+    .addFileData("com.foo.project", CHANGED_CONTENT_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null))
+    .addFileData("com.foo.project", SAME_CONTENT_NO_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), null))
+    .addFileData("com.foo.project", SAME_CONTENT_SCM_ON_SERVER_XOO, new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), "1.1"))
     .build();
 
   @Before
@@ -87,7 +96,7 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets fileScm = getChangesets(baseDir, 0);
+    BatchReport.Changesets fileScm = getChangesets(baseDir, "src/sample.xoo");
 
     assertThat(fileScm.getChangesetIndexByLineList()).hasSize(5);
 
@@ -107,15 +116,19 @@ public class ScmMediumTest {
     assertThat(changesetLine5.getAuthor()).isEqualTo("simon");
   }
 
-  private BatchReport.Changesets getChangesets(File baseDir, int fileId) {
+  private BatchReport.Changesets getChangesets(File baseDir, String path) {
     File reportDir = new File(baseDir, ".sonar/batch-report");
     BatchReportReader reader = new BatchReportReader(reportDir);
 
     Component project = reader.readComponent(reader.readMetadata().getRootComponentRef());
     Component dir = reader.readComponent(project.getChildRef(0));
-    Component file = reader.readComponent(dir.getChildRef(fileId));
-    BatchReport.Changesets fileScm = reader.readChangesets(file.getRef());
-    return fileScm;
+    for (Integer fileRef : dir.getChildRefList()) {
+      Component file = reader.readComponent(fileRef);
+      if (file.getPath().equals(path)) {
+        return reader.readChangesets(file.getRef());
+      }
+    }
+    return null;
   }
 
   @Test
@@ -139,17 +152,17 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets changesets = getChangesets(baseDir, 0);
+    BatchReport.Changesets changesets = getChangesets(baseDir, "src/sample.xoo");
 
     assertThat(changesets).isNull();
   }
 
   @Test
-  public void sample2_dont_need_blame() throws IOException {
+  public void log_files_with_missing_blame() throws IOException {
 
     File baseDir = prepareProject();
-    File xooFile = new File(baseDir, "src/sample2.xoo");
-    FileUtils.write(xooFile, "Sample xoo\ncontent\n3\n4\n5");
+    File xooFileWithoutBlame = new File(baseDir, "src/sample_no_blame.xoo");
+    FileUtils.write(xooFileWithoutBlame, "Sample xoo\ncontent\n3\n4\n5");
 
     tester.newTask()
       .properties(ImmutableMap.<String, String>builder()
@@ -164,25 +177,78 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets file1Scm = getChangesets(baseDir, 0);
+    BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo");
     assertThat(file1Scm).isNotNull();
 
-    BatchReport.Changesets file2Scm = getChangesets(baseDir, 1);
-    assertThat(file2Scm).isNull();
+    BatchReport.Changesets fileWithoutBlameScm = getChangesets(baseDir, "src/sample_no_blame.xoo");
+    assertThat(fileWithoutBlameScm).isNull();
+
+    assertThat(logTester.logs()).containsSubsequence("2 files to be analyzed", "1/2 files analyzed", MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES,
+      "  * " + PathUtils.sanitize(xooFileWithoutBlame.toPath().toString()));
+  }
+
+  // SONAR-6397
+  @Test
+  public void optimize_blame() throws IOException {
+
+    File baseDir = prepareProject();
+    File changedContentScmOnServer = new File(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO);
+    FileUtils.write(changedContentScmOnServer, SAMPLE_XOO_CONTENT + "\nchanged");
+    File xooScmFile = new File(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO + ".scm");
+    FileUtils.write(xooScmFile,
+      // revision,author,dateTime
+      "1,foo,2013-01-04\n" +
+        "1,bar,2013-01-04\n" +
+        "2,biz,2014-01-04\n");
+
+    File sameContentScmOnServer = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO);
+    FileUtils.write(sameContentScmOnServer, SAMPLE_XOO_CONTENT);
+    // No need to write .scm file since this file should not be blamed
+
+    File sameContentNoScmOnServer = new File(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO);
+    FileUtils.write(sameContentNoScmOnServer, SAMPLE_XOO_CONTENT);
+    xooScmFile = new File(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO + ".scm");
+    FileUtils.write(xooScmFile,
+      // revision,author,dateTime
+      "1,foo,2013-01-04\n" +
+        "1,bar,2013-01-04\n");
+
+    tester.newTask()
+      .properties(ImmutableMap.<String, String>builder()
+        .put("sonar.task", "scan")
+        .put("sonar.projectBaseDir", baseDir.getAbsolutePath())
+        .put("sonar.projectKey", "com.foo.project")
+        .put("sonar.projectName", "Foo Project")
+        .put("sonar.projectVersion", "1.0-SNAPSHOT")
+        .put("sonar.projectDescription", "Description of Foo Project")
+        .put("sonar.sources", "src")
+        .put("sonar.scm.provider", "xoo")
+        .build())
+      .start();
+
+    assertThat(getChangesets(baseDir, "src/sample.xoo")).isNotNull();
+
+    assertThat(getChangesets(baseDir, CHANGED_CONTENT_SCM_ON_SERVER_XOO)).isNotNull();
+
+    assertThat(getChangesets(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO)).isNull();
+
+    assertThat(getChangesets(baseDir, SAME_CONTENT_NO_SCM_ON_SERVER_XOO)).isNotNull();
+
+    assertThat(logTester.logs()).containsSubsequence("3 files to be analyzed", "3/3 files analyzed");
+    assertThat(logTester.logs()).doesNotContain(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES);
   }
 
   @Test
   public void forceReload() throws IOException {
 
     File baseDir = prepareProject();
-    File xooFileNoScm = new File(baseDir, "src/sample2.xoo");
+    File xooFileNoScm = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO);
     FileUtils.write(xooFileNoScm, SAMPLE_XOO_CONTENT);
-    File xooScmFile = new File(baseDir, "src/sample2.xoo.scm");
+    File xooScmFile = new File(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO + ".scm");
     FileUtils.write(xooScmFile,
       // revision,author,dateTime
       "1,foo,2013-01-04\n" +
-        "1,bar,2013-01-04\n"
-      );
+        "1,bar,2013-01-04\n");
 
     TaskBuilder taskBuilder = tester.newTask()
       .properties(ImmutableMap.<String, String>builder()
@@ -200,10 +266,10 @@ public class ScmMediumTest {
 
     taskBuilder.start();
 
-    BatchReport.Changesets file1Scm = getChangesets(baseDir, 0);
+    BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo");
     assertThat(file1Scm).isNotNull();
 
-    BatchReport.Changesets file2Scm = getChangesets(baseDir, 1);
+    BatchReport.Changesets file2Scm = getChangesets(baseDir, SAME_CONTENT_SCM_ON_SERVER_XOO);
     assertThat(file2Scm).isNotNull();
   }
 
@@ -225,7 +291,7 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets file1Scm = getChangesets(baseDir, 0);
+    BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo");
     assertThat(file1Scm).isNotNull();
   }
 
@@ -247,7 +313,7 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets file1Scm = getChangesets(baseDir, 0);
+    BatchReport.Changesets file1Scm = getChangesets(baseDir, "src/sample.xoo");
     assertThat(file1Scm).isNotNull();
   }
 
@@ -265,8 +331,7 @@ public class ScmMediumTest {
         "2,julien,2013-01-04\n" +
         "3,julien,2013-02-03\n" +
         "3,julien,2013-02-03\n" +
-        "4,simon,2013-03-04\n"
-      );
+        "4,simon,2013-03-04\n");
 
     return baseDir;
   }
@@ -291,7 +356,7 @@ public class ScmMediumTest {
         .build())
       .start();
 
-    BatchReport.Changesets changesets = getChangesets(baseDir, 0);
+    BatchReport.Changesets changesets = getChangesets(baseDir, "src/sample.xoo");
     assertThat(changesets).isNull();
   }