SONAR-5831 Provide the ability to explicitly ask the SCM sensor to reload all blame information

8 files changed, 132 insertions, 31 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
index c62ec99bd1c..42f21fe9104 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
@@ -51,7 +51,7 @@ public class BatchPluginRepository implements PluginRepository {
   private static final Logger LOG = LoggerFactory.getLogger(BatchPluginRepository.class);
   private static final String CORE_PLUGIN = "core";
 
-  private PluginsReferential pluginsReferential;
+  private PluginsRepository pluginsReferential;
   private Map<String, Plugin> pluginsByKey;
   private Map<String, PluginMetadata> metadataByKey;
   private Settings settings;
@@ -59,7 +59,7 @@ public class BatchPluginRepository implements PluginRepository {
   private final DefaultAnalysisMode analysisMode;
   private final BatchPluginJarInstaller pluginInstaller;
 
-  public BatchPluginRepository(PluginsReferential pluginsReferential, Settings settings, DefaultAnalysisMode analysisMode,
+  public BatchPluginRepository(PluginsRepository pluginsReferential, Settings settings, DefaultAnalysisMode analysisMode,
     BatchPluginJarInstaller pluginInstaller) {
     this.pluginsReferential = pluginsReferential;
     this.settings = settings;
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java
index 7d611676d2c..d023b5e75e0 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/DefaultPluginsReferential.java
@@ -37,9 +37,9 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * A {@link PluginsReferential} implementation that put downloaded plugins in a FS cache.
+ * A {@link PluginsRepository} implementation that put downloaded plugins in a FS cache.
  */
-public class DefaultPluginsReferential implements PluginsReferential {
+public class DefaultPluginsReferential implements PluginsRepository {
 
   private static final Logger LOG = LoggerFactory.getLogger(DefaultPluginsReferential.class);
 
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java
index f30a891be30..f55308c348c 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java
@@ -107,7 +107,7 @@ public class GlobalContainer extends ComponentContainer {
       DefaultI18n.class,
       new GlobalRepositoriesProvider(),
       UserRepository.class);
-    if (getComponentByType(PluginsReferential.class) == null) {
+    if (getComponentByType(PluginsRepository.class) == null) {
       add(DefaultPluginsReferential.class);
     }
     if (getComponentByType(GlobalRepositoriesLoader.class) == null) {
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsReferential.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsRepository.java
index 330c8bfb337..58473fd6af1 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsReferential.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginsRepository.java
@@ -31,7 +31,7 @@ import java.util.Map;
  * Plugin referential.
  * @since 4.4
  */
-public interface PluginsReferential {
+public interface PluginsRepository {
 
   /**
    * Return list of remote plugins to be installed
diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
index 8930a759b91..68a58b5df6e 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java
@@ -28,11 +28,12 @@ import org.sonar.api.batch.debt.internal.DefaultDebtModel;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Metric;
 import org.sonar.api.platform.PluginMetadata;
-import org.sonar.batch.bootstrap.PluginsReferential;
+import org.sonar.batch.bootstrap.PluginsRepository;
 import org.sonar.batch.bootstrap.TaskProperties;
 import org.sonar.batch.bootstrapper.Batch;
 import org.sonar.batch.bootstrapper.EnvironmentInformation;
 import org.sonar.batch.protocol.input.ActiveRule;
+import org.sonar.batch.protocol.input.FileData;
 import org.sonar.batch.protocol.input.GlobalRepositories;
 import org.sonar.batch.protocol.input.ProjectRepositories;
 import org.sonar.batch.protocol.input.issues.PreviousIssue;
@@ -68,9 +69,9 @@ public class BatchMediumTester {
   }
 
   public static class BatchMediumTesterBuilder {
-    private final FakeGlobalReferentialsLoader globalRefProvider = new FakeGlobalReferentialsLoader();
-    private final FakeProjectReferentialsLoader projectRefProvider = new FakeProjectReferentialsLoader();
-    private final FakePluginsReferential pluginsReferential = new FakePluginsReferential();
+    private final FakeGlobalRepositoriesLoader globalRefProvider = new FakeGlobalRepositoriesLoader();
+    private final FakeProjectRepositoriesLoader projectRefProvider = new FakeProjectRepositoriesLoader();
+    private final FakePluginsRepository pluginsReferential = new FakePluginsRepository();
     private final FakePreviousIssuesLoader previousIssues = new FakePreviousIssuesLoader();
     private final Map<String, String> bootstrapProperties = new HashMap<String, String>();
 
@@ -121,6 +122,11 @@ public class BatchMediumTester {
       return this;
     }
 
+    public BatchMediumTesterBuilder addFileData(String moduleKey, String path, FileData fileData) {
+      projectRefProvider.addFileData(moduleKey, path, fileData);
+      return this;
+    }
+
     public BatchMediumTesterBuilder addPreviousIssue(PreviousIssue issue) {
       previousIssues.getPreviousIssues().add(issue);
       return this;
@@ -180,9 +186,9 @@ public class BatchMediumTester {
 
     public TaskResult start() {
       TaskResult result = new TaskResult();
-      tester.batch.executeTask(taskProperties,
-        result
-        );
+      Map<String, String> props = new HashMap<>();
+      props.putAll(taskProperties);
+      tester.batch.executeTask(props, result);
       return result;
     }
 
@@ -197,7 +203,7 @@ public class BatchMediumTester {
     }
   }
 
-  private static class FakeGlobalReferentialsLoader implements GlobalRepositoriesLoader {
+  private static class FakeGlobalRepositoriesLoader implements GlobalRepositoriesLoader {
 
     private int metricId = 1;
 
@@ -212,7 +218,7 @@ public class BatchMediumTester {
       return ref.globalSettings();
     }
 
-    public FakeGlobalReferentialsLoader add(Metric metric) {
+    public FakeGlobalRepositoriesLoader add(Metric metric) {
       Boolean optimizedBestValue = metric.isOptimizedBestValue();
       ref.metrics().add(new org.sonar.batch.protocol.input.Metric(metricId,
         metric.key(),
@@ -230,7 +236,7 @@ public class BatchMediumTester {
     }
   }
 
-  private static class FakeProjectReferentialsLoader implements ProjectRepositoriesLoader {
+  private static class FakeProjectRepositoriesLoader implements ProjectRepositoriesLoader {
 
     private ProjectRepositories ref = new ProjectRepositories();
 
@@ -239,18 +245,24 @@ public class BatchMediumTester {
       return ref;
     }
 
-    public FakeProjectReferentialsLoader addQProfile(String language, String name) {
+    public FakeProjectRepositoriesLoader addQProfile(String language, String name) {
       ref.addQProfile(new org.sonar.batch.protocol.input.QProfile(name, name, language, new Date()));
       return this;
     }
 
-    public FakeProjectReferentialsLoader addActiveRule(ActiveRule activeRule) {
+    public FakeProjectRepositoriesLoader addActiveRule(ActiveRule activeRule) {
       ref.addActiveRule(activeRule);
       return this;
     }
+
+    public FakeProjectRepositoriesLoader addFileData(String moduleKey, String path, FileData fileData) {
+      ref.addFileData(moduleKey, path, fileData);
+      return this;
+    }
+
   }
 
-  private static class FakePluginsReferential implements PluginsReferential {
+  private static class FakePluginsRepository implements PluginsRepository {
 
     private List<RemotePlugin> pluginList = new ArrayList<RemotePlugin>();
     private Map<RemotePlugin, File> pluginFiles = new HashMap<RemotePlugin, File>();
@@ -266,14 +278,14 @@ public class BatchMediumTester {
       return pluginFiles.get(remote);
     }
 
-    public FakePluginsReferential addPlugin(String pluginKey, File location) {
+    public FakePluginsRepository addPlugin(String pluginKey, File location) {
       RemotePlugin plugin = new RemotePlugin(pluginKey, false);
       pluginList.add(plugin);
       pluginFiles.put(plugin, location);
       return this;
     }
 
-    public FakePluginsReferential addPlugin(String pluginKey, SonarPlugin pluginInstance) {
+    public FakePluginsRepository addPlugin(String pluginKey, SonarPlugin pluginInstance) {
       localPlugins.put(DefaultPluginMetadata.create(pluginKey), pluginInstance);
       return this;
     }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
index d95f870b5ac..e81e4d5f6cc 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmConfiguration.java
@@ -25,6 +25,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.CoreProperties;
+import org.sonar.api.Properties;
+import org.sonar.api.Property;
+import org.sonar.api.PropertyType;
 import org.sonar.api.batch.AnalysisMode;
 import org.sonar.api.batch.InstantiationStrategy;
 import org.sonar.api.batch.bootstrap.ProjectReactor;
@@ -37,10 +40,25 @@ import javax.annotation.Nullable;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+@Properties({
+  @Property(
+    key = ScmConfiguration.FORCE_RELOAD_KEY,
+    defaultValue = "false",
+    type = PropertyType.BOOLEAN,
+    name = "Force reloading of SCM information for all files",
+    description = "By default only files modified since previous analysis are inspected. Set this parameter to true to force the reloading.",
+    module = false,
+    project = false,
+    global = false,
+    category = CoreProperties.CATEGORY_SCM
+  )
+})
 @InstantiationStrategy(InstantiationStrategy.PER_BATCH)
 public final class ScmConfiguration implements BatchComponent, Startable {
   private static final Logger LOG = LoggerFactory.getLogger(ScmConfiguration.class);
 
+  public static final String FORCE_RELOAD_KEY = "sonar.scm.forceReloadAll";
+
   private final ProjectReactor projectReactor;
   private final Settings settings;
   private final Map<String, ScmProvider> providerPerKey = new LinkedHashMap<String, ScmProvider>();
@@ -140,6 +158,10 @@ public final class ScmConfiguration implements BatchComponent, Startable {
     return settings.getBoolean(CoreProperties.SCM_DISABLED_KEY);
   }
 
+  public boolean forceReloadAll() {
+    return settings.getBoolean(FORCE_RELOAD_KEY);
+  }
+
   @Override
   public void stop() {
     // Nothing to do
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
index 6e4207d410b..2a33ed3b3fb 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scm/ScmSensor.java
@@ -73,10 +73,7 @@ public final class ScmSensor implements Sensor {
       return;
     }
 
-    List<InputFile> filesToBlame = new LinkedList<InputFile>();
-    for (InputFile f : fs.inputFiles(fs.predicates().all())) {
-      copyPreviousMeasuresForUnmodifiedFiles(context, filesToBlame, f);
-    }
+    List<InputFile> filesToBlame = collectFilesToBlame(context);
     if (!filesToBlame.isEmpty()) {
       LOG.info("SCM provider for this project is: " + configuration.provider().key());
       TimeProfiler profiler = new TimeProfiler().start("Retrieve SCM blame information");
@@ -87,6 +84,21 @@ public final class ScmSensor implements Sensor {
     }
   }
 
+  private List<InputFile> collectFilesToBlame(final SensorContext context) {
+    if (configuration.forceReloadAll()) {
+      LOG.warn("Forced reloading of SCM data for all files.");
+    }
+    List<InputFile> filesToBlame = new LinkedList<InputFile>();
+    for (InputFile f : fs.inputFiles(fs.predicates().all())) {
+      if (!configuration.forceReloadAll()) {
+        copyPreviousMeasuresForUnmodifiedFiles(context, filesToBlame, f);
+      } else {
+        filesToBlame.add(f);
+      }
+    }
+    return filesToBlame;
+  }
+
   private void copyPreviousMeasuresForUnmodifiedFiles(final SensorContext context, List<InputFile> filesToBlame, InputFile f) {
     FileData fileData = projectReferentials.fileData(projectDefinition.getKeyWithBranch(), f.relativePath());
 
diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
index 13994e0081c..fb9601f182d 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/scm/ScmMediumTest.java
@@ -20,6 +20,7 @@
 package org.sonar.batch.mediumtest.scm;
 
 import com.google.common.collect.ImmutableMap;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -31,7 +32,9 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.batch.mediumtest.BatchMediumTester;
+import org.sonar.batch.mediumtest.BatchMediumTester.TaskBuilder;
 import org.sonar.batch.mediumtest.TaskResult;
+import org.sonar.batch.protocol.input.FileData;
 import org.sonar.xoo.XooPlugin;
 
 import java.io.File;
@@ -41,6 +44,8 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 public class ScmMediumTest {
 
+  private static final String SAMPLE_XOO_CONTENT = "Sample xoo\ncontent";
+
   @org.junit.Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
@@ -50,6 +55,7 @@ public class ScmMediumTest {
   public BatchMediumTester tester = BatchMediumTester.builder()
     .registerPlugin("xoo", new XooPlugin())
     .addDefaultQProfile("xoo", "Sonar Way")
+    .addFileData("com.foo.project", "src/sample2.xoo", new FileData(DigestUtils.md5Hex(SAMPLE_XOO_CONTENT), false, "1=;2=", "1=;2=", "1=;2="))
     .build();
 
   @Before
@@ -145,7 +151,57 @@ public class ScmMediumTest {
         .put("sonar.scm.provider", "xoo")
         .build())
       .start();
+  }
+
+  @Test
+  public void copyPreviousMeasuresOrForceReload() throws IOException {
+
+    File baseDir = prepareProject();
+    File xooFileNoScm = new File(baseDir, "src/sample2.xoo");
+    FileUtils.write(xooFileNoScm, SAMPLE_XOO_CONTENT);
+
+    TaskBuilder taskBuilder = tester.newTask()
+      .properties(ImmutableMap.<String, String>builder()
+        .put("sonar.task", "scan")
+        .put("sonar.projectBaseDir", baseDir.getAbsolutePath())
+        .put("sonar.projectKey", "com.foo.project")
+        .put("sonar.projectName", "Foo Project")
+        .put("sonar.projectVersion", "1.0-SNAPSHOT")
+        .put("sonar.projectDescription", "Description of Foo Project")
+        .put("sonar.sources", "src")
+        .put("sonar.scm.provider", "xoo")
+        .build());
+
+    TaskResult result = taskBuilder.start();
+
+    assertThat(result.measures()).hasSize(1 + 2 * 4);
 
+    assertThat(result.measures()).contains(new DefaultMeasure<Integer>()
+      .forMetric(CoreMetrics.LINES)
+      .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo"))
+      .withValue(2));
+
+    assertThat(result.measures()).contains(new DefaultMeasure<String>()
+      .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE)
+      .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo"))
+      .withValue("1=;2="));
+
+    // Force reload
+    File xooScmFile = new File(baseDir, "src/sample2.xoo.scm");
+    FileUtils.write(xooScmFile,
+      // revision,author,dateTime
+      "1,foo,2013-01-04\n" +
+        "1,bar,2013-01-04\n"
+      );
+
+    result = taskBuilder
+      .property("sonar.scm.forceReloadAll", "true")
+      .start();
+
+    assertThat(result.measures()).contains(new DefaultMeasure<String>()
+      .forMetric(CoreMetrics.SCM_AUTHORS_BY_LINE)
+      .onFile(new DefaultInputFile("com.foo.project", "src/sample2.xoo"))
+      .withValue("1=foo;2=bar"));
   }
 
   @Test
@@ -215,12 +271,10 @@ public class ScmMediumTest {
     File srcDir = new File(baseDir, "src");
     srcDir.mkdir();
 
-    File xooFile = new File(srcDir, "sample.xoo");
-    File xooMeasureFile = new File(srcDir, "sample.xoo.measures");
-    File xooScmFile = new File(srcDir, "sample.xoo.scm");
-    FileUtils.write(xooFile, "Sample xoo\ncontent\n3\n4\n5");
-    FileUtils.write(xooMeasureFile, "lines:5");
-    FileUtils.write(xooScmFile,
+    File xooFile1 = new File(srcDir, "sample.xoo");
+    FileUtils.write(xooFile1, "Sample xoo\ncontent\n3\n4\n5");
+    File xooScmFile1 = new File(srcDir, "sample.xoo.scm");
+    FileUtils.write(xooScmFile1,
       // revision,author,dateTime
       "1,,2013-01-04\n" +
         "1,julien,2013-01-04\n" +
@@ -228,6 +282,7 @@ public class ScmMediumTest {
         "2,julien,2013-02-03\n" +
         "3,simon,2013-03-04\n"
       );
+
     return baseDir;
   }