SONAR-10430 add timing logs + support for dump into temp csv file

dump is enabled by setting property "sonar.filemove.dumpCsv" to "true"

6 files changed, 238 insertions, 13 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
index 800583830a3..e804c25ab68 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java
@@ -49,6 +49,7 @@ import org.sonar.server.computation.task.projectanalysis.duplication.IntegrateCr
 import org.sonar.server.computation.task.projectanalysis.event.EventRepositoryImpl;
 import org.sonar.server.computation.task.projectanalysis.filemove.FileSimilarityImpl;
 import org.sonar.server.computation.task.projectanalysis.filemove.MutableMovedFilesRepositoryImpl;
+import org.sonar.server.computation.task.projectanalysis.filemove.ScoreMatrixDumper;
 import org.sonar.server.computation.task.projectanalysis.filemove.SourceSimilarityImpl;
 import org.sonar.server.computation.task.projectanalysis.filesystem.ComputationTempFolderProvider;
 import org.sonar.server.computation.task.projectanalysis.issue.BaseIssuesLoader;
@@ -268,6 +269,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop
       ShortBranchIssueMerger.class,
 
       // filemove
+      ScoreMatrixDumper.class,
       SourceSimilarityImpl.class,
       FileSimilarityImpl.class,
       MutableMovedFilesRepositoryImpl.class,
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java
index f1a27767fb5..b8a93fe34a1 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java
@@ -19,7 +19,6 @@
  */
 package org.sonar.server.computation.task.projectanalysis.filemove;
 
-import com.google.common.base.Splitter;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -37,6 +36,7 @@ import javax.annotation.concurrent.Immutable;
 import org.sonar.api.resources.Qualifiers;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.logs.Profiler;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.component.ComponentDto;
@@ -53,7 +53,6 @@ import org.sonar.server.computation.task.projectanalysis.filemove.FileSimilarity
 import org.sonar.server.computation.task.projectanalysis.source.SourceLinesHashRepository;
 import org.sonar.server.computation.task.step.ComputationStep;
 
-import static com.google.common.base.Splitter.on;
 import static com.google.common.collect.FluentIterable.from;
 import static java.util.Arrays.asList;
 import static org.sonar.server.computation.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER;
@@ -62,7 +61,6 @@ public class FileMoveDetectionStep implements ComputationStep {
   protected static final int MIN_REQUIRED_SCORE = 85;
   private static final Logger LOG = Loggers.get(FileMoveDetectionStep.class);
   private static final List<String> FILE_QUALIFIERS = asList(Qualifiers.FILE, Qualifiers.UNIT_TEST_FILE);
-  private static final Splitter LINES_HASHES_SPLITTER = on('\n');
 
   private final AnalysisMetadataHolder analysisMetadataHolder;
   private final TreeRootHolder rootHolder;
@@ -70,15 +68,18 @@ public class FileMoveDetectionStep implements ComputationStep {
   private final FileSimilarity fileSimilarity;
   private final MutableMovedFilesRepository movedFilesRepository;
   private final SourceLinesHashRepository sourceLinesHash;
+  private final ScoreMatrixDumper scoreMatrixDumper;
 
   public FileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient,
-    FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash) {
+    FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash,
+    ScoreMatrixDumper scoreMatrixDumper) {
     this.analysisMetadataHolder = analysisMetadataHolder;
     this.rootHolder = rootHolder;
     this.dbClient = dbClient;
     this.fileSimilarity = fileSimilarity;
     this.movedFilesRepository = movedFilesRepository;
     this.sourceLinesHash = sourceLinesHash;
+    this.scoreMatrixDumper = scoreMatrixDumper;
   }
 
   @Override
@@ -93,7 +94,9 @@ public class FileMoveDetectionStep implements ComputationStep {
       LOG.debug("First analysis. Do nothing.");
       return;
     }
+    Profiler p = Profiler.createIfTrace(LOG);
 
+    p.start();
     Map<String, DbComponent> dbFilesByKey = getDbFilesByKey();
     if (dbFilesByKey.isEmpty()) {
       LOG.debug("Previous snapshot has no file. Do nothing.");
@@ -117,10 +120,13 @@ public class FileMoveDetectionStep implements ComputationStep {
 
     // retrieve file data from report
     Map<String, File> reportFileSourcesByKey = getReportFileSourcesByKey(reportFilesByKey, addedFileKeys);
+    p.stopTrace("loaded");
 
     // compute score matrix
+    p.start();
     ScoreMatrix scoreMatrix = computeScoreMatrix(dbFilesByKey, removedFileKeys, reportFileSourcesByKey);
-    printIfDebug(scoreMatrix);
+    p.stopTrace("Score matrix computed");
+    scoreMatrixDumper.dumpAsCsv(scoreMatrix);
 
     // not a single match with score higher than MIN_REQUIRED_SCORE => abort
     if (scoreMatrix.getMaxScore() < MIN_REQUIRED_SCORE) {
@@ -128,19 +134,22 @@ public class FileMoveDetectionStep implements ComputationStep {
       return;
     }
 
+    p.start();
     MatchesByScore matchesByScore = MatchesByScore.create(scoreMatrix);
 
     ElectedMatches electedMatches = electMatches(removedFileKeys, reportFileSourcesByKey, matchesByScore);
+    p.stopTrace("Matches elected");
 
     registerMatches(dbFilesByKey, reportFilesByKey, electedMatches);
   }
 
   private void registerMatches(Map<String, DbComponent> dbFilesByKey, Map<String, Component> reportFilesByKey, ElectedMatches electedMatches) {
+    LOG.debug("{} files moves found", electedMatches.size());
     for (Match validatedMatch : electedMatches) {
       movedFilesRepository.setOriginalFile(
         reportFilesByKey.get(validatedMatch.getReportKey()),
         toOriginalFile(dbFilesByKey.get(validatedMatch.getDbKey())));
-      LOG.debug("File move found: {}", validatedMatch);
+      LOG.trace("File move found: {}", validatedMatch);
     }
   }
 
@@ -224,12 +233,6 @@ public class FileMoveDetectionStep implements ComputationStep {
     return new File(dbComponent.getPath(), fileSourceDto.getLineHashes());
   }
 
-  private static void printIfDebug(ScoreMatrix scoreMatrix) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("ScoreMatrix:\n" + scoreMatrix.toCsv(';'));
-    }
-  }
-
   private static ElectedMatches electMatches(Set<String> dbFileKeys, Map<String, File> reportFileSourcesByKey, MatchesByScore matchesByScore) {
     ElectedMatches electedMatches = new ElectedMatches(matchesByScore, dbFileKeys, reportFileSourcesByKey);
     Multimap<String, Match> matchesPerFileForScore = ArrayListMultimap.create();
@@ -329,5 +332,9 @@ public class FileMoveDetectionStep implements ComputationStep {
     public Iterator<Match> iterator() {
       return matches.iterator();
     }
+
+    public int size() {
+      return matches.size();
+    }
   }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java
new file mode 100644
index 00000000000..811820794bb
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java
@@ -0,0 +1,24 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+public interface ScoreMatrixDumper {
+  void dumpAsCsv(ScoreMatrix scoreMatrix);
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java
new file mode 100644
index 00000000000..5b97eed763d
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java
@@ -0,0 +1,58 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.ce.queue.CeTask;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class ScoreMatrixDumperImpl implements ScoreMatrixDumper {
+  private static final Logger LOG = Loggers.get(ScoreMatrixDumperImpl.class);
+
+  private final Configuration configuration;
+  private final CeTask ceTask;
+
+  public ScoreMatrixDumperImpl(Configuration configuration, CeTask ceTask) {
+    this.configuration = configuration;
+    this.ceTask = ceTask;
+  }
+
+  @Override
+  public void dumpAsCsv(ScoreMatrix scoreMatrix) {
+    if (configuration.getBoolean("sonar.filemove.dumpCsv").orElse(false)) {
+      try {
+        Path tempFile = Files.createTempFile(String.format("score-matrix-%s", ceTask.getUuid()), ".csv");
+        try (BufferedWriter writer = Files.newBufferedWriter(tempFile, UTF_8)) {
+          writer.write(scoreMatrix.toCsv(';'));
+        }
+        LOG.info("File move similarity score matrix dumped as CSV: {}", tempFile);
+      } catch (IOException e) {
+        LOG.error("Failed to dump ScoreMatrix as CSV", e);
+      }
+    }
+  }
+}
diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java
index 0c844c8dbd4..783bbbfc197 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java
@@ -218,9 +218,10 @@ public class FileMoveDetectionStepTest {
 
   private SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class);
   private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl());
+  private ScoreMatrixDumper scoreMatrixDumper = mock(ScoreMatrixDumper.class);
 
   private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient,
-    fileSimilarity, movedFilesRepository, sourceLinesHash);
+    fileSimilarity, movedFilesRepository, sourceLinesHash, scoreMatrixDumper);
 
   @Before
   public void setUp() throws Exception {
diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java
new file mode 100644
index 00000000000..8b6e1396b66
--- /dev/null
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java
@@ -0,0 +1,133 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.AbstractFileFilter;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.ce.queue.CeTask;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(DataProviderRunner.class)
+public class ScoreMatrixDumperImplTest {
+  private static final ScoreMatrix A_SCORE_MATRIX = new ScoreMatrix(
+    ImmutableSet.of("A", "B"),
+    ImmutableMap.of("1", new FileSimilarity.File("path_1", ImmutableList.of("foo", "bar"))),
+    new int[][] {{10}, {2}},
+    10);
+  private MapSettings settings = new MapSettings();
+  private Configuration configuration = settings.asConfig();
+  private CeTask ceTask = mock(CeTask.class);
+  private ScoreMatrixDumper underTest = new ScoreMatrixDumperImpl(configuration, ceTask);
+  private static Path tempDir;
+
+  @BeforeClass
+  public static void lookupTempDir() throws IOException {
+    Path tempFile = Files.createTempFile("a", "b");
+    Files.delete(tempFile);
+    tempDir = tempFile.getParent();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() {
+      @Override
+      public boolean accept(File file) {
+        if (file.getName().contains("score-matrix-")) {
+          file.delete();
+        }
+        return false;
+      }
+    }, null);
+  }
+
+  @Test
+  public void dumpAsCsv_creates_csv_dump_of_score_matrix_if_property_is_true() throws IOException {
+    String taskUuid = "acme";
+    when(ceTask.getUuid()).thenReturn(taskUuid);
+    settings.setProperty("sonar.filemove.dumpCsv", "true");
+
+    underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+    Collection<File> files = listDumpFilesForTaskUuid(taskUuid);
+    assertThat(files).hasSize(1);
+    assertThat(files.iterator().next()).hasContent(A_SCORE_MATRIX.toCsv(';'));
+  }
+
+  @Test
+  public void dumpAsCsv_has_no_effect_if_configuration_is_empty() throws IOException {
+    String taskUuid = randomAlphabetic(6);
+    when(ceTask.getUuid()).thenReturn(taskUuid);
+
+    underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+    assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty();
+  }
+
+  @Test
+  @UseDataProvider("notTruePropertyValues")
+  public void dumpAsCsv_has_no_effect_if_property_is_not_true(String value) throws IOException {
+    String taskUuid = randomAlphabetic(6);
+    when(ceTask.getUuid()).thenReturn(taskUuid);
+    settings.setProperty("sonar.filemove.dumpCsv", value);
+
+    underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+    assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty();
+  }
+
+  @DataProvider
+  public static Object[][] notTruePropertyValues() {
+    return new Object[][] {
+      {randomAlphabetic(6)},
+      {"false"},
+    };
+  }
+
+  private static Collection<File> listDumpFilesForTaskUuid(String taskUuid) throws IOException {
+    return FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() {
+      @Override
+      public boolean accept(File file) {
+        String name = file.getName();
+        return name.startsWith("score-matrix-" + taskUuid) && name.endsWith(".csv");
+      }
+    }, null);
+  }
+}