import org.sonar.server.computation.task.projectanalysis.event.EventRepositoryImpl;
import org.sonar.server.computation.task.projectanalysis.filemove.FileSimilarityImpl;
import org.sonar.server.computation.task.projectanalysis.filemove.MutableMovedFilesRepositoryImpl;
+import org.sonar.server.computation.task.projectanalysis.filemove.ScoreMatrixDumper;
import org.sonar.server.computation.task.projectanalysis.filemove.SourceSimilarityImpl;
import org.sonar.server.computation.task.projectanalysis.filesystem.ComputationTempFolderProvider;
import org.sonar.server.computation.task.projectanalysis.issue.BaseIssuesLoader;
ShortBranchIssueMerger.class,
// filemove
+ ScoreMatrixDumper.class,
SourceSimilarityImpl.class,
FileSimilarityImpl.class,
MutableMovedFilesRepositoryImpl.class,
*/
package org.sonar.server.computation.task.projectanalysis.filemove;
-import com.google.common.base.Splitter;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.sonar.api.resources.Qualifiers;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
+import org.sonar.core.util.logs.Profiler;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.component.ComponentDto;
import org.sonar.server.computation.task.projectanalysis.source.SourceLinesHashRepository;
import org.sonar.server.computation.task.step.ComputationStep;
-import static com.google.common.base.Splitter.on;
import static com.google.common.collect.FluentIterable.from;
import static java.util.Arrays.asList;
import static org.sonar.server.computation.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER;
protected static final int MIN_REQUIRED_SCORE = 85;
private static final Logger LOG = Loggers.get(FileMoveDetectionStep.class);
private static final List<String> FILE_QUALIFIERS = asList(Qualifiers.FILE, Qualifiers.UNIT_TEST_FILE);
- private static final Splitter LINES_HASHES_SPLITTER = on('\n');
private final AnalysisMetadataHolder analysisMetadataHolder;
private final TreeRootHolder rootHolder;
private final FileSimilarity fileSimilarity;
private final MutableMovedFilesRepository movedFilesRepository;
private final SourceLinesHashRepository sourceLinesHash;
+ private final ScoreMatrixDumper scoreMatrixDumper;
public FileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient,
- FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash) {
+ FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash,
+ ScoreMatrixDumper scoreMatrixDumper) {
this.analysisMetadataHolder = analysisMetadataHolder;
this.rootHolder = rootHolder;
this.dbClient = dbClient;
this.fileSimilarity = fileSimilarity;
this.movedFilesRepository = movedFilesRepository;
this.sourceLinesHash = sourceLinesHash;
+ this.scoreMatrixDumper = scoreMatrixDumper;
}
@Override
LOG.debug("First analysis. Do nothing.");
return;
}
+ Profiler p = Profiler.createIfTrace(LOG);
+ p.start();
Map<String, DbComponent> dbFilesByKey = getDbFilesByKey();
if (dbFilesByKey.isEmpty()) {
LOG.debug("Previous snapshot has no file. Do nothing.");
// retrieve file data from report
Map<String, File> reportFileSourcesByKey = getReportFileSourcesByKey(reportFilesByKey, addedFileKeys);
+ p.stopTrace("loaded");
// compute score matrix
+ p.start();
ScoreMatrix scoreMatrix = computeScoreMatrix(dbFilesByKey, removedFileKeys, reportFileSourcesByKey);
- printIfDebug(scoreMatrix);
+ p.stopTrace("Score matrix computed");
+ scoreMatrixDumper.dumpAsCsv(scoreMatrix);
// not a single match with score higher than MIN_REQUIRED_SCORE => abort
if (scoreMatrix.getMaxScore() < MIN_REQUIRED_SCORE) {
return;
}
+ p.start();
MatchesByScore matchesByScore = MatchesByScore.create(scoreMatrix);
ElectedMatches electedMatches = electMatches(removedFileKeys, reportFileSourcesByKey, matchesByScore);
+ p.stopTrace("Matches elected");
registerMatches(dbFilesByKey, reportFilesByKey, electedMatches);
}
private void registerMatches(Map<String, DbComponent> dbFilesByKey, Map<String, Component> reportFilesByKey, ElectedMatches electedMatches) {
+ LOG.debug("{} files moves found", electedMatches.size());
for (Match validatedMatch : electedMatches) {
movedFilesRepository.setOriginalFile(
reportFilesByKey.get(validatedMatch.getReportKey()),
toOriginalFile(dbFilesByKey.get(validatedMatch.getDbKey())));
- LOG.debug("File move found: {}", validatedMatch);
+ LOG.trace("File move found: {}", validatedMatch);
}
}
return new File(dbComponent.getPath(), fileSourceDto.getLineHashes());
}
- private static void printIfDebug(ScoreMatrix scoreMatrix) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("ScoreMatrix:\n" + scoreMatrix.toCsv(';'));
- }
- }
-
private static ElectedMatches electMatches(Set<String> dbFileKeys, Map<String, File> reportFileSourcesByKey, MatchesByScore matchesByScore) {
ElectedMatches electedMatches = new ElectedMatches(matchesByScore, dbFileKeys, reportFileSourcesByKey);
Multimap<String, Match> matchesPerFileForScore = ArrayListMultimap.create();
public Iterator<Match> iterator() {
return matches.iterator();
}
+
+ public int size() {
+ return matches.size();
+ }
}
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+public interface ScoreMatrixDumper {
+ void dumpAsCsv(ScoreMatrix scoreMatrix);
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.ce.queue.CeTask;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class ScoreMatrixDumperImpl implements ScoreMatrixDumper {
+ private static final Logger LOG = Loggers.get(ScoreMatrixDumperImpl.class);
+
+ private final Configuration configuration;
+ private final CeTask ceTask;
+
+ public ScoreMatrixDumperImpl(Configuration configuration, CeTask ceTask) {
+ this.configuration = configuration;
+ this.ceTask = ceTask;
+ }
+
+ @Override
+ public void dumpAsCsv(ScoreMatrix scoreMatrix) {
+ if (configuration.getBoolean("sonar.filemove.dumpCsv").orElse(false)) {
+ try {
+ Path tempFile = Files.createTempFile(String.format("score-matrix-%s", ceTask.getUuid()), ".csv");
+ try (BufferedWriter writer = Files.newBufferedWriter(tempFile, UTF_8)) {
+ writer.write(scoreMatrix.toCsv(';'));
+ }
+ LOG.info("File move similarity score matrix dumped as CSV: {}", tempFile);
+ } catch (IOException e) {
+ LOG.error("Failed to dump ScoreMatrix as CSV", e);
+ }
+ }
+ }
+}
private SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class);
private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl());
+ private ScoreMatrixDumper scoreMatrixDumper = mock(ScoreMatrixDumper.class);
private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient,
- fileSimilarity, movedFilesRepository, sourceLinesHash);
+ fileSimilarity, movedFilesRepository, sourceLinesHash, scoreMatrixDumper);
@Before
public void setUp() throws Exception {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.computation.task.projectanalysis.filemove;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.AbstractFileFilter;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.ce.queue.CeTask;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(DataProviderRunner.class)
+public class ScoreMatrixDumperImplTest {
+ private static final ScoreMatrix A_SCORE_MATRIX = new ScoreMatrix(
+ ImmutableSet.of("A", "B"),
+ ImmutableMap.of("1", new FileSimilarity.File("path_1", ImmutableList.of("foo", "bar"))),
+ new int[][] {{10}, {2}},
+ 10);
+ private MapSettings settings = new MapSettings();
+ private Configuration configuration = settings.asConfig();
+ private CeTask ceTask = mock(CeTask.class);
+ private ScoreMatrixDumper underTest = new ScoreMatrixDumperImpl(configuration, ceTask);
+ private static Path tempDir;
+
+ @BeforeClass
+ public static void lookupTempDir() throws IOException {
+ Path tempFile = Files.createTempFile("a", "b");
+ Files.delete(tempFile);
+ tempDir = tempFile.getParent();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() {
+ @Override
+ public boolean accept(File file) {
+ if (file.getName().contains("score-matrix-")) {
+ file.delete();
+ }
+ return false;
+ }
+ }, null);
+ }
+
+ @Test
+ public void dumpAsCsv_creates_csv_dump_of_score_matrix_if_property_is_true() throws IOException {
+ String taskUuid = "acme";
+ when(ceTask.getUuid()).thenReturn(taskUuid);
+ settings.setProperty("sonar.filemove.dumpCsv", "true");
+
+ underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+ Collection<File> files = listDumpFilesForTaskUuid(taskUuid);
+ assertThat(files).hasSize(1);
+ assertThat(files.iterator().next()).hasContent(A_SCORE_MATRIX.toCsv(';'));
+ }
+
+ @Test
+ public void dumpAsCsv_has_no_effect_if_configuration_is_empty() throws IOException {
+ String taskUuid = randomAlphabetic(6);
+ when(ceTask.getUuid()).thenReturn(taskUuid);
+
+ underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+ assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty();
+ }
+
+ @Test
+ @UseDataProvider("notTruePropertyValues")
+ public void dumpAsCsv_has_no_effect_if_property_is_not_true(String value) throws IOException {
+ String taskUuid = randomAlphabetic(6);
+ when(ceTask.getUuid()).thenReturn(taskUuid);
+ settings.setProperty("sonar.filemove.dumpCsv", value);
+
+ underTest.dumpAsCsv(A_SCORE_MATRIX);
+
+ assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty();
+ }
+
+ @DataProvider
+ public static Object[][] notTruePropertyValues() {
+ return new Object[][] {
+ {randomAlphabetic(6)},
+ {"false"},
+ };
+ }
+
+ private static Collection<File> listDumpFilesForTaskUuid(String taskUuid) throws IOException {
+ return FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() {
+ @Override
+ public boolean accept(File file) {
+ String name = file.getName();
+ return name.startsWith("score-matrix-" + taskUuid) && name.endsWith(".csv");
+ }
+ }, null);
+ }
+}
projects / sonarqube.git / commitdiff