From: Sébastien Lesaint Date: Wed, 14 Feb 2018 16:50:49 +0000 (+0100) Subject: SONAR-10430 add timing logs + support for dump into temp csv file X-Git-Tag: 7.5~1126 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=d1ca41a7073e6ff6f9481a138f29c0fae38d36de;p=sonarqube.git SONAR-10430 add timing logs + support for dump into temp csv file dump is enabled by setting property "sonar.filemove.dumpCsv" to "true" --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index 800583830a3..e804c25ab68 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -49,6 +49,7 @@ import org.sonar.server.computation.task.projectanalysis.duplication.IntegrateCr import org.sonar.server.computation.task.projectanalysis.event.EventRepositoryImpl; import org.sonar.server.computation.task.projectanalysis.filemove.FileSimilarityImpl; import org.sonar.server.computation.task.projectanalysis.filemove.MutableMovedFilesRepositoryImpl; +import org.sonar.server.computation.task.projectanalysis.filemove.ScoreMatrixDumper; import org.sonar.server.computation.task.projectanalysis.filemove.SourceSimilarityImpl; import org.sonar.server.computation.task.projectanalysis.filesystem.ComputationTempFolderProvider; import org.sonar.server.computation.task.projectanalysis.issue.BaseIssuesLoader; @@ -268,6 +269,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop ShortBranchIssueMerger.class, // filemove + ScoreMatrixDumper.class, SourceSimilarityImpl.class, FileSimilarityImpl.class, MutableMovedFilesRepositoryImpl.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java index f1a27767fb5..b8a93fe34a1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStep.java @@ -19,7 +19,6 @@ */ package org.sonar.server.computation.task.projectanalysis.filemove; -import com.google.common.base.Splitter; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -37,6 +36,7 @@ import javax.annotation.concurrent.Immutable; import org.sonar.api.resources.Qualifiers; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.core.util.logs.Profiler; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; @@ -53,7 +53,6 @@ import org.sonar.server.computation.task.projectanalysis.filemove.FileSimilarity import org.sonar.server.computation.task.projectanalysis.source.SourceLinesHashRepository; import org.sonar.server.computation.task.step.ComputationStep; -import static com.google.common.base.Splitter.on; import static com.google.common.collect.FluentIterable.from; import static java.util.Arrays.asList; import static org.sonar.server.computation.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER; @@ -62,7 +61,6 @@ public class FileMoveDetectionStep implements ComputationStep { protected static final int MIN_REQUIRED_SCORE = 85; private static final Logger LOG = Loggers.get(FileMoveDetectionStep.class); private static final List FILE_QUALIFIERS = asList(Qualifiers.FILE, Qualifiers.UNIT_TEST_FILE); - private static final Splitter LINES_HASHES_SPLITTER = on('\n'); private final AnalysisMetadataHolder analysisMetadataHolder; private final TreeRootHolder rootHolder; @@ -70,15 +68,18 @@ public class FileMoveDetectionStep implements ComputationStep { private final FileSimilarity fileSimilarity; private final MutableMovedFilesRepository movedFilesRepository; private final SourceLinesHashRepository sourceLinesHash; + private final ScoreMatrixDumper scoreMatrixDumper; public FileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient, - FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash) { + FileSimilarity fileSimilarity, MutableMovedFilesRepository movedFilesRepository, SourceLinesHashRepository sourceLinesHash, + ScoreMatrixDumper scoreMatrixDumper) { this.analysisMetadataHolder = analysisMetadataHolder; this.rootHolder = rootHolder; this.dbClient = dbClient; this.fileSimilarity = fileSimilarity; this.movedFilesRepository = movedFilesRepository; this.sourceLinesHash = sourceLinesHash; + this.scoreMatrixDumper = scoreMatrixDumper; } @Override @@ -93,7 +94,9 @@ public class FileMoveDetectionStep implements ComputationStep { LOG.debug("First analysis. Do nothing."); return; } + Profiler p = Profiler.createIfTrace(LOG); + p.start(); Map dbFilesByKey = getDbFilesByKey(); if (dbFilesByKey.isEmpty()) { LOG.debug("Previous snapshot has no file. Do nothing."); @@ -117,10 +120,13 @@ public class FileMoveDetectionStep implements ComputationStep { // retrieve file data from report Map reportFileSourcesByKey = getReportFileSourcesByKey(reportFilesByKey, addedFileKeys); + p.stopTrace("loaded"); // compute score matrix + p.start(); ScoreMatrix scoreMatrix = computeScoreMatrix(dbFilesByKey, removedFileKeys, reportFileSourcesByKey); - printIfDebug(scoreMatrix); + p.stopTrace("Score matrix computed"); + scoreMatrixDumper.dumpAsCsv(scoreMatrix); // not a single match with score higher than MIN_REQUIRED_SCORE => abort if (scoreMatrix.getMaxScore() < MIN_REQUIRED_SCORE) { @@ -128,19 +134,22 @@ public class FileMoveDetectionStep implements ComputationStep { return; } + p.start(); MatchesByScore matchesByScore = MatchesByScore.create(scoreMatrix); ElectedMatches electedMatches = electMatches(removedFileKeys, reportFileSourcesByKey, matchesByScore); + p.stopTrace("Matches elected"); registerMatches(dbFilesByKey, reportFilesByKey, electedMatches); } private void registerMatches(Map dbFilesByKey, Map reportFilesByKey, ElectedMatches electedMatches) { + LOG.debug("{} files moves found", electedMatches.size()); for (Match validatedMatch : electedMatches) { movedFilesRepository.setOriginalFile( reportFilesByKey.get(validatedMatch.getReportKey()), toOriginalFile(dbFilesByKey.get(validatedMatch.getDbKey()))); - LOG.debug("File move found: {}", validatedMatch); + LOG.trace("File move found: {}", validatedMatch); } } @@ -224,12 +233,6 @@ public class FileMoveDetectionStep implements ComputationStep { return new File(dbComponent.getPath(), fileSourceDto.getLineHashes()); } - private static void printIfDebug(ScoreMatrix scoreMatrix) { - if (LOG.isDebugEnabled()) { - LOG.debug("ScoreMatrix:\n" + scoreMatrix.toCsv(';')); - } - } - private static ElectedMatches electMatches(Set dbFileKeys, Map reportFileSourcesByKey, MatchesByScore matchesByScore) { ElectedMatches electedMatches = new ElectedMatches(matchesByScore, dbFileKeys, reportFileSourcesByKey); Multimap matchesPerFileForScore = ArrayListMultimap.create(); @@ -329,5 +332,9 @@ public class FileMoveDetectionStep implements ComputationStep { public Iterator iterator() { return matches.iterator(); } + + public int size() { + return matches.size(); + } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java new file mode 100644 index 00000000000..811820794bb --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumper.java @@ -0,0 +1,24 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.filemove; + +public interface ScoreMatrixDumper { + void dumpAsCsv(ScoreMatrix scoreMatrix); +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java new file mode 100644 index 00000000000..5b97eed763d --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImpl.java @@ -0,0 +1,58 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.filemove; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.sonar.api.config.Configuration; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.ce.queue.CeTask; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class ScoreMatrixDumperImpl implements ScoreMatrixDumper { + private static final Logger LOG = Loggers.get(ScoreMatrixDumperImpl.class); + + private final Configuration configuration; + private final CeTask ceTask; + + public ScoreMatrixDumperImpl(Configuration configuration, CeTask ceTask) { + this.configuration = configuration; + this.ceTask = ceTask; + } + + @Override + public void dumpAsCsv(ScoreMatrix scoreMatrix) { + if (configuration.getBoolean("sonar.filemove.dumpCsv").orElse(false)) { + try { + Path tempFile = Files.createTempFile(String.format("score-matrix-%s", ceTask.getUuid()), ".csv"); + try (BufferedWriter writer = Files.newBufferedWriter(tempFile, UTF_8)) { + writer.write(scoreMatrix.toCsv(';')); + } + LOG.info("File move similarity score matrix dumped as CSV: {}", tempFile); + } catch (IOException e) { + LOG.error("Failed to dump ScoreMatrix as CSV", e); + } + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java index 0c844c8dbd4..783bbbfc197 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/FileMoveDetectionStepTest.java @@ -218,9 +218,10 @@ public class FileMoveDetectionStepTest { private SourceLinesHashRepository sourceLinesHash = mock(SourceLinesHashRepository.class); private FileSimilarity fileSimilarity = new FileSimilarityImpl(new SourceSimilarityImpl()); + private ScoreMatrixDumper scoreMatrixDumper = mock(ScoreMatrixDumper.class); private FileMoveDetectionStep underTest = new FileMoveDetectionStep(analysisMetadataHolder, treeRootHolder, dbClient, - fileSimilarity, movedFilesRepository, sourceLinesHash); + fileSimilarity, movedFilesRepository, sourceLinesHash, scoreMatrixDumper); @Before public void setUp() throws Exception { diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java new file mode 100644 index 00000000000..8b6e1396b66 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/filemove/ScoreMatrixDumperImplTest.java @@ -0,0 +1,133 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.filemove; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.AbstractFileFilter; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.sonar.api.config.Configuration; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.ce.queue.CeTask; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(DataProviderRunner.class) +public class ScoreMatrixDumperImplTest { + private static final ScoreMatrix A_SCORE_MATRIX = new ScoreMatrix( + ImmutableSet.of("A", "B"), + ImmutableMap.of("1", new FileSimilarity.File("path_1", ImmutableList.of("foo", "bar"))), + new int[][] {{10}, {2}}, + 10); + private MapSettings settings = new MapSettings(); + private Configuration configuration = settings.asConfig(); + private CeTask ceTask = mock(CeTask.class); + private ScoreMatrixDumper underTest = new ScoreMatrixDumperImpl(configuration, ceTask); + private static Path tempDir; + + @BeforeClass + public static void lookupTempDir() throws IOException { + Path tempFile = Files.createTempFile("a", "b"); + Files.delete(tempFile); + tempDir = tempFile.getParent(); + } + + @Before + public void setUp() throws Exception { + FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() { + @Override + public boolean accept(File file) { + if (file.getName().contains("score-matrix-")) { + file.delete(); + } + return false; + } + }, null); + } + + @Test + public void dumpAsCsv_creates_csv_dump_of_score_matrix_if_property_is_true() throws IOException { + String taskUuid = "acme"; + when(ceTask.getUuid()).thenReturn(taskUuid); + settings.setProperty("sonar.filemove.dumpCsv", "true"); + + underTest.dumpAsCsv(A_SCORE_MATRIX); + + Collection files = listDumpFilesForTaskUuid(taskUuid); + assertThat(files).hasSize(1); + assertThat(files.iterator().next()).hasContent(A_SCORE_MATRIX.toCsv(';')); + } + + @Test + public void dumpAsCsv_has_no_effect_if_configuration_is_empty() throws IOException { + String taskUuid = randomAlphabetic(6); + when(ceTask.getUuid()).thenReturn(taskUuid); + + underTest.dumpAsCsv(A_SCORE_MATRIX); + + assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty(); + } + + @Test + @UseDataProvider("notTruePropertyValues") + public void dumpAsCsv_has_no_effect_if_property_is_not_true(String value) throws IOException { + String taskUuid = randomAlphabetic(6); + when(ceTask.getUuid()).thenReturn(taskUuid); + settings.setProperty("sonar.filemove.dumpCsv", value); + + underTest.dumpAsCsv(A_SCORE_MATRIX); + + assertThat(listDumpFilesForTaskUuid(taskUuid)).isEmpty(); + } + + @DataProvider + public static Object[][] notTruePropertyValues() { + return new Object[][] { + {randomAlphabetic(6)}, + {"false"}, + }; + } + + private static Collection listDumpFilesForTaskUuid(String taskUuid) throws IOException { + return FileUtils.listFiles(tempDir.toFile(), new AbstractFileFilter() { + @Override + public boolean accept(File file) { + String name = file.getName(); + return name.startsWith("score-matrix-" + taskUuid) && name.endsWith(".csv"); + } + }, null); + } +}