* SONAR-11546 - add time measures to scm logs * SONAR-11546 - add time measure for CPD executor and SCM publisher stepstags/8.0
} | } | ||||
public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache, | public CpdExecutor(CpdSettings settings, SonarCpdBlockIndex index, ReportPublisher publisher, InputComponentStore inputComponentCache, | ||||
ExecutorService executorService) { | |||||
ExecutorService executorService) { | |||||
this.settings = settings; | this.settings = settings; | ||||
this.index = index; | this.index = index; | ||||
this.publisher = publisher; | this.publisher = publisher; | ||||
int filesWithoutBlocks = index.noIndexedFiles() - index.noResources(); | int filesWithoutBlocks = index.noIndexedFiles() - index.noResources(); | ||||
if (filesWithoutBlocks > 0) { | if (filesWithoutBlocks > 0) { | ||||
LOG.info("{} {} had no CPD blocks", filesWithoutBlocks, pluralize(filesWithoutBlocks)); | |||||
LOG.info("CPD Executor {} {} had no CPD blocks", filesWithoutBlocks, pluralize(filesWithoutBlocks)); | |||||
} | } | ||||
total = components.size(); | total = components.size(); | ||||
progressReport.start(String.format("Calculating CPD for %d %s", total, pluralize(total))); | |||||
progressReport.start(String.format("CPD Executor Calculating CPD for %d %s", total, pluralize(total))); | |||||
try { | try { | ||||
for (FileBlocks fileBlocks : components) { | for (FileBlocks fileBlocks : components) { | ||||
runCpdAnalysis(executorService, fileBlocks.getInputFile(), fileBlocks.getBlocks(), timeout); | runCpdAnalysis(executorService, fileBlocks.getInputFile(), fileBlocks.getBlocks(), timeout); | ||||
count++; | count++; | ||||
} | } | ||||
progressReport.stop("CPD calculation finished"); | |||||
progressReport.stopAndLogTotalTime("CPD Executor CPD calculation finished"); | |||||
} catch (Exception e) { | } catch (Exception e) { | ||||
progressReport.stop(""); | progressReport.stop(""); | ||||
throw e; | throw e; |
import javax.annotation.Nullable; | import javax.annotation.Nullable; | ||||
import org.apache.commons.lang.StringUtils; | import org.apache.commons.lang.StringUtils; | ||||
import org.sonar.api.batch.fs.InputFile; | import org.sonar.api.batch.fs.InputFile; | ||||
import org.sonar.api.batch.fs.internal.DefaultInputFile; | |||||
import org.sonar.api.batch.scm.BlameCommand.BlameOutput; | import org.sonar.api.batch.scm.BlameCommand.BlameOutput; | ||||
import org.sonar.api.batch.scm.BlameLine; | import org.sonar.api.batch.scm.BlameLine; | ||||
import org.sonar.api.batch.fs.internal.DefaultInputFile; | |||||
import org.sonar.api.utils.log.Logger; | import org.sonar.api.utils.log.Logger; | ||||
import org.sonar.api.utils.log.Loggers; | import org.sonar.api.utils.log.Loggers; | ||||
import org.sonar.scanner.protocol.output.ScannerReport; | import org.sonar.scanner.protocol.output.ScannerReport; | ||||
count = 0; | count = 0; | ||||
total = filesToBlame.size(); | total = filesToBlame.size(); | ||||
progressReport = new ProgressReport("Report about progress of SCM blame", TimeUnit.SECONDS.toMillis(10)); | progressReport = new ProgressReport("Report about progress of SCM blame", TimeUnit.SECONDS.toMillis(10)); | ||||
progressReport.start(total + " files to be analyzed"); | |||||
progressReport.start("SCM Publisher " + total + " " + pluralize(total) + " to be analyzed"); | |||||
} | } | ||||
@Override | @Override | ||||
writer.writeComponentChangesets(scmBuilder.build()); | writer.writeComponentChangesets(scmBuilder.build()); | ||||
allFilesToBlame.remove(file); | allFilesToBlame.remove(file); | ||||
count++; | count++; | ||||
progressReport.message(count + "/" + total + " files analyzed"); | |||||
progressReport.message(count + "/" + total + " " + pluralize(count) + " have been analyzed"); | |||||
} | } | ||||
private static void validateLine(BlameLine line, int lineId, InputFile file) { | private static void validateLine(BlameLine line, int lineId, InputFile file) { | ||||
} | } | ||||
public void finish(boolean success) { | public void finish(boolean success) { | ||||
progressReport.stop(count + "/" + total + " files analyzed"); | |||||
progressReport.stopAndLogTotalTime("SCM Publisher " + count + "/" + total + " " + pluralize(count) + " have been analyzed"); | |||||
if (success && !allFilesToBlame.isEmpty()) { | if (success && !allFilesToBlame.isEmpty()) { | ||||
LOG.warn("Missing blame information for the following files:"); | LOG.warn("Missing blame information for the following files:"); | ||||
for (InputFile f : allFilesToBlame) { | for (InputFile f : allFilesToBlame) { | ||||
LOG.warn("This may lead to missing/broken features in SonarQube"); | LOG.warn("This may lead to missing/broken features in SonarQube"); | ||||
} | } | ||||
} | } | ||||
private static String pluralize(long filesCount) { | |||||
return filesCount == 1 ? "source file" : "source files"; | |||||
} | |||||
} | } |
import org.sonar.api.batch.fs.FileSystem; | import org.sonar.api.batch.fs.FileSystem; | ||||
import org.sonar.api.batch.fs.InputFile; | import org.sonar.api.batch.fs.InputFile; | ||||
import org.sonar.api.batch.fs.InputFile.Status; | import org.sonar.api.batch.fs.InputFile.Status; | ||||
import org.sonar.api.batch.fs.internal.DefaultInputFile; | |||||
import org.sonar.api.batch.scm.ScmProvider; | import org.sonar.api.batch.scm.ScmProvider; | ||||
import org.sonar.api.utils.log.Logger; | import org.sonar.api.utils.log.Logger; | ||||
import org.sonar.api.utils.log.Loggers; | import org.sonar.api.utils.log.Loggers; | ||||
import org.sonar.api.batch.fs.internal.DefaultInputFile; | |||||
import org.sonar.scanner.protocol.output.ScannerReport; | import org.sonar.scanner.protocol.output.ScannerReport; | ||||
import org.sonar.scanner.protocol.output.ScannerReport.Changesets.Builder; | import org.sonar.scanner.protocol.output.ScannerReport.Changesets.Builder; | ||||
import org.sonar.scanner.protocol.output.ScannerReportWriter; | import org.sonar.scanner.protocol.output.ScannerReportWriter; | ||||
ScmProvider provider = configuration.provider(); | ScmProvider provider = configuration.provider(); | ||||
if (provider == null) { | if (provider == null) { | ||||
LOG.info("No SCM system was detected. You can use the '" + CoreProperties.SCM_PROVIDER_KEY + "' property to explicitly specify it."); | |||||
LOG.info("SCM Publisher No SCM system was detected. You can use the '" + CoreProperties.SCM_PROVIDER_KEY + "' property to explicitly specify it."); | |||||
return; | return; | ||||
} | } | ||||
List<InputFile> filesToBlame = collectFilesToBlame(writer); | List<InputFile> filesToBlame = collectFilesToBlame(writer); | ||||
if (!filesToBlame.isEmpty()) { | if (!filesToBlame.isEmpty()) { | ||||
String key = provider.key(); | String key = provider.key(); | ||||
LOG.info("SCM provider for this project is: " + key); | |||||
LOG.info("SCM Publisher SCM provider for this project is: " + key); | |||||
DefaultBlameOutput output = new DefaultBlameOutput(writer, filesToBlame); | DefaultBlameOutput output = new DefaultBlameOutput(writer, filesToBlame); | ||||
try { | try { | ||||
provider.blameCommand().blame(new DefaultBlameInput(fs, filesToBlame), output); | provider.blameCommand().blame(new DefaultBlameInput(fs, filesToBlame), output); |
private static final Logger LOG = Loggers.get(ProgressReport.class); | private static final Logger LOG = Loggers.get(ProgressReport.class); | ||||
private final long period; | private final long period; | ||||
private long startTime; | |||||
private String message = ""; | private String message = ""; | ||||
private final Thread thread; | private final Thread thread; | ||||
private String stopMessage = null; | private String stopMessage = null; | ||||
public void start(String startMessage) { | public void start(String startMessage) { | ||||
log(startMessage); | log(startMessage); | ||||
thread.start(); | thread.start(); | ||||
startTime = currentTimeMillis(); | |||||
} | } | ||||
public void message(String message) { | public void message(String message) { | ||||
} | } | ||||
} | } | ||||
public void stopAndLogTotalTime(@Nullable String stopMessage) { | |||||
long stopTime = currentTimeMillis(); | |||||
stopMessage += String.format(" (done) | time=%dms", timeDiff(stopTime, startTime)); | |||||
stop(stopMessage); | |||||
} | |||||
private static void log(String message) { | private static void log(String message) { | ||||
synchronized (LOG) { | synchronized (LOG) { | ||||
LOG.info(message); | LOG.info(message); | ||||
} | } | ||||
} | } | ||||
private static long currentTimeMillis() { | |||||
return System.currentTimeMillis(); | |||||
} | |||||
private static long timeDiff(long endTime, long startTime) { | |||||
return endTime - startTime; | |||||
} | |||||
} | } |
import org.sonar.api.batch.fs.InputFile; | import org.sonar.api.batch.fs.InputFile; | ||||
import org.sonar.api.utils.log.LogTester; | import org.sonar.api.utils.log.LogTester; | ||||
import org.sonar.api.utils.log.LoggerLevel; | import org.sonar.api.utils.log.LoggerLevel; | ||||
import org.sonar.scanner.mediumtest.ScannerMediumTester; | |||||
import org.sonar.scanner.mediumtest.AnalysisResult; | import org.sonar.scanner.mediumtest.AnalysisResult; | ||||
import org.sonar.scanner.mediumtest.ScannerMediumTester; | |||||
import org.sonar.scanner.protocol.output.ScannerReport; | import org.sonar.scanner.protocol.output.ScannerReport; | ||||
import org.sonar.xoo.XooPlugin; | import org.sonar.xoo.XooPlugin; | ||||
import org.sonar.xoo.rule.XooRulesDefinition; | import org.sonar.xoo.rule.XooRulesDefinition; | ||||
assertThat(result.inputFiles()).hasSize(2); | assertThat(result.inputFiles()).hasSize(2); | ||||
assertThat(logTester.logs()).contains("Not enough content in 'src/sample2.xoo' to have CPD blocks, it will not be part of the duplication detection"); | assertThat(logTester.logs()).contains("Not enough content in 'src/sample2.xoo' to have CPD blocks, it will not be part of the duplication detection"); | ||||
assertThat(logTester.logs()).contains("1 file had no CPD blocks"); | |||||
assertThat(logTester.logs()).contains("CPD Executor 1 file had no CPD blocks"); | |||||
} | } | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.net.URISyntaxException; | import java.net.URISyntaxException; | ||||
import java.nio.charset.StandardCharsets; | import java.nio.charset.StandardCharsets; | ||||
import java.util.regex.Pattern; | |||||
import org.apache.commons.codec.digest.DigestUtils; | import org.apache.commons.codec.digest.DigestUtils; | ||||
import org.apache.commons.io.FileUtils; | import org.apache.commons.io.FileUtils; | ||||
import org.assertj.core.util.Files; | import org.assertj.core.util.Files; | ||||
ScannerReport.Changesets fileWithoutBlameScm = getChangesets(baseDir, "src/sample_no_blame.xoo"); | ScannerReport.Changesets fileWithoutBlameScm = getChangesets(baseDir, "src/sample_no_blame.xoo"); | ||||
assertThat(fileWithoutBlameScm).isNull(); | assertThat(fileWithoutBlameScm).isNull(); | ||||
assertThat(logTester.logs()).containsSubsequence("2 files to be analyzed", "1/2 files analyzed", MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, | |||||
assertThat(logTester.logs()).containsSubsequence("SCM Publisher 2 source files to be analyzed", MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, | |||||
" * src/sample_no_blame.xoo"); | " * src/sample_no_blame.xoo"); | ||||
assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("SCM Publisher 1/2 source file have been analyzed \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); | |||||
} | } | ||||
// SONAR-6397 | // SONAR-6397 | ||||
// 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total | // 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total | ||||
assertThat(logTester.logs()).containsSubsequence("8 files indexed"); | assertThat(logTester.logs()).containsSubsequence("8 files indexed"); | ||||
assertThat(logTester.logs()).containsSubsequence("4 files to be analyzed", "3/4 files analyzed"); | |||||
assertThat(logTester.logs()).containsSubsequence("SCM Publisher 4 source files to be analyzed"); | |||||
assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("SCM Publisher 3/4 source files have been analyzed \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); | |||||
assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); | assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); | ||||
} | } | ||||
// 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total | // 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total | ||||
assertThat(logTester.logs()).containsSubsequence("8 files indexed"); | assertThat(logTester.logs()).containsSubsequence("8 files indexed"); | ||||
assertThat(logTester.logs()).containsSubsequence("4 files to be analyzed", "3/4 files analyzed"); | |||||
assertThat(logTester.logs()).containsSubsequence("SCM Publisher 4 source files to be analyzed"); | |||||
assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("SCM Publisher 3/4 source files have been analyzed \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); | |||||
assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); | assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); | ||||
} | } | ||||
package org.sonar.scanner.util; | package org.sonar.scanner.util; | ||||
import java.util.Set; | import java.util.Set; | ||||
import java.util.regex.Pattern; | |||||
import org.junit.Rule; | import org.junit.Rule; | ||||
import org.junit.Test; | import org.junit.Test; | ||||
import org.junit.rules.DisableOnDebug; | import org.junit.rules.DisableOnDebug; | ||||
logged = logTester.logs().contains("Some message"); | logged = logTester.logs().contains("Some message"); | ||||
} | } | ||||
underTest.stop("stop"); | underTest.stop("stop"); | ||||
assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("stop", s))).isTrue(); | |||||
} | |||||
@Test | |||||
public void do_log_with_time() { | |||||
underTest.start("start"); | |||||
underTest.stopAndLogTotalTime("stop"); | |||||
assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("stop \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); | |||||
} | } | ||||
private static boolean isDaemon(String name) { | private static boolean isDaemon(String name) { |