From 505300a3c10164c805c5d8c8e9745f2f1d29fdf0 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Fri, 4 Jan 2019 17:08:00 +0100 Subject: [PATCH] SONAR-11508 Rework FS logging --- ...tractCoverageAndDuplicationExclusions.java | 20 +++--- .../filesystem/AbstractExclusionFilters.java | 18 +++-- ...ojectCoverageAndDuplicationExclusions.java | 1 - .../filesystem/ProjectExclusionFilters.java | 1 - .../scan/filesystem/ProjectFileIndexer.java | 33 +++++---- .../mediumtest/fs/FileSystemMediumTest.java | 68 +++++++++++++++++++ 6 files changed, 105 insertions(+), 36 deletions(-) diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractCoverageAndDuplicationExclusions.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractCoverageAndDuplicationExclusions.java index c061aa5add0..bb8f5fc22f1 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractCoverageAndDuplicationExclusions.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractCoverageAndDuplicationExclusions.java @@ -21,19 +21,20 @@ package org.sonar.scanner.scan.filesystem; import java.util.Collection; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.concurrent.Immutable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.utils.WildcardPattern; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import static java.util.stream.Collectors.toList; @Immutable public abstract class AbstractCoverageAndDuplicationExclusions { - private static final Logger LOG = LoggerFactory.getLogger(AbstractCoverageAndDuplicationExclusions.class); + private static final Logger LOG = Loggers.get(AbstractCoverageAndDuplicationExclusions.class); private final Function pathExtractor; private final String[] coverageExclusionConfig; private final String[] duplicationExclusionConfig; @@ -57,12 +58,12 @@ public abstract class AbstractCoverageAndDuplicationExclusions { return duplicationExclusionConfig; } - void log() { + void log(String indent) { if (!coverageExclusionPatterns.isEmpty()) { - log("Excluded sources for coverage: ", coverageExclusionPatterns); + log("Excluded sources for coverage:", coverageExclusionPatterns, indent); } if (!duplicationExclusionPatterns.isEmpty()) { - log("Excluded sources for duplication: ", duplicationExclusionPatterns); + log("Excluded sources for duplication:", duplicationExclusionPatterns, indent); } } @@ -84,12 +85,9 @@ public abstract class AbstractCoverageAndDuplicationExclusions { .anyMatch(p -> p.match(path)); } - private static void log(String title, Collection patterns) { + private static void log(String title, Collection patterns, String ident) { if (!patterns.isEmpty()) { - LOG.info(title); - for (WildcardPattern pattern : patterns) { - LOG.info(" {}", pattern); - } + LOG.info("{}{} {}", ident, title, patterns.stream().map(WildcardPattern::toString).collect(Collectors.joining(", "))); } } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractExclusionFilters.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractExclusionFilters.java index 0a74fad2261..9a348fee7f5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractExclusionFilters.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/AbstractExclusionFilters.java @@ -22,6 +22,7 @@ package org.sonar.scanner.scan.filesystem; import java.nio.file.Path; import java.util.Arrays; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -55,11 +56,11 @@ public abstract class AbstractExclusionFilters { this.testExclusionsPattern = prepareTestExclusions(testExclusions); } - protected void log() { - log("Included sources: ", mainInclusionsPattern); - log("Excluded sources: ", mainExclusionsPattern); - log("Included tests: ", testInclusionsPattern); - log("Excluded tests: ", testExclusionsPattern); + public void log(String indent) { + log("Included sources:", mainInclusionsPattern, indent); + log("Excluded sources:", mainExclusionsPattern, indent); + log("Included tests:", testInclusionsPattern, indent); + log("Excluded tests:", testExclusionsPattern, indent); } private String[] inclusions(Function configProvider, String propertyKey) { @@ -82,12 +83,9 @@ public abstract class AbstractExclusionFilters { return mainInclusionsPattern.length > 0 || mainExclusionsPattern.length > 0 || testInclusionsPattern.length > 0 || testExclusionsPattern.length > 0; } - private static void log(String title, PathPattern[] patterns) { + private static void log(String title, PathPattern[] patterns, String indent) { if (patterns.length > 0) { - LOG.info(title); - for (PathPattern pattern : patterns) { - LOG.info(" {}", pattern); - } + LOG.info("{}{} {}", indent, title, Arrays.stream(patterns).map(PathPattern::toString).collect(Collectors.joining(", "))); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectCoverageAndDuplicationExclusions.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectCoverageAndDuplicationExclusions.java index f44e5cef36c..1cf496f1310 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectCoverageAndDuplicationExclusions.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectCoverageAndDuplicationExclusions.java @@ -28,6 +28,5 @@ public class ProjectCoverageAndDuplicationExclusions extends AbstractCoverageAnd public ProjectCoverageAndDuplicationExclusions(ProjectConfiguration projectConfig) { super(projectConfig::getStringArray, DefaultInputFile::getProjectRelativePath); - log(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectExclusionFilters.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectExclusionFilters.java index 99bc42a60ff..db1ec72338c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectExclusionFilters.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectExclusionFilters.java @@ -25,7 +25,6 @@ public class ProjectExclusionFilters extends AbstractExclusionFilters { public ProjectExclusionFilters(Configuration projectConfig) { super(projectConfig::getStringArray); - log(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 83d1382ec3e..dfad08dfcf7 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -35,12 +35,12 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.fs.internal.InputModuleHierarchy; import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.scan.ModuleConfiguration; import org.sonar.scanner.scan.ModuleConfigurationProvider; @@ -52,8 +52,9 @@ import org.sonar.scanner.util.ProgressReport; */ public class ProjectFileIndexer { - private static final Logger LOG = LoggerFactory.getLogger(ProjectFileIndexer.class); - private final AbstractExclusionFilters projectExclusionFilters; + private static final Logger LOG = Loggers.get(ProjectFileIndexer.class); + private final ProjectExclusionFilters projectExclusionFilters; + private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; private final InputComponentStore componentStore; private final InputModuleHierarchy inputModuleHierarchy; private final GlobalConfiguration globalConfig; @@ -62,20 +63,24 @@ public class ProjectFileIndexer { private ProgressReport progressReport; - public ProjectFileIndexer(InputComponentStore componentStore, AbstractExclusionFilters exclusionFilters, + public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, ProjectServerSettings projectServerSettings, - FileIndexer fileIndexer) { + FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions) { this.componentStore = componentStore; this.inputModuleHierarchy = inputModuleHierarchy; this.globalConfig = globalConfig; this.projectServerSettings = projectServerSettings; this.fileIndexer = fileIndexer; this.projectExclusionFilters = exclusionFilters; + this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; } public void index() { progressReport = new ProgressReport("Report about progress of file indexation", TimeUnit.SECONDS.toMillis(10)); - progressReport.start("Index files"); + progressReport.start("Indexing files..."); + LOG.info("Project configuration:"); + projectExclusionFilters.log(" "); + projectCoverageAndDuplicationExclusions.log(" "); AtomicInteger excludedByPatternsCount = new AtomicInteger(0); @@ -96,16 +101,18 @@ public class ProjectFileIndexer { } private void index(DefaultInputModule module, AtomicInteger excludedByPatternsCount) { - if (componentStore.allModules().size() > 1) { - LOG.info(" Indexing files from module {}", module.getName()); - LOG.info(" Base dir: {}", module.getBaseDir().toAbsolutePath()); - logPaths(" Source paths: ", module.getBaseDir(), module.getSourceDirsOrFiles()); - logPaths(" Test paths: ", module.getBaseDir(), module.getTestDirsOrFiles()); - } // Emulate creation of module level settings ModuleConfiguration moduleConfig = new ModuleConfigurationProvider().provide(globalConfig, module, projectServerSettings); ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig); ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions = new ModuleCoverageAndDuplicationExclusions(moduleConfig); + if (componentStore.allModules().size() > 1) { + LOG.info("Indexing files of module '{}'", module.getName()); + LOG.info(" Base dir: {}", module.getBaseDir().toAbsolutePath()); + logPaths(" Source paths: ", module.getBaseDir(), module.getSourceDirsOrFiles()); + logPaths(" Test paths: ", module.getBaseDir(), module.getTestDirsOrFiles()); + moduleExclusionFilters.log(" "); + moduleCoverageAndDuplicationExclusions.log(" "); + } indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getSourceDirsOrFiles(), Type.MAIN, excludedByPatternsCount); indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, module.getTestDirsOrFiles(), Type.TEST, excludedByPatternsCount); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java index 1eaec75bde6..d00a5cf3951 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java @@ -886,4 +886,72 @@ public class FileSystemMediumTest { assertThat(result.inputFiles()).hasSize(1); } + @Test + public void log_all_exclusions_properties_per_modules() throws IOException { + File baseDir = temp.getRoot(); + File baseDirModuleA = new File(baseDir, "moduleA"); + File baseDirModuleB = new File(baseDir, "moduleB"); + File srcDirA = new File(baseDirModuleA, "src"); + srcDirA.mkdirs(); + File srcDirB = new File(baseDirModuleB, "src"); + srcDirB.mkdirs(); + + File xooFileA = new File(srcDirA, "sample.xoo"); + FileUtils.write(xooFileA, "Sample xoo\ncontent", StandardCharsets.UTF_8); + + File xooFileB = new File(srcDirB, "sample.xoo"); + FileUtils.write(xooFileB, "Sample xoo\ncontent", StandardCharsets.UTF_8); + + AnalysisResult result = tester.newAnalysis() + .properties(ImmutableMap.builder() + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.sources", "src") + .put("sonar.modules", "moduleA,moduleB") + .put("sonar.inclusions", "**/global.inclusions") + .put("sonar.test.inclusions", "**/global.test.inclusions") + .put("sonar.exclusions", "**/global.exclusions") + .put("sonar.test.exclusions", "**/global.test.exclusions") + .put("sonar.coverage.exclusions", "**/coverage.exclusions") + .put("sonar.cpd.exclusions", "**/cpd.exclusions") + .build()) + .execute(); + + assertThat(logTester.logs(LoggerLevel.INFO)) + .containsSequence("Indexing files...", + "Project configuration:", + " Included sources: **/global.inclusions", + " Excluded sources: **/global.exclusions, **/global.test.inclusions", + " Included tests: **/global.test.inclusions", + " Excluded tests: **/global.test.exclusions", + " Excluded sources for coverage: **/coverage.exclusions", + " Excluded sources for duplication: **/cpd.exclusions", + "Indexing files of module 'moduleA'", + " Base dir: " + baseDirModuleA.getAbsolutePath(), + " Source paths: src", + " Included sources: **/global.inclusions", + " Excluded sources: **/global.exclusions, **/global.test.inclusions", + " Included tests: **/global.test.inclusions", + " Excluded tests: **/global.test.exclusions", + " Excluded sources for coverage: **/coverage.exclusions", + " Excluded sources for duplication: **/cpd.exclusions", + "Indexing files of module 'moduleB'", + " Base dir: " + baseDirModuleB.getAbsolutePath(), + " Source paths: src", + " Included sources: **/global.inclusions", + " Excluded sources: **/global.exclusions, **/global.test.inclusions", + " Included tests: **/global.test.inclusions", + " Excluded tests: **/global.test.exclusions", + " Excluded sources for coverage: **/coverage.exclusions", + " Excluded sources for duplication: **/cpd.exclusions", + "Indexing files of module 'com.foo.project'", + " Base dir: " + baseDir.getAbsolutePath(), + " Included sources: **/global.inclusions", + " Excluded sources: **/global.exclusions, **/global.test.inclusions", + " Included tests: **/global.test.inclusions", + " Excluded tests: **/global.test.exclusions", + " Excluded sources for coverage: **/coverage.exclusions", + " Excluded sources for duplication: **/cpd.exclusions"); + } + } -- 2.39.5