From: Michal Duda Date: Fri, 11 Sep 2020 12:30:31 +0000 (+0200) Subject: SONAR-13867 Collect number of indexed C/C++ files in the scanner X-Git-Tag: 8.5.0.37579~71 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=eb32a549b5e193d244a20504b357dac05e61a58a;p=sonarqube.git SONAR-13867 Collect number of indexed C/C++ files in the scanner --- diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MetadataPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MetadataPublisher.java index 3eb0ed89517..67e6ea79269 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MetadataPublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MetadataPublisher.java @@ -25,7 +25,6 @@ import java.time.Instant; import java.util.LinkedList; import java.util.Map.Entry; import java.util.regex.Pattern; -import javax.annotation.Nullable; import org.sonar.api.batch.fs.internal.AbstractProjectOrModule; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.scm.ScmProvider; @@ -44,6 +43,7 @@ import org.sonar.scanner.rule.QProfile; import org.sonar.scanner.rule.QualityProfiles; import org.sonar.scanner.scan.ScanProperties; import org.sonar.scanner.scan.branch.BranchConfiguration; +import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.scm.ScmRevision; @@ -60,12 +60,12 @@ public class MetadataPublisher implements ReportPublisherStep { private final BranchConfiguration branchConfiguration; private final ScmRevision scmRevision; private final ForkDateSupplier forkDateSupplier; - @Nullable + private final InputComponentStore componentStore; private final ScmConfiguration scmConfiguration; public MetadataPublisher(ProjectInfo projectInfo, InputModuleHierarchy moduleHierarchy, ScanProperties properties, QualityProfiles qProfiles, CpdSettings cpdSettings, ScannerPluginRepository pluginRepository, BranchConfiguration branchConfiguration, - ScmRevision scmRevision, ForkDateSupplier forkDateSupplier, @Nullable ScmConfiguration scmConfiguration) { + ScmRevision scmRevision, ForkDateSupplier forkDateSupplier, InputComponentStore componentStore, ScmConfiguration scmConfiguration) { this.projectInfo = projectInfo; this.moduleHierarchy = moduleHierarchy; this.properties = properties; @@ -75,14 +75,10 @@ public class MetadataPublisher implements ReportPublisherStep { this.branchConfiguration = branchConfiguration; this.scmRevision = scmRevision; this.forkDateSupplier = forkDateSupplier; + this.componentStore = componentStore; this.scmConfiguration = scmConfiguration; } - public MetadataPublisher(ProjectInfo projectInfo, InputModuleHierarchy moduleHierarchy, ScanProperties properties, QualityProfiles qProfiles, - CpdSettings cpdSettings, ScannerPluginRepository pluginRepository, BranchConfiguration branchConfiguration, ScmRevision scmRevision, ForkDateSupplier forkDateSupplier) { - this(projectInfo, moduleHierarchy, properties, qProfiles, cpdSettings, pluginRepository, branchConfiguration, scmRevision, forkDateSupplier, null); - } - @Override public void publish(ScannerReportWriter writer) { AbstractProjectOrModule rootProject = moduleHierarchy.root(); @@ -103,6 +99,7 @@ public class MetadataPublisher implements ReportPublisherStep { addScmInformation(builder); addForkPoint(builder); + addNotAnalyzedFileCountsByLanguage(builder); for (QProfile qp : qProfiles.findAll()) { builder.putQprofilesPerLanguage(qp.getLanguage(), ScannerReport.Metadata.QProfile.newBuilder() @@ -142,16 +139,16 @@ public class MetadataPublisher implements ReportPublisherStep { } } - if (scmConfiguration != null) { - ScmProvider scmProvider = scmConfiguration.provider(); - if (scmProvider != null) { - Path projectBasedir = moduleHierarchy.root().getBaseDir(); - try { - builder.setRelativePathFromScmRoot(toSonarQubePath(scmProvider.relativePathFromScmRoot(projectBasedir))); - } catch (UnsupportedOperationException e) { - LOG.debug(e.getMessage()); - } - } + ScmProvider scmProvider = scmConfiguration.provider(); + if (scmProvider == null) { + return; + } + + Path projectBasedir = moduleHierarchy.root().getBaseDir(); + try { + builder.setRelativePathFromScmRoot(toSonarQubePath(scmProvider.relativePathFromScmRoot(projectBasedir))); + } catch (UnsupportedOperationException e) { + LOG.debug(e.getMessage()); } } @@ -166,6 +163,10 @@ public class MetadataPublisher implements ReportPublisherStep { } } + private void addNotAnalyzedFileCountsByLanguage(ScannerReport.Metadata.Builder builder) { + builder.putAllNotAnalyzedFilesByLanguage(componentStore.getNotAnalysedFilesByLanguage()); + } + private void addBranchInformation(ScannerReport.Metadata.Builder builder) { builder.setBranchName(branchConfiguration.branchName()); BranchType branchType = toProtobufBranchType(branchConfiguration.branchType()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 21435f623ad..ed533e97999 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -30,16 +30,16 @@ import org.sonar.api.CoreProperties; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.InputFileFilter; +import org.sonar.api.batch.fs.internal.DefaultIndexedFile; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.SensorStrategy; import org.sonar.api.batch.scm.IgnoreCommand; import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; -import org.sonar.api.batch.fs.internal.DefaultIndexedFile; -import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.fs.internal.DefaultInputModule; -import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader; import org.sonar.scanner.scan.ScanProperties; import org.sonar.scanner.util.ProgressReport; @@ -52,6 +52,7 @@ import static java.lang.String.format; public class FileIndexer { private static final Logger LOG = Loggers.get(FileIndexer.class); + private final AnalysisWarnings analysisWarnings; private final ScanProperties properties; private final InputFileFilter[] filters; @@ -89,10 +90,11 @@ public class FileIndexer { } public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, - ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, - MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties) { - this(project, scannerComponentIdGenerator, componentStore, projectExclusionFilters, projectCoverageAndDuplicationExclusions, issueExclusionsLoader, metadataGenerator, - sensorStrategy, languageDetection, analysisWarnings, properties, new InputFileFilter[0]); + ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, + IssueExclusionsLoader issueExclusionsLoader, MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, + LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties) { + this(project, scannerComponentIdGenerator, componentStore, projectExclusionFilters, projectCoverageAndDuplicationExclusions, issueExclusionsLoader, + metadataGenerator, sensorStrategy, languageDetection, analysisWarnings, properties, new InputFileFilter[0]); } void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, @@ -269,5 +271,4 @@ public class FileIndexer { private static String pluralizeFiles(int count) { return count == 1 ? "file" : "files"; } - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java index 627da73d5fe..1f757db9bbe 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.scan.filesystem; +import com.google.common.collect.ImmutableMap; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -29,8 +30,11 @@ import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.annotation.CheckForNull; +import org.sonar.api.SonarEdition; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.InputComponent; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; @@ -47,6 +51,9 @@ import static org.sonar.api.utils.Preconditions.checkState; * exclusion patterns are already applied. */ public class InputComponentStore extends DefaultFileSystem.Cache { + private static final Map FILE_PATTERN_BY_LANGUAGE = ImmutableMap.of( + "C", Pattern.compile(".*\\.c", Pattern.CASE_INSENSITIVE), + "C++", Pattern.compile(".*\\.cpp|.*\\.cc|.*\\.cxx|.*\\.c\\+\\+", Pattern.CASE_INSENSITIVE)); private final SortedSet globalLanguagesCache = new TreeSet<>(); private final Map> languagesCache = new HashMap<>(); @@ -58,9 +65,12 @@ public class InputComponentStore extends DefaultFileSystem.Cache { private final Map> filesByNameCache = new HashMap<>(); private final Map> filesByExtensionCache = new HashMap<>(); private final BranchConfiguration branchConfiguration; + private final SonarRuntime sonarRuntime; + private final Map notAnalysedFilesByLanguage = new HashMap<>(); - public InputComponentStore(BranchConfiguration branchConfiguration) { + public InputComponentStore(BranchConfiguration branchConfiguration, SonarRuntime sonarRuntime) { this.branchConfiguration = branchConfiguration; + this.sonarRuntime = sonarRuntime; } public Collection all() { @@ -97,6 +107,8 @@ public class InputComponentStore extends DefaultFileSystem.Cache { public InputComponentStore put(String moduleKey, InputFile inputFile) { DefaultInputFile file = (DefaultInputFile) inputFile; + updateNotAnalysedCAndCppFileCount(file); + addToLanguageCache(moduleKey, file); inputFileByModuleCache.computeIfAbsent(moduleKey, x -> new HashMap<>()).put(file.getModuleRelativePath(), inputFile); inputModuleKeyByFileCache.put(inputFile, moduleKey); @@ -169,4 +181,19 @@ public class InputComponentStore extends DefaultFileSystem.Cache { throw new UnsupportedOperationException(); } + private void updateNotAnalysedCAndCppFileCount(DefaultInputFile inputFile) { + if (!SonarEdition.COMMUNITY.equals(sonarRuntime.getEdition())) { + return; + } + + FILE_PATTERN_BY_LANGUAGE.forEach((language, filePattern) -> { + if (filePattern.matcher(inputFile.filename()).matches()) { + notAnalysedFilesByLanguage.put(language, notAnalysedFilesByLanguage.getOrDefault(language, 0) + 1); + } + }); + } + + public Map getNotAnalysedFilesByLanguage() { + return ImmutableMap.copyOf(notAnalysedFilesByLanguage); + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 5f4da8fae26..e33909d47fb 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -64,7 +64,7 @@ public class ProjectFileIndexer { private static final Logger LOG = Loggers.get(ProjectFileIndexer.class); private final ProjectExclusionFilters projectExclusionFilters; private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions; - private ScmConfiguration scmConfiguration; + private final ScmConfiguration scmConfiguration; private final InputComponentStore componentStore; private final InputModuleHierarchy inputModuleHierarchy; private final GlobalConfiguration globalConfig; @@ -189,14 +189,15 @@ public class ProjectFileIndexer { return count == 1 ? "file" : "files"; } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, - List sources, Type type, ExclusionCounter exclusionCounter) { + private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List sources, Type type, ExclusionCounter exclusionCounter) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter); } else { - fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, ignoreCommand); + fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter, + ignoreCommand); } } } catch (IOException e) { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java index 7598a64e485..f8614ce4fcc 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java @@ -36,6 +36,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentMatchers; +import org.sonar.api.SonarRuntime; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.core.util.CloseableIterator; @@ -72,10 +73,11 @@ public class CpdExecutorTest { public ExpectedException thrown = ExpectedException.none(); private CpdExecutor executor; - private ExecutorService executorService = mock(ExecutorService.class); - private CpdSettings settings = mock(CpdSettings.class); - private ReportPublisher publisher = mock(ReportPublisher.class); - private SonarCpdBlockIndex index = new SonarCpdBlockIndex(publisher, settings); + private final ExecutorService executorService = mock(ExecutorService.class); + private final CpdSettings settings = mock(CpdSettings.class); + private final ReportPublisher publisher = mock(ReportPublisher.class); + private final SonarRuntime sonarRuntime = mock(SonarRuntime.class); + private final SonarCpdBlockIndex index = new SonarCpdBlockIndex(publisher, settings); private ScannerReportReader reader; private DefaultInputFile batchComponent1; private DefaultInputFile batchComponent2; @@ -90,7 +92,7 @@ public class CpdExecutorTest { when(publisher.getWriter()).thenReturn(new ScannerReportWriter(outputDir)); DefaultInputProject project = TestInputFileBuilder.newDefaultInputProject("foo", baseDir); - componentStore = new InputComponentStore(mock(BranchConfiguration.class)); + componentStore = new InputComponentStore(mock(BranchConfiguration.class), sonarRuntime); executor = new CpdExecutor(settings, index, publisher, componentStore, executorService); reader = new ScannerReportReader(outputDir); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java index c1458524e51..74bc606295f 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/ComponentsPublisherTest.java @@ -27,6 +27,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.CoreProperties; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; @@ -49,6 +50,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ComponentsPublisherTest { + + private final SonarRuntime sonarRuntime = mock(SonarRuntime.class); + @Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -77,7 +81,7 @@ public class ComponentsPublisherTest { .setWorkDir(temp.newFolder()); DefaultInputProject project = new DefaultInputProject(rootDef, 1); - InputComponentStore store = new InputComponentStore(branchConfiguration); + InputComponentStore store = new InputComponentStore(branchConfiguration, sonarRuntime); Path moduleBaseDir = temp.newFolder().toPath(); ProjectDefinition module1Def = ProjectDefinition.create() @@ -143,7 +147,7 @@ public class ComponentsPublisherTest { .setWorkDir(temp.newFolder()); DefaultInputProject project = new DefaultInputProject(rootDef, 1); - InputComponentStore store = new InputComponentStore(branchConfiguration); + InputComponentStore store = new InputComponentStore(branchConfiguration, sonarRuntime); DefaultInputFile file = new TestInputFileBuilder("foo", "src/Foo.java", 5) .setLines(2) @@ -180,7 +184,7 @@ public class ComponentsPublisherTest { .setWorkDir(temp.newFolder()); DefaultInputProject project = new DefaultInputProject(rootDef, 1); - InputComponentStore store = new InputComponentStore(branchConfiguration); + InputComponentStore store = new InputComponentStore(branchConfiguration, sonarRuntime); ComponentsPublisher publisher = new ComponentsPublisher(project, store); publisher.publish(writer); @@ -210,7 +214,7 @@ public class ComponentsPublisherTest { .setWorkDir(temp.newFolder()); DefaultInputProject project = new DefaultInputProject(rootDef, 1); - InputComponentStore store = new InputComponentStore(branchConfiguration); + InputComponentStore store = new InputComponentStore(branchConfiguration, sonarRuntime); ComponentsPublisher publisher = new ComponentsPublisher(project, store); publisher.publish(writer); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java index 53d37e99363..f6f176d8cff 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/MetadataPublisherTest.java @@ -56,6 +56,7 @@ import org.sonar.scanner.rule.QualityProfiles; import org.sonar.scanner.scan.ScanProperties; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.branch.BranchType; +import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scanner.scm.ScmRevision; @@ -73,19 +74,18 @@ public class MetadataPublisherTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - private DefaultInputModule rootModule; private MetadataPublisher underTest; - private ScanProperties properties = mock(ScanProperties.class); - private QualityProfiles qProfiles = mock(QualityProfiles.class); - private ProjectInfo projectInfo = mock(ProjectInfo.class); - private CpdSettings cpdSettings = mock(CpdSettings.class); - private InputModuleHierarchy inputModuleHierarchy; - private ForkDateSupplier forkDateSupplier = mock(ForkDateSupplier.class); - private ScannerPluginRepository pluginRepository = mock(ScannerPluginRepository.class); + private final ScanProperties properties = mock(ScanProperties.class); + private final QualityProfiles qProfiles = mock(QualityProfiles.class); + private final ProjectInfo projectInfo = mock(ProjectInfo.class); + private final CpdSettings cpdSettings = mock(CpdSettings.class); + private final ForkDateSupplier forkDateSupplier = mock(ForkDateSupplier.class); + private final ScannerPluginRepository pluginRepository = mock(ScannerPluginRepository.class); private BranchConfiguration branches; private ScmConfiguration scmConfiguration; - private ScmProvider scmProvider = mock(ScmProvider.class); - private ScmRevision scmRevision = mock(ScmRevision.class); + private final ScmProvider scmProvider = mock(ScmProvider.class); + private final ScmRevision scmRevision = mock(ScmRevision.class); + private final InputComponentStore componentStore = mock(InputComponentStore.class); @Before public void prepare() throws IOException { @@ -101,11 +101,11 @@ public class MetadataPublisherTest { Path rootBaseDir = temp.newFolder().toPath(); Path moduleBaseDir = rootBaseDir.resolve("moduleDir"); Files.createDirectory(moduleBaseDir); - rootModule = new DefaultInputModule(def + DefaultInputModule rootModule = new DefaultInputModule(def .setBaseDir(rootBaseDir.toFile()) .setKey("root") .setWorkDir(temp.newFolder()), TestInputFileBuilder.nextBatchId()); - inputModuleHierarchy = mock(InputModuleHierarchy.class); + InputModuleHierarchy inputModuleHierarchy = mock(InputModuleHierarchy.class); when(inputModuleHierarchy.root()).thenReturn(rootModule); DefaultInputModule child = new DefaultInputModule(ProjectDefinition.create() .setKey("module") @@ -118,7 +118,7 @@ public class MetadataPublisherTest { scmConfiguration = mock(ScmConfiguration.class); when(scmConfiguration.provider()).thenReturn(scmProvider); underTest = new MetadataPublisher(projectInfo, inputModuleHierarchy, properties, qProfiles, cpdSettings, - pluginRepository, branches, scmRevision, forkDateSupplier, scmConfiguration); + pluginRepository, branches, scmRevision, forkDateSupplier, componentStore, scmConfiguration); } @Test @@ -141,6 +141,7 @@ public class MetadataPublisherTest { assertThat(metadata.getProjectKey()).isEqualTo("root"); assertThat(metadata.getModulesProjectRelativePathByKeyMap()).containsOnly(entry("module", "modulePath"), entry("root", "")); assertThat(metadata.getProjectVersion()).isEmpty(); + assertThat(metadata.getNotAnalyzedFilesByLanguageCount()).isZero(); assertThat(metadata.getQprofilesPerLanguageMap()).containsOnly(entry("java", org.sonar.scanner.protocol.output.ScannerReport.Metadata.QProfile.newBuilder() .setKey("q1") .setName("Q1") @@ -148,15 +149,29 @@ public class MetadataPublisherTest { .setRulesUpdatedAt(date.getTime()) .build())); assertThat(metadata.getPluginsByKey()).containsOnly(entry("java", org.sonar.scanner.protocol.output.ScannerReport.Metadata.Plugin.newBuilder() - .setKey("java") - .setUpdatedAt(12345) - .build()), + .setKey("java") + .setUpdatedAt(12345) + .build()), entry("php", org.sonar.scanner.protocol.output.ScannerReport.Metadata.Plugin.newBuilder() .setKey("php") .setUpdatedAt(45678) .build())); } + @Test + public void write_not_analysed_file_counts() throws Exception { + when(componentStore.getNotAnalysedFilesByLanguage()).thenReturn(ImmutableMap.of("c", 10, "cpp", 20)); + + File outputDir = temp.newFolder(); + ScannerReportWriter writer = new ScannerReportWriter(outputDir); + + underTest.publish(writer); + + ScannerReportReader reader = new ScannerReportReader(outputDir); + ScannerReport.Metadata metadata = reader.readMetadata(); + assertThat(metadata.getNotAnalyzedFilesByLanguageMap()).contains(entry("c", 10), entry("cpp", 20)); + } + @Test public void write_project_organization() throws Exception { when(properties.organizationKey()).thenReturn(Optional.of("SonarSource")); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java index 4de345c222f..de69cdaea76 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/report/SourcePublisherTest.java @@ -27,6 +27,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; @@ -58,7 +59,7 @@ public class SourcePublisherTest { .build(); DefaultInputProject rootProject = TestInputFileBuilder.newDefaultInputProject(moduleKey, baseDir); - InputComponentStore componentStore = new InputComponentStore(mock(BranchConfiguration.class)); + InputComponentStore componentStore = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); componentStore.put(moduleKey, inputFile); publisher = new SourcePublisher(componentStore); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/rule/QProfileVerifierTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/rule/QProfileVerifierTest.java index c427e5c9ccf..4720972d391 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/rule/QProfileVerifierTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/rule/QProfileVerifierTest.java @@ -24,6 +24,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.slf4j.Logger; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.filesystem.InputComponentStore; @@ -42,7 +43,7 @@ public class QProfileVerifierTest { @Before public void before() { - store = new InputComponentStore(mock(BranchConfiguration.class)); + store = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); profiles = mock(QualityProfiles.class); QProfile javaProfile = new QProfile("p1", "My Java profile", "java", null); when(profiles.findByLanguage("java")).thenReturn(javaProfile); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java index 2a581f0c5ee..b3786854552 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java @@ -21,6 +21,7 @@ package org.sonar.scanner.scan; import java.util.Arrays; import org.junit.Test; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.scan.branch.BranchConfiguration; @@ -35,7 +36,7 @@ public class ModuleIndexerTest { private DefaultInputModuleHierarchy moduleHierarchy; public void createIndexer() { - InputComponentStore componentStore = new InputComponentStore(mock(BranchConfiguration.class)); + InputComponentStore componentStore = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); moduleHierarchy = mock(DefaultInputModuleHierarchy.class); indexer = new ModuleIndexer(componentStore, moduleHierarchy); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java index abfbabcce30..f435cef1532 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java @@ -24,9 +24,12 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.LinkedList; import java.util.List; +import javax.annotation.Nullable; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.SonarEdition; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Status; @@ -38,10 +41,16 @@ import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import org.sonar.scanner.scan.branch.BranchConfiguration; +import static java.util.Optional.ofNullable; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class InputComponentStoreTest { + + private final SonarRuntime sonarRuntime = mock(SonarRuntime.class); + @ClassRule public static TemporaryFolder temp = new TemporaryFolder(); @@ -60,7 +69,7 @@ public class InputComponentStoreTest { DefaultInputProject rootProject = TestInputFileBuilder.newDefaultInputProject(rootDef); DefaultInputModule subModule = TestInputFileBuilder.newDefaultInputModule(moduleDef); - InputComponentStore store = new InputComponentStore(mock(BranchConfiguration.class)); + InputComponentStore store = new InputComponentStore(mock(BranchConfiguration.class), sonarRuntime); store.put(subModule); DefaultInputFile fooFile = new TestInputFileBuilder(rootModuleKey, "src/main/java/Foo.java") @@ -95,13 +104,20 @@ public class InputComponentStoreTest { } static class InputComponentStoreTester extends InputComponentStore { - InputComponentStoreTester() { - super(mock(BranchConfiguration.class)); + InputComponentStoreTester(SonarRuntime sonarRuntime) { + super(mock(BranchConfiguration.class), sonarRuntime); + } + + InputFile addFile(String moduleKey, String relpath, @Nullable String language) { + TestInputFileBuilder fileBuilder = new TestInputFileBuilder(moduleKey, relpath); + ofNullable(language).ifPresent(fileBuilder::setLanguage); + DefaultInputFile file = fileBuilder.build(); + put(moduleKey, file); + return file; } - InputFile addFile(String moduleKey, String relpath, String language) { - DefaultInputFile file = new TestInputFileBuilder(moduleKey, relpath) - .setLanguage(language) + InputFile addFile(String moduleKey, String relPath) { + DefaultInputFile file = new TestInputFileBuilder(moduleKey, relPath) .build(); put(moduleKey, file); return file; @@ -109,8 +125,8 @@ public class InputComponentStoreTest { } @Test - public void should_add_languages_per_module_and_globally() throws IOException { - InputComponentStoreTester tester = new InputComponentStoreTester(); + public void should_add_languages_per_module_and_globally() { + InputComponentStoreTester tester = new InputComponentStoreTester(sonarRuntime); String mod1Key = "mod1"; tester.addFile(mod1Key, "src/main/java/Foo.java", "java"); @@ -124,8 +140,8 @@ public class InputComponentStoreTest { } @Test - public void should_find_files_per_module_and_globally() throws IOException { - InputComponentStoreTester tester = new InputComponentStoreTester(); + public void should_find_files_per_module_and_globally() { + InputComponentStoreTester tester = new InputComponentStoreTester(sonarRuntime); String mod1Key = "mod1"; InputFile mod1File = tester.addFile(mod1Key, "src/main/java/Foo.java", "java"); @@ -137,4 +153,55 @@ public class InputComponentStoreTest { assertThat(tester.filesByModule(mod2Key)).containsExactly(mod2File); assertThat(tester.inputFiles()).containsExactlyInAnyOrder(mod1File, mod2File); } + + @Test + public void stores_not_analysed_c_file_count_in_sq_community_edition() { + when(sonarRuntime.getEdition()).thenReturn(SonarEdition.COMMUNITY); + InputComponentStoreTester underTest = new InputComponentStoreTester(sonarRuntime); + String mod1Key = "mod1"; + underTest.addFile(mod1Key, "src/main/java/Foo.java", "java"); + underTest.addFile(mod1Key, "src/main/c/file1.c"); + underTest.addFile(mod1Key, "src/main/c/file2.c"); + String mod2Key = "mod2"; + underTest.addFile(mod2Key, "src/main/groovy/Foo.groovy", "groovy"); + underTest.addFile(mod2Key, "src/main/c/file3.c"); + + assertThat(underTest.getNotAnalysedFilesByLanguage()).hasSize(1); + assertThat(underTest.getNotAnalysedFilesByLanguage()).containsEntry("C", 3); + } + + @Test + public void stores_not_analysed_cpp_file_count_in_sq_community_edition() { + when(sonarRuntime.getEdition()).thenReturn(SonarEdition.COMMUNITY); + InputComponentStoreTester underTest = new InputComponentStoreTester(sonarRuntime); + String mod1Key = "mod1"; + underTest.addFile(mod1Key, "src/main/java/Foo.java", "java"); + underTest.addFile(mod1Key, "src/main/c/file1.c"); + underTest.addFile(mod1Key, "src/main/c/file2.cpp"); + underTest.addFile(mod1Key, "src/main/c/file3.cxx"); + underTest.addFile(mod1Key, "src/main/c/file4.c++"); + underTest.addFile(mod1Key, "src/main/c/file5.cc"); + underTest.addFile(mod1Key, "src/main/c/file6.CPP"); + String mod2Key = "mod2"; + underTest.addFile(mod2Key, "src/main/groovy/Foo.groovy", "groovy"); + underTest.addFile(mod2Key, "src/main/c/file3.cpp"); + + assertThat(underTest.getNotAnalysedFilesByLanguage()).hasSize(2); + assertThat(underTest.getNotAnalysedFilesByLanguage()).containsEntry("C++", 6); + } + + @Test + public void does_not_store_not_analysed_file_counts_in_sq_non_community_editions() { + when(sonarRuntime.getEdition()).thenReturn(SonarEdition.DEVELOPER); + InputComponentStoreTester underTest = new InputComponentStoreTester(sonarRuntime); + String mod1Key = "mod1"; + underTest.addFile(mod1Key, "src/main/java/Foo.java", "java"); + underTest.addFile(mod1Key, "src/main/java/file1.c"); + underTest.addFile(mod1Key, "src/main/java/file2.c"); + String mod2Key = "mod2"; + underTest.addFile(mod2Key, "src/main/groovy/Foo.groovy", "groovy"); + underTest.addFile(mod2Key, "src/main/groovy/file4.c"); + + assertThat(underTest.getNotAnalysedFilesByLanguage()).isEmpty(); + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java index 0b9e8b2571a..522172db47f 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java @@ -24,6 +24,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.fs.internal.SensorStrategy; @@ -49,7 +50,7 @@ public class ModuleInputComponentStoreTest { @Before public void setUp() throws IOException { DefaultInputProject root = TestInputFileBuilder.newDefaultInputProject(projectKey, temp.newFolder()); - componentStore = new InputComponentStore(mock(BranchConfiguration.class)); + componentStore = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); } @Test diff --git a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java index 7349fa688f8..20b8f682017 100644 --- a/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java +++ b/sonar-scanner-protocol/src/main/java/org/sonar/scanner/protocol/viewer/ScannerReportViewerApp.java @@ -32,9 +32,12 @@ import java.util.Map; import java.util.Scanner; import javax.annotation.CheckForNull; import javax.swing.*; -import javax.swing.UIManager.*; -import javax.swing.event.*; -import javax.swing.tree.*; +import javax.swing.UIManager.LookAndFeelInfo; +import javax.swing.event.TreeSelectionEvent; +import javax.swing.event.TreeSelectionListener; +import javax.swing.tree.DefaultMutableTreeNode; +import javax.swing.tree.DefaultTreeModel; +import javax.swing.tree.TreeSelectionModel; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.sonar.core.util.CloseableIterator; @@ -94,6 +97,8 @@ public class ScannerReportViewerApp { private JEditorPane cpdTextBlocksEditor; private JScrollPane significantCodeTab; private JEditorPane significantCodeEditor; + private JScrollPane metadataTab; + private JEditorPane metadataEditor; /** * Create the application. @@ -187,6 +192,7 @@ public class ScannerReportViewerApp { updateAdHocRules(); updateQualityProfiles(); updatePlugins(); + updateMetadata(); } private void loadComponents() { @@ -348,6 +354,21 @@ public class ScannerReportViewerApp { } } + private void updateMetadata() { + metadataEditor.setText(""); + + StringBuilder builder = new StringBuilder(); + Metadata data = reader.readMetadata(); + builder.append("Project key: ").append(data.getProjectKey()).append("\n"); + builder.append("Project version: ").append(data.getProjectVersion()).append("\n"); + builder.append("Scm revision ID: ").append(data.getScmRevisionId()).append("\n"); + if (data.getNotAnalyzedFilesByLanguageCount() > 0) { + builder.append("Not analyzed files in project:").append("\n"); + data.getNotAnalyzedFilesByLanguageMap().forEach((key, value) -> builder.append(" ").append(key).append(": ").append(value).append("\n")); + } + metadataEditor.setText(builder.toString()); + } + private void updateActiveRules() { activeRuleEditor.setText(""); @@ -588,6 +609,12 @@ public class ScannerReportViewerApp { significantCodeEditor = new JEditorPane(); significantCodeTab.setViewportView(significantCodeEditor); + metadataTab = new JScrollPane(); + tabbedPane.addTab("Metadata", null, metadataTab, null); + + metadataEditor = new JEditorPane(); + metadataTab.setViewportView(metadataEditor); + treeScrollPane = new JScrollPane(); treeScrollPane.setPreferredSize(new Dimension(200, 400)); splitPane.setLeftComponent(treeScrollPane); diff --git a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto index b984bf7abbb..3e3a1f80351 100644 --- a/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto +++ b/sonar-scanner-protocol/src/main/protobuf/scanner_report.proto @@ -58,6 +58,8 @@ message Metadata { int64 forkDate = 19; + map not_analyzed_files_by_language = 20; + message QProfile { string key = 1; string name = 2;