diff options
Diffstat (limited to 'sonar-scanner-engine')
65 files changed, 1903 insertions, 645 deletions
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java index 7f56cb87cce..94ccedfa946 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java @@ -63,6 +63,6 @@ public class AnalysisDataIT { // then List<ScannerReport.AnalysisData> analysisData = result.analysisData(); assertThat(analysisData) - .anyMatch(data -> data.getKey().equals("architecture.file_graph." + Xoo.KEY)); + .anyMatch(data -> data.getKey().startsWith("architecture.graph.")); } } diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java index 63317e362b9..6cfd6ea5f94 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java @@ -117,7 +117,7 @@ class BootstrapMediumIT { void should_fail_if_invalid_json_input() { var in = new ByteArrayInputStream("}".getBytes()); - var exitCode = ScannerMain.run(in); + var exitCode = ScannerMain.run(in, System.out); assertThat(exitCode).isEqualTo(1); assertThat(logTester.getLogs(Level.ERROR)).hasSize(1); @@ -128,7 +128,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_key() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes())); + {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null key. Value='aValueWithoutKey'"); } @@ -136,7 +136,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -144,7 +144,7 @@ class BootstrapMediumIT { @Test void should_warn_if_not_provided_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -152,7 +152,7 @@ class BootstrapMediumIT { @Test void should_warn_if_duplicate_property_keys() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Duplicated properties. Key='aKey'"); } @@ -160,7 +160,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -168,7 +168,7 @@ class BootstrapMediumIT { @Test void should_warn_if_empty_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{}]}""".getBytes())); + {"scannerProperties": [{}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -229,7 +229,7 @@ class BootstrapMediumIT { } private int runScannerEngine(ScannerProperties scannerProperties) { - return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes())); + return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes()), System.out); } static class ScannerProperties { diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index 0a1ecff943b..babeadf518e 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -28,14 +28,21 @@ import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; +import java.util.Map; import java.util.Random; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.event.Level; import org.sonar.api.CoreProperties; import org.sonar.api.SonarEdition; @@ -49,6 +56,7 @@ import org.sonar.api.utils.System2; import org.sonar.scanner.mediumtest.AnalysisResult; import org.sonar.scanner.mediumtest.ScannerMediumTester; import org.sonar.scanner.mediumtest.ScannerMediumTester.AnalysisBuilder; +import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.xoo.XooPlugin; import org.sonar.xoo.global.DeprecatedGlobalSensor; import org.sonar.xoo.global.GlobalProjectSensor; @@ -59,7 +67,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.tuple; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowableOfType; -import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.slf4j.event.Level.DEBUG; @@ -150,7 +157,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("2 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("2 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(String.join("\n", logTester.logs())).doesNotContain("'src/sample.java' generated metadata"); } @@ -173,7 +180,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("2 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("2 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(logTester.logs()).contains("'src/sample.java' generated metadata with charset 'UTF-8'"); } @@ -200,7 +207,7 @@ class FileSystemMediumIT { .execute(); assertThat(logTester.logs()).containsAnyOf("'src/main/sample.java' indexed with no language", "'src\\main\\sample.java' indexed with no language"); - assertThat(logTester.logs()).contains("3 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("3 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/main/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(logTester.logs()).doesNotContain("'src/main/sample.java' generated metadata", "'src\\main\\sample.java' generated metadata"); assertThat(logTester.logs()).doesNotContain("'src/test/sample.java' generated metadata", "'src\\test\\sample.java' generated metadata"); @@ -231,7 +238,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("1 file indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("1 file indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.unknown' indexed with no language"); assertThat(logTester.logs()).contains("'src/sample.unknown' generated metadata with charset 'UTF-8'"); DefaultInputFile inputFile = (DefaultInputFile) result.inputFile("src/sample.unknown"); @@ -456,6 +463,7 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisDoesNotFailOnBrokenSymlink() throws IOException { prepareBrokenSymlinkTestScenario(); @@ -465,6 +473,7 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisWarnsAndIgnoresBrokenSymlink() throws IOException { Path link = prepareBrokenSymlinkTestScenario(); @@ -477,8 +486,8 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisIgnoresSymbolicLinkWithTargetOutsideBaseDir() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); File srcDir = new File(baseDir, "src"); assertThat(srcDir.mkdir()).isTrue(); @@ -497,23 +506,34 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) + void analysisIgnoresSymbolicLinkWithRelativeTargetOutsideBaseDir() throws IOException { + File srcDir = new File(baseDir, "src"); + assertThat(srcDir.mkdir()).isTrue(); + + File otherDir = createDirectory(temp.toPath().resolve("other_dir")).toFile(); + writeFile(otherDir, "target_outside.xoo"); + + Path linkPath = srcDir.toPath().resolve("target_link"); + Path link = Files.createSymbolicLink(linkPath, Paths.get("../../other_dir/target_outside.xoo")); + + tester.newAnalysis().properties(builder.build()).execute(); + + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file not located in project basedir.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.WARN)).contains(logMessage); + } + + @Test + @DisabledOnOs(OS.WINDOWS) void analysisIgnoresSymbolicLinkWithTargetOutsideModule() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); - File baseDirModuleA = new File(baseDir, "module_a"); - File baseDirModuleB = new File(baseDir, "module_b"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("module_a", "src"); + File srcDirB = createModuleWithSubdirectory("module_b", "src"); File target = writeFile(srcDirA, "target.xoo", "Sample xoo\ncontent"); Path link = Paths.get(srcDirB.getPath(), "target_link.xoo"); Files.createSymbolicLink(link, target.toPath()); - builder = ImmutableMap.<String, String>builder() - .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) - .put("sonar.projectKey", "com.foo.project") - .put("sonar.modules", "module_a,module_b"); + builder.put("sonar.modules", "module_a,module_b"); AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); @@ -524,13 +544,49 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) + void analysisIgnoresSymbolicLinkWithRelativeTargetOutsideModule() throws IOException { + File srcA = createModuleWithSubdirectory("module_a", "src"); + File srcB = createModuleWithSubdirectory("module_b", "src"); + + Path target = srcB.toPath().resolve("target.xoo"); + FileUtils.write(target.toFile(), "Sample xoo\ncontent", StandardCharsets.UTF_8); + Path link = srcA.toPath().resolve("target_link"); + Files.createSymbolicLink(link, Paths.get("../../module_b/src/target.xoo")); + + builder.put("sonar.modules", "module_a,module_b"); + + AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); + + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file not located in module basedir.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.INFO)).contains(logMessage); + InputFile fileA = result.inputFile("module_b/src/target.xoo"); + assertThat(fileA).isNotNull(); + } + + @Test + @DisabledOnOs(OS.WINDOWS) + void analysisDoesNotIgnoreSymbolicLinkWithRelativePath() throws IOException { + File src = createModuleWithSubdirectory("module_a", "src"); + Path target = src.toPath().resolve("target.xoo"); + FileUtils.write(target.toFile(), "Sample xoo\ncontent", StandardCharsets.UTF_8); + Path link = src.toPath().resolve("target_link"); + Files.createSymbolicLink(link, Paths.get("target.xoo")); + + builder.put("sonar.modules", "module_a"); + + AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); + + InputFile targetFile = result.inputFile("module_a/src/target.xoo"); + assertThat(targetFile).isNotNull(); + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file that does not exist.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.WARN)).doesNotContain(logMessage); + } + + @Test void test_inclusions_on_multi_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "tests"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "tests"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "tests"); + File srcDirB = createModuleWithSubdirectory("moduleB", "tests"); writeFile(srcDirA, "sampleTestA.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sampleTestB.xoo", "Sample xoo\ncontent"); @@ -571,12 +627,8 @@ class FileSystemMediumIT { @Test void test_module_level_inclusions_override_parent_on_multi_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sampleA.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sampleB.xoo", "Sample xoo\ncontent"); @@ -608,12 +660,8 @@ class FileSystemMediumIT { @Test void warn_user_for_outdated_scanner_side_inherited_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -641,12 +689,8 @@ class FileSystemMediumIT { @Test void support_global_server_side_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -671,12 +715,8 @@ class FileSystemMediumIT { @Test void support_global_server_side_global_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -701,12 +741,8 @@ class FileSystemMediumIT { @Test void warn_user_for_outdated_server_side_inherited_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -979,12 +1015,8 @@ class FileSystemMediumIT { @Test void log_all_exclusions_properties_per_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -1013,7 +1045,7 @@ class FileSystemMediumIT { " Excluded sources for coverage: **/coverage.exclusions", " Excluded sources for duplication: **/cpd.exclusions", "Indexing files of module 'moduleA'", - " Base dir: " + baseDirModuleA.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS), + " Base dir: " + srcDirA.toPath().getParent().toRealPath(LinkOption.NOFOLLOW_LINKS), " Included sources: **/global.inclusions", " Excluded sources: **/global.exclusions, **/global.test.inclusions", " Included tests: **/global.test.inclusions", @@ -1021,7 +1053,7 @@ class FileSystemMediumIT { " Excluded sources for coverage: **/coverage.exclusions", " Excluded sources for duplication: **/cpd.exclusions", "Indexing files of module 'moduleB'", - " Base dir: " + baseDirModuleB.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS), + " Base dir: " + srcDirB.toPath().getParent().toRealPath(LinkOption.NOFOLLOW_LINKS), " Included sources: **/global.inclusions", " Excluded sources: **/global.exclusions, **/global.test.inclusions", " Included tests: **/global.test.inclusions", @@ -1147,7 +1179,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("1 file indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("1 file indexed (done) | time=")); assertThat(result.inputFile("sample.xoo")).isNotNull(); } @@ -1314,6 +1346,200 @@ class FileSystemMediumIT { .hasMessageEndingWith("Failed to preprocess files"); } + @ParameterizedTest + @ValueSource(booleans = { + true, + false + }) + void shouldScanAndAnalyzeAllHiddenFiles(boolean setHiddenFileScanningExplicitly) throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisBuilder analysisBuilder = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true"); + + if (setHiddenFileScanningExplicitly) { + // default is assumed to be false, here we set it explicitly + analysisBuilder.property("sonar.scanner.excludeHiddenFiles", "false"); + } + + AnalysisResult result = analysisBuilder.execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedIsHidden = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedIsHidden, true); + // we expect the sensor to process all files, regardless of visibility + assertFileIssue(result, filePath, true); + } + assertThat(result.inputFiles()).hasSize(10); + } + + @Test + void shouldScanAllFilesAndOnlyAnalyzeNonHiddenFiles() throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "false") + .execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedHiddenStatus = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedHiddenStatus, true); + // sensor should not process hidden files, we only expect issues on non-hidden files + assertFileIssue(result, filePath, !expectedHiddenStatus); + } + assertThat(result.inputFiles()).hasSize(10); + } + + @ParameterizedTest + @ValueSource(booleans = { + true, + false + }) + void shouldNotScanAndAnalyzeHiddenFilesWhenHiddenFileScanningIsDisabled(boolean sensorHiddenFileProcessingEnabled) throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.scanner.excludeHiddenFiles", "true") + // hidden files are not scanned, so issues can't be raised on them regardless if the sensor wants to process them + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", String.valueOf(sensorHiddenFileProcessingEnabled)) + .execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedHiddenStatus = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedHiddenStatus, false); + if (!expectedHiddenStatus) { + assertFileIssue(result, filePath, true); + } + } + assertThat(result.inputFiles()).hasSize(1); + } + + @Test + void hiddenFilesAssignedToALanguageShouldNotBePublishedByDefault() throws IOException { + tester + .addRules(new XooRulesDefinition()); + + File srcDir = new File(baseDir, "src"); + assertThat(srcDir.mkdir()).isTrue(); + + File hiddenFile = writeFile(srcDir, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(hiddenFile.toPath()); + File hiddenFileWithoutLanguage = writeFile(srcDir, ".bar", "Sample bar\ncontent"); + setFileAsHiddenOnWindows(hiddenFileWithoutLanguage.toPath()); + writeFile(srcDir, "file.xoo", "Sample xoo\ncontent"); + + AnalysisResult result = tester.newAnalysis() + .properties(builder + .put("sonar.sources", "src") + .build()) + .execute(); + + DefaultInputFile hiddenInputFile = (DefaultInputFile) result.inputFile("src/.xoo"); + + assertThat(hiddenInputFile).isNotNull(); + assertThat(hiddenInputFile.isPublished()).isFalse(); + assertThatThrownBy(() -> result.getReportComponent(hiddenInputFile)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unable to find report for component"); + + DefaultInputFile hiddenInputFileWithoutLanguage = (DefaultInputFile) result.inputFile("src/.bar"); + assertThat(hiddenInputFileWithoutLanguage).isNotNull(); + assertThat(hiddenInputFileWithoutLanguage.isPublished()).isFalse(); + assertThatThrownBy(() -> result.getReportComponent(hiddenInputFileWithoutLanguage)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unable to find report for component"); + + DefaultInputFile visibleInputFile = (DefaultInputFile) result.inputFile("src/file.xoo"); + assertThat(visibleInputFile).isNotNull(); + assertThat(visibleInputFile.isPublished()).isTrue(); + assertThat(result.getReportComponent(visibleInputFile)).isNotNull(); + } + + @Test + void shouldDetectHiddenFilesFromMultipleModules() throws IOException { + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); + + File fileModuleA = writeFile(srcDirA, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(fileModuleA.toPath()); + File fileModuleB = writeFile(srcDirB, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(fileModuleB.toPath()); + + AnalysisResult result = tester.newAnalysis() + .properties(ImmutableMap.<String, String>builder() + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.sources", "src") + .put("sonar.modules", "moduleA,moduleB") + .build()) + .execute(); + + assertHiddenFileScan(result, "moduleA/src/.xoo", true, true); + assertHiddenFileScan(result, "moduleB/src/.xoo", true, true); + } + + @Test + void shouldScanAndAnalyzeAllHiddenFilesWithRespectToExclusions() throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + + + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.scm.provider", "xoo") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true") + .property("sonar.exclusions", "**/.nestedHidden/**,**/*.ignore") + .execute(); + + Set<String> excludedFiles = Set.of( + // sonar.exclusions + "xources/.hidden/.nestedHidden/.xoo", + "xources/.hidden/.nestedHidden/Class.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo.ignore", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo", + // scm ignore + "xources/nonHidden/.hiddenInVisibleFolder/.xoo"); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedIsHidden = pathToHiddenStatus.getValue(); + + if (excludedFiles.contains(filePath)) { + assertThat(result.inputFile(filePath)).isNull(); + } else { + assertHiddenFileScan(result, filePath, expectedIsHidden, true); + // we expect the sensor to process all non-excluded files, regardless of visibility + assertFileIssue(result, filePath, true); + } + } + assertThat(result.inputFiles()).hasSize(5); + } + + private File createModuleWithSubdirectory(String moduleName, String subDirName) { + File moduleBaseDir = new File(baseDir, moduleName); + File srcDir = moduleBaseDir.toPath().resolve(subDirName).toFile(); + assertThat(srcDir.mkdirs()).isTrue(); + return srcDir; + } + private static void assertAnalysedFiles(AnalysisResult result, String... files) { assertThat(result.inputFiles().stream().map(InputFile::toString).toList()).contains(files); } @@ -1341,7 +1567,6 @@ class FileSystemMediumIT { } private Path prepareBrokenSymlinkTestScenario() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); File srcDir = new File(baseDir, "src"); assertThat(srcDir.mkdir()).isTrue(); @@ -1353,4 +1578,69 @@ class FileSystemMediumIT { return link; } + private Map<String, Boolean> hiddenFileProjectExpectedHiddenStatus() { + return Map.of( + "xources/.hidden/.xoo", true, + "xources/.hidden/Class.xoo", true, + "xources/.hidden/.nestedHidden/.xoo", true, + "xources/.hidden/.nestedHidden/Class.xoo", true, + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", true, + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo", true, + "xources/nonHidden/.xoo", true, + "xources/nonHidden/Class.xoo", false, + "xources/nonHidden/.hiddenInVisibleFolder/.xoo", true, + "xources/nonHidden/.hiddenInVisibleFolder/Class.xoo", true); + } + + private static void prepareHiddenFileProject() throws IOException { + if (!SystemUtils.IS_OS_WINDOWS) { + return; + } + + // On Windows, we need to set the hidden attribute on the file system + Set<String> dirAndFilesToHideOnWindows = Set.of( + "xources/.hidden", + "xources/.hidden/.xoo", + "xources/.hidden/.nestedHidden", + "xources/.hidden/.nestedHidden/.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", + "xources/nonHidden/.xoo", + "xources/nonHidden/.hiddenInVisibleFolder", + "xources/nonHidden/.hiddenInVisibleFolder/.xoo"); + + for (String path : dirAndFilesToHideOnWindows) { + Path pathFromResources = Path.of("test-resources/mediumtest/xoo/sample-with-hidden-files", path); + setFileAsHiddenOnWindows(pathFromResources); + } + } + + private static void setFileAsHiddenOnWindows(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(path, "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } + + private static void assertHiddenFileScan(AnalysisResult result, String filePath, boolean expectedHiddenStatus, boolean hiddenFilesShouldBeScanned) { + InputFile file = result.inputFile(filePath); + + if (!hiddenFilesShouldBeScanned && expectedHiddenStatus) { + assertThat(file).isNull(); + } else { + assertThat(file).withFailMessage(String.format("File \"%s\" was not analyzed", filePath)).isNotNull(); + assertThat(file.isHidden()) + .withFailMessage(String.format("Expected file \"%s\" hidden status to be \"%s\", however was \"%s\"", filePath, expectedHiddenStatus, file.isHidden())) + .isEqualTo(expectedHiddenStatus); + } + } + + private static void assertFileIssue(AnalysisResult result, String filePath, boolean expectToHaveIssue) { + InputFile file = result.inputFile(filePath); + assertThat(file).isNotNull(); + List<ScannerReport.Issue> issues = result.issuesFor(file); + if (expectToHaveIssue) { + assertThat(issues).hasSize(1); + } else { + assertThat(issues).isEmpty(); + } + } } diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java index e1a158bfac9..904e09cc5c5 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java @@ -241,7 +241,7 @@ public class ScmMediumIT { assertThat(getChangesets(baseDir, NO_BLAME_SCM_ON_SERVER_XOO)).isNull(); // 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total - assertThat(logTester.logs()).containsSubsequence("8 files indexed"); + assertThat(logTester.logs()).anyMatch(s -> s.startsWith("8 files indexed (done) | time=")); assertThat(logTester.logs()).containsSubsequence("SCM Publisher 4 source files to be analyzed"); assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("SCM Publisher 3/4 source files have been analyzed \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 0fe2c3ad479..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,17 +21,21 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.sonar.api.utils.MessageException; @@ -49,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -67,9 +73,11 @@ public class ScannerMain { LOG.info("SonarScanner Engine completed successfully"); return 0; - } catch (Exception e) { - handleException(e); + } catch (Throwable throwable) { + handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -87,30 +95,28 @@ public class ScannerMain { return sb.toString(); } - private static void handleException(Exception e) { - var messageException = unwrapMessageException(e); + private static void handleException(Throwable throwable) { + var messageException = unwrapMessageException(throwable); if (messageException.isPresent()) { // Don't show the stacktrace for a message exception to not pollute the logs if (LoggerFactory.getLogger(ScannerMain.class).isDebugEnabled()) { - LOG.error(messageException.get(), e); + LOG.error(messageException.get(), throwable); } else { LOG.error(messageException.get()); } } else { - LOG.error("Error during SonarScanner Engine execution", e); + LOG.error("Error during SonarScanner Engine execution", throwable); } } - private static Optional<String> unwrapMessageException(Exception t) { - Throwable y = t; - do { - if (y instanceof MessageException messageException) { - return Optional.of(messageException.getMessage()); - } - y = y.getCause(); - } while (y != null); - - return Optional.empty(); + private static Optional<String> unwrapMessageException(@Nullable Throwable throwable) { + if (throwable == null) { + return Optional.empty(); + } else if (throwable instanceof MessageException messageException) { + return Optional.of(messageException.getMessage()); + } else { + return unwrapMessageException(throwable.getCause()); + } } private static @NotNull Map<String, String> parseInputProperties(InputStream in) { @@ -157,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 9c41891dbab..133f4387856 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -99,7 +99,6 @@ import org.sonar.scanner.scan.InputModuleHierarchyProvider; import org.sonar.scanner.scan.InputProjectProvider; import org.sonar.scanner.scan.ModuleIndexer; import org.sonar.scanner.scan.MutableProjectReactorProvider; -import org.sonar.scanner.scan.MutableProjectSettings; import org.sonar.scanner.scan.ProjectBuildersExecutor; import org.sonar.scanner.scan.ProjectConfigurationProvider; import org.sonar.scanner.scan.ProjectLock; @@ -116,6 +115,7 @@ import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; import org.sonar.scanner.scan.filesystem.FilePreprocessor; +import org.sonar.scanner.scan.filesystem.HiddenFilesProjectData; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; @@ -200,6 +200,7 @@ public class SpringScannerContainer extends SpringComponentContainer { FilePreprocessor.class, ProjectFilePreprocessor.class, ProjectExclusionFilters.class, + HiddenFilesProjectData.class, // rules new ActiveRulesProvider(), @@ -226,7 +227,6 @@ public class SpringScannerContainer extends SpringComponentContainer { ContextPropertiesCache.class, TelemetryCache.class, - MutableProjectSettings.class, SonarGlobalPropertiesFilter.class, ProjectConfigurationProvider.class, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java index 703cd038fd0..d4f22ad9704 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java @@ -97,8 +97,11 @@ public class RulesSeverityDetector { } private static Map<String, Result.Level> getDriverDefinedRuleSeverities(Run run) { - return run.getTool().getDriver().getRules() - .stream() + Set<ReportingDescriptor> rules = run.getTool().getDriver().getRules(); + if (rules == null) { + return emptyMap(); + } + return rules.stream() .filter(RulesSeverityDetector::hasRuleDefinedLevel) .collect(toMap(ReportingDescriptor::getId, x -> Result.Level.valueOf(x.getDefaultConfiguration().getLevel().name()))); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java index 5b9abf383cf..bdf5a9a1114 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java @@ -83,7 +83,7 @@ public class RunMapper { private List<NewAdHocRule> toNewAdHocRules(Run run, String driverName, Map<String, Result.Level> ruleSeveritiesByRuleId, Map<String, Result.Level> ruleSeveritiesByRuleIdForNewCCT) { - Set<ReportingDescriptor> driverRules = run.getTool().getDriver().getRules(); + Set<ReportingDescriptor> driverRules = Optional.ofNullable(run.getTool().getDriver().getRules()).orElse(Set.of()); Set<ReportingDescriptor> extensionRules = hasExtensions(run.getTool()) ? run.getTool().getExtensions().stream().filter(RunMapper::hasRules).flatMap(extension -> extension.getRules().stream()).collect(toSet()) : Set.of(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java index 088ebfb0052..d9eed8bedc8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java @@ -87,7 +87,7 @@ public class ScannerWsClientProvider { String responseTimeout = defaultIfBlank(scannerProps.property(SONAR_SCANNER_RESPONSE_TIMEOUT), valueOf(DEFAULT_RESPONSE_TIMEOUT)); String envVarToken = defaultIfBlank(system.envVariable(TOKEN_ENV_VARIABLE), null); String token = defaultIfBlank(scannerProps.property(TOKEN_PROPERTY), envVarToken); - String login = defaultIfBlank(scannerProps.property(CoreProperties.LOGIN), token); + String login = defaultIfBlank(token, scannerProps.property(CoreProperties.LOGIN)); boolean skipSystemTrustMaterial = Boolean.parseBoolean(defaultIfBlank(scannerProps.property(SKIP_SYSTEM_TRUST_MATERIAL), "false")); var sslContext = configureSsl(parseSslConfig(scannerProps, sonarUserHome), system, skipSystemTrustMaterial); connectorBuilder diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java index 51bb79c9d23..24db0ddec64 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.io.UncheckedIOException; import java.lang.reflect.Type; import java.nio.file.Files; import java.nio.file.Path; @@ -139,7 +140,7 @@ public class CliCacheService { String checksum = metadataResponse.sha256(); // If we have a matching checksum dir with the existing CLI file, then we are up to date. if (!cachedCliFile(checksum).exists()) { - LOG.debug("CLI checksum mismatch"); + LOG.debug("SCA CLI update detected"); downloadCli(metadataResponse.id(), checksum); telemetryCache.put("scanner.sca.get.cli.cache.hit", "false"); } else { @@ -175,8 +176,8 @@ public class CliCacheService { }.getType(); return new Gson().fromJson(reader, listOfMetadata); } - } catch (Exception e) { - throw new IllegalStateException("Unable to load CLI metadata", e); + } catch (IOException e) { + throw new UncheckedIOException(e); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java index 8a8e90cce25..6b3418a8f6a 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java @@ -29,20 +29,22 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.platform.Server; import org.sonar.api.utils.System2; import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scanner.config.DefaultConfiguration; import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scm.git.JGitUtils; @@ -54,37 +56,42 @@ import org.sonar.scm.git.JGitUtils; */ public class CliService { private static final Logger LOG = LoggerFactory.getLogger(CliService.class); - public static final String EXCLUDED_MANIFESTS_PROP_KEY = "sonar.sca.excludedManifests"; + public static final String SCA_EXCLUSIONS_KEY = "sonar.sca.exclusions"; + public static final String LEGACY_SCA_EXCLUSIONS_KEY = "sonar.sca.excludedManifests"; private final ProcessWrapperFactory processWrapperFactory; private final TelemetryCache telemetryCache; private final System2 system2; private final Server server; private final ScmConfiguration scmConfiguration; + private final ProjectExclusionFilters projectExclusionFilters; - public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration) { + public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration, + ProjectExclusionFilters projectExclusionFilters) { this.processWrapperFactory = processWrapperFactory; this.telemetryCache = telemetryCache; this.system2 = system2; this.server = server; this.scmConfiguration = scmConfiguration; + this.projectExclusionFilters = projectExclusionFilters; } - public File generateManifestsZip(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { + public File generateManifestsArchive(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { long startTime = system2.now(); boolean success = false; try { - String zipName = "dependency-files.zip"; - Path zipPath = module.getWorkDir().resolve(zipName); + String archiveName = "dependency-files.tar.xz"; + Path archivePath = module.getWorkDir().resolve(archiveName); List<String> args = new ArrayList<>(); args.add(cliExecutable.getAbsolutePath()); args.add("projects"); args.add("save-lockfiles"); - args.add("--zip"); - args.add("--zip-filename"); - args.add(zipPath.toAbsolutePath().toString()); + args.add("--xz"); + args.add("--xz-filename"); + args.add(archivePath.toAbsolutePath().toString()); args.add("--directory"); args.add(module.getBaseDir().toString()); + args.add("--recursive"); String excludeFlag = getExcludeFlag(module, configuration); if (excludeFlag != null) { @@ -92,8 +99,7 @@ public class CliService { args.add(excludeFlag); } - boolean scaDebug = configuration.getBoolean("sonar.sca.debug").orElse(false); - if (LOG.isDebugEnabled() || scaDebug) { + if (LOG.isDebugEnabled()) { LOG.info("Setting CLI to debug mode"); args.add("--debug"); } @@ -104,18 +110,16 @@ public class CliService { envProperties.put("TIDELIFT_ALLOW_MANIFEST_FAILURES", "1"); envProperties.put("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE", "1"); envProperties.put("TIDELIFT_CLI_SQ_SERVER_VERSION", server.getVersion()); - // EXCLUDED_MANIFESTS_PROP_KEY is a special case which we handle via --args, not environment variables - Set<String> ignoredProperties = Set.of(EXCLUDED_MANIFESTS_PROP_KEY); - envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration, ignoredProperties)); + envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration)); LOG.info("Running command: {}", args); LOG.info("Environment properties: {}", envProperties); Consumer<String> logConsumer = LOG.atLevel(Level.INFO)::log; processWrapperFactory.create(module.getWorkDir(), logConsumer, logConsumer, envProperties, args.toArray(new String[0])).execute(); - LOG.info("Generated manifests zip file: {}", zipName); + LOG.info("Generated manifests archive file: {}", archiveName); success = true; - return zipPath.toFile(); + return archivePath.toFile(); } finally { telemetryCache.put("scanner.sca.execution.cli.duration", String.valueOf(system2.now() - startTime)); telemetryCache.put("scanner.sca.execution.cli.success", String.valueOf(success)); @@ -123,7 +127,7 @@ public class CliService { } private @Nullable String getExcludeFlag(DefaultInputModule module, DefaultConfiguration configuration) throws IOException { - List<String> configExcludedPaths = getConfigExcludedPaths(configuration); + List<String> configExcludedPaths = getConfigExcludedPaths(configuration, projectExclusionFilters); List<String> scmIgnoredPaths = getScmIgnoredPaths(module); ArrayList<String> mergedExclusionPaths = new ArrayList<>(); @@ -143,12 +147,15 @@ public class CliService { return toCsvString(mergedExclusionPaths); } - private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration) { - String[] excludedPaths = configuration.getStringArray(EXCLUDED_MANIFESTS_PROP_KEY); - if (excludedPaths == null) { - return List.of(); - } - return Arrays.stream(excludedPaths).toList(); + private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration, ProjectExclusionFilters projectExclusionFilters) { + String[] sonarExclusions = projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN); + String[] scaExclusions = configuration.getStringArray(SCA_EXCLUSIONS_KEY); + String[] scaExclusionsLegacy = configuration.getStringArray(LEGACY_SCA_EXCLUSIONS_KEY); + + return Stream.of(sonarExclusions, scaExclusions, scaExclusionsLegacy) + .flatMap(Arrays::stream) + .distinct() + .toList(); } private List<String> getScmIgnoredPaths(DefaultInputModule module) { @@ -170,7 +177,13 @@ public class CliService { } return scmIgnoredPaths.stream() .map(ignoredPathRel -> { - boolean isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel)); + + boolean isDirectory = false; + try { + isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel.replace("/", File.separator))); + } catch (java.nio.file.InvalidPathException e) { + // if it's not a valid path, it's not a directory so we can just pass to the Tidelift CLI + } // Directories need to get turned into a glob for the Tidelift CLI return isDirectory ? (ignoredPathRel + "/**") : ignoredPathRel; }) diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java index 06142fadb8f..143e144c2dc 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java @@ -21,6 +21,8 @@ package org.sonar.scanner.sca; import java.io.File; import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.StopWatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.internal.DefaultInputModule; @@ -65,18 +67,25 @@ public class ScaExecutor { return; } + var stopwatch = new StopWatch(); + stopwatch.start(); LOG.info("Checking for latest CLI"); File cliFile = cliCacheService.cacheCli(); LOG.info("Collecting manifests for the dependency analysis..."); if (cliFile.exists()) { try { - File generatedZip = cliService.generateManifestsZip(root, cliFile, configuration); + File generatedZip = cliService.generateManifestsArchive(root, cliFile, configuration); LOG.debug("Zip ready for report: {}", generatedZip); reportPublisher.getWriter().writeScaFile(generatedZip); LOG.debug("Manifest zip written to report"); } catch (IOException | IllegalStateException e) { LOG.error("Error gathering manifests", e); + } finally { + stopwatch.stop(); + if (LOG.isInfoEnabled()) { + LOG.info("Load SCA project dependencies (done) | time={}ms", stopwatch.getTime(TimeUnit.MILLISECONDS)); + } } } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java index 5c848b4ddbc..a697aef3e20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java @@ -30,6 +30,13 @@ import org.sonar.scanner.config.DefaultConfiguration; public class ScaProperties { private static final Pattern sonarScaPropertyRegex = Pattern.compile("^sonar\\.sca\\.([a-zA-Z]+)$"); private static final String SONAR_SCA_PREFIX = "sonar.sca."; + private static final Set<String> IGNORED_PROPERTIES = Set.of( + // sonar.sca.exclusions is a special case which we handle when building --exclude + "sonar.sca.exclusions", + // excludedManifests is a special case which we handle when building --exclude + "sonar.sca.excludedManifests", + // keep recursive enabled to better match sonar-scanner behavior + "sonar.sca.recursiveManifestSearch"); private ScaProperties() { } @@ -46,22 +53,16 @@ public class ScaProperties { * { "sonar.someOtherProperty" : "value" } returns an empty map * * @param configuration the scanner configuration possibly containing sonar.sca.* properties - * @param ignoredPropertyNames property names that should not be processed as a property * @return a map of Tidelift CLI compatible environment variable names to their configuration values */ - public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration, Set<String> ignoredPropertyNames) { + public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration) { HashMap<String, String> props = new HashMap<>(configuration.getProperties()); - // recursive mode defaults to true - if (!props.containsKey("sonar.sca.recursiveManifestSearch")) { - props.put("sonar.sca.recursiveManifestSearch", "true"); - } - return props .entrySet() .stream() .filter(entry -> entry.getKey().startsWith(SONAR_SCA_PREFIX)) - .filter(entry -> !ignoredPropertyNames.contains(entry.getKey())) + .filter(entry -> !IGNORED_PROPERTIES.contains(entry.getKey())) .collect(Collectors.toMap(entry -> convertPropToEnvVariable(entry.getKey()), Map.Entry::getValue)); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java deleted file mode 100644 index 15912f8a510..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import jakarta.annotation.Priority; -import org.sonar.api.config.internal.Settings; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ModuleConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(1) -public class MutableModuleSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableModuleSettings(ModuleConfiguration config) { - super(config.getDefinitions(), config.getEncryption()); - addProperties(config.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java deleted file mode 100644 index df24cbe81e5..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.sonar.api.config.internal.Settings; -import org.sonar.scanner.bootstrap.GlobalConfiguration; - -import jakarta.annotation.Priority; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ProjectConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(2) -public class MutableProjectSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableProjectSettings(GlobalConfiguration globalConfig) { - super(globalConfig.getDefinitions(), globalConfig.getEncryption()); - addProperties(globalConfig.getProperties()); - } - - public void complete(ProjectConfiguration projectConfig) { - addProperties(projectConfig.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java index c12ec245924..e5543d4f9c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java @@ -26,7 +26,6 @@ import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.springframework.context.annotation.Bean; - public class ProjectConfigurationProvider { private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -37,7 +36,7 @@ public class ProjectConfigurationProvider { @Bean("ProjectConfiguration") public ProjectConfiguration provide(DefaultInputProject project, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, - ProjectServerSettings projectServerSettings, MutableProjectSettings projectSettings) { + ProjectServerSettings projectServerSettings) { Map<String, String> settings = new LinkedHashMap<>(); settings.putAll(globalServerSettings.properties()); settings.putAll(projectServerSettings.properties()); @@ -45,10 +44,7 @@ public class ProjectConfigurationProvider { settings = sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(settings, globalServerSettings.properties()); - ProjectConfiguration projectConfig = new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); - projectSettings.complete(projectConfig); - return projectConfig; + return new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); } - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java index 4315c762481..8ddb889912d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java @@ -54,7 +54,6 @@ public class SpringModuleScanContainer extends SpringComponentContainer { add( module.definition(), module, - MutableModuleSettings.class, SonarGlobalPropertiesFilter.class, ModuleConfigurationProvider.class, @@ -68,8 +67,7 @@ public class SpringModuleScanContainer extends SpringComponentContainer { ModuleSensorOptimizer.class, ModuleSensorContext.class, - ModuleSensorExtensionDictionary.class - ); + ModuleSensorExtensionDictionary.class); } private void addExtensions() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java index 314e923ce71..242bc015574 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -24,17 +24,15 @@ import java.nio.file.AccessDeniedException; import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; -import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; -import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; public class DirectoryFileVisitor implements FileVisitor<Path> { @@ -43,27 +41,31 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { private final FileVisitAction fileVisitAction; private final DefaultInputModule module; private final ModuleExclusionFilters moduleExclusionFilters; - private final InputModuleHierarchy inputModuleHierarchy; private final InputFile.Type type; + private final HiddenFilesVisitorHelper hiddenFilesVisitorHelper; - DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + InputModuleHierarchy inputModuleHierarchy, InputFile.Type type, HiddenFilesProjectData hiddenFilesProjectData) { this.fileVisitAction = fileVisitAction; this.module = module; this.moduleExclusionFilters = moduleExclusionFilters; this.inputModuleHierarchy = inputModuleHierarchy; this.type = type; + this.hiddenFilesVisitorHelper = new HiddenFilesVisitorHelper(hiddenFilesProjectData, module, moduleConfig); } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + if (hiddenFilesVisitorHelper.shouldVisitDir(dir)) { + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { + if (hiddenFilesVisitorHelper.shouldVisitFile(file)) { fileVisitAction.execute(file); } return FileVisitResult.CONTINUE; @@ -129,25 +131,12 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + hiddenFilesVisitorHelper.exitDirectory(dir); return FileVisitResult.CONTINUE; } - private static boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } - @FunctionalInterface interface FileVisitAction { void execute(Path file) throws IOException; } } - diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 7f31c949132..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -63,12 +63,13 @@ public class FileIndexer { private final ModuleRelativePathWarner moduleRelativePathWarner; private final InputFileFilterRepository inputFileFilterRepository; private final Languages languages; + private final HiddenFilesProjectData hiddenFilesProjectData; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, - InputFileFilterRepository inputFileFilterRepository, Languages languages) { + InputFileFilterRepository inputFileFilterRepository, Languages languages, HiddenFilesProjectData hiddenFilesProjectData) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -83,15 +84,18 @@ public class FileIndexer { this.moduleRelativePathWarner = moduleRelativePathWarner; this.inputFileFilterRepository = inputFileFilterRepository; this.languages = languages; + this.hiddenFilesProjectData = hiddenFilesProjectData; } - void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, - Type type, ProgressReport progressReport) { + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, Type type, + ProgressReport progressReport) { Path projectRelativePath = project.getBaseDir().relativize(sourceFile); Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, @@ -102,11 +106,12 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(sourceFile)); + scmChangedFiles.getOldRelativeFilePath(sourceFile), + isHidden); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), f -> f.setStatus(statusDetection.findStatusFromScm(f))); - if (language != null && isPublishAllFiles(language.key())) { + if (!isHidden && language != null && isPublishAllFiles(language.key())) { inputFile.setPublished(true); } if (!accept(inputFile)) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java index 544fe46c43b..a87c5f11fc9 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -147,22 +147,35 @@ public class FilePreprocessor { return true; } - Path target = Files.readSymbolicLink(absolutePath); - if (!Files.exists(target)) { + Optional<Path> target = resolvePathToTarget(absolutePath); + if (target.isEmpty() || !Files.exists(target.get())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file that does not exist.", absolutePath); return false; } - if (!target.startsWith(project.getBaseDir())) { + if (!target.get().startsWith(project.getBaseDir())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file not located in project basedir.", absolutePath); return false; } - if (!target.startsWith(moduleBaseDirectory)) { + if (!target.get().startsWith(moduleBaseDirectory)) { LOG.info("File '{}' is ignored. It is a symbolic link targeting a file not located in module basedir.", absolutePath); return false; } return true; } + + private static Optional<Path> resolvePathToTarget(Path symbolicLinkAbsolutePath) throws IOException { + Path target = Files.readSymbolicLink(symbolicLinkAbsolutePath); + if (target.isAbsolute()) { + return Optional.of(target); + } + + try { + return Optional.of(symbolicLinkAbsolutePath.getParent().resolve(target).toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize()); + } catch (IOException e) { + return Optional.empty(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java new file mode 100644 index 00000000000..d779a054455 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +public class HiddenFilesProjectData { + + final Map<DefaultInputModule, Set<Path>> hiddenFilesByModule = new HashMap<>(); + private final SonarUserHome sonarUserHome; + private Path cachedSonarUserHomePath; + + public HiddenFilesProjectData(SonarUserHome sonarUserHome) { + this.sonarUserHome = sonarUserHome; + } + + public void markAsHiddenFile(Path file, DefaultInputModule module) { + hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); + } + + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { + Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); + } + return false; + } + + public Path getCachedSonarUserHomePath() throws IOException { + if (cachedSonarUserHomePath == null) { + cachedSonarUserHomePath = resolveRealPath(sonarUserHome.getPath()); + } + return cachedSonarUserHomePath; + } + + public void clearHiddenFilesData() { + // Allowing the GC to collect the map, should only be done after all indexing is complete + hiddenFilesByModule.clear(); + } + + public Path resolveRealPath(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + return path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + } + return path; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java new file mode 100644 index 00000000000..607a859ef44 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.scan.ModuleConfiguration; + +public class HiddenFilesVisitorHelper { + + private static final String EXCLUDE_HIDDEN_FILES_PROPERTY = "sonar.scanner.excludeHiddenFiles"; + private final HiddenFilesProjectData hiddenFilesProjectData; + private final DefaultInputModule module; + final boolean excludeHiddenFiles; + private Path moduleWorkDir; + Path rootHiddenDir; + + public HiddenFilesVisitorHelper(HiddenFilesProjectData hiddenFilesProjectData, DefaultInputModule module, ModuleConfiguration moduleConfig) { + this.hiddenFilesProjectData = hiddenFilesProjectData; + this.module = module; + this.excludeHiddenFiles = moduleConfig.getBoolean(EXCLUDE_HIDDEN_FILES_PROPERTY).orElse(false); + } + + public boolean shouldVisitDir(Path path) throws IOException { + boolean isHidden = isHiddenDir(path); + + if (isHidden && (excludeHiddenFiles || isExcludedHiddenDirectory(path))) { + return false; + } + if (isHidden) { + enterHiddenDirectory(path); + } + return true; + } + + private boolean isExcludedHiddenDirectory(Path path) throws IOException { + return getCachedModuleWorkDir().equals(path) || hiddenFilesProjectData.getCachedSonarUserHomePath().equals(path); + } + + void enterHiddenDirectory(Path dir) { + if (!insideHiddenDirectory()) { + rootHiddenDir = dir; + } + } + + public void exitDirectory(Path path) { + if (insideHiddenDirectory() && rootHiddenDir.equals(path)) { + resetRootHiddenDir(); + } + } + + void resetRootHiddenDir() { + this.rootHiddenDir = null; + } + + public boolean shouldVisitFile(Path path) throws IOException { + boolean isHidden = insideHiddenDirectory() || Files.isHidden(path); + + if (!excludeHiddenFiles && isHidden) { + hiddenFilesProjectData.markAsHiddenFile(path, module); + } + + return !excludeHiddenFiles || !isHidden; + } + + private Path getCachedModuleWorkDir() throws IOException { + if (moduleWorkDir == null) { + moduleWorkDir = hiddenFilesProjectData.resolveRealPath(module.getWorkDir()); + } + return moduleWorkDir; + } + + // visible for testing + boolean insideHiddenDirectory() { + return rootHiddenDir != null; + } + + protected static boolean isHiddenDir(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java index 6ef26dafd07..68b6d1db580 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java @@ -19,12 +19,15 @@ */ package org.sonar.scanner.scan.filesystem; +import java.util.Set; import java.util.SortedSet; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.sonar.api.batch.ScannerSide; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; -import org.sonar.api.batch.fs.internal.SensorStrategy; import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.SensorStrategy; @ScannerSide public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @@ -73,11 +76,29 @@ public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @Override public Iterable<InputFile> getFilesByName(String filename) { - return inputComponentStore.getFilesByName(filename); + Iterable<InputFile> allFilesByName = inputComponentStore.getFilesByName(filename); + if (strategy.isGlobal()) { + return allFilesByName; + } + + return filterByModule(allFilesByName); } @Override public Iterable<InputFile> getFilesByExtension(String extension) { - return inputComponentStore.getFilesByExtension(extension); + Iterable<InputFile> allFilesByExtension = inputComponentStore.getFilesByExtension(extension); + if (strategy.isGlobal()) { + return allFilesByExtension; + } + + return filterByModule(allFilesByExtension); + } + + private Iterable<InputFile> filterByModule(Iterable<InputFile> projectInputFiles) { + Set<InputFile> projectInputFilesSet = StreamSupport.stream(projectInputFiles.spliterator(), false) + .collect(Collectors.toSet()); + return StreamSupport.stream(inputComponentStore.filesByModule(moduleKey).spliterator(), false) + .filter(projectInputFilesSet::contains) + .toList(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java index 5daa384d3ac..9c969f6ae20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java @@ -25,35 +25,54 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.predicates.ChangedFilePredicate; +import org.sonar.api.batch.fs.internal.predicates.NonHiddenFilesPredicate; public class MutableFileSystem extends DefaultFileSystem { - private boolean restrictToChangedFiles = false; + + boolean restrictToChangedFiles = false; + boolean allowHiddenFileAnalysis = false; public MutableFileSystem(Path baseDir, Cache cache, FilePredicates filePredicates) { super(baseDir, cache, filePredicates); } - public MutableFileSystem(Path baseDir) { + MutableFileSystem(Path baseDir) { super(baseDir); } @Override public Iterable<InputFile> inputFiles(FilePredicate requestPredicate) { - if (restrictToChangedFiles) { - return super.inputFiles(new ChangedFilePredicate(requestPredicate)); - } - return super.inputFiles(requestPredicate); + return super.inputFiles(applyAdditionalPredicate(requestPredicate)); } @Override public InputFile inputFile(FilePredicate requestPredicate) { + return super.inputFile(applyAdditionalPredicate(requestPredicate)); + } + + private FilePredicate applyAdditionalPredicate(FilePredicate requestPredicate) { + return applyHiddenFilePredicate(applyChangedFilePredicate(requestPredicate)); + } + + private FilePredicate applyHiddenFilePredicate(FilePredicate predicate) { + if (allowHiddenFileAnalysis) { + return predicate; + } + return predicates().and(new NonHiddenFilesPredicate(), predicate); + } + + private FilePredicate applyChangedFilePredicate(FilePredicate predicate) { if (restrictToChangedFiles) { - return super.inputFile(new ChangedFilePredicate(requestPredicate)); + return predicates().and(new ChangedFilePredicate(), predicate); } - return super.inputFile(requestPredicate); + return predicate; } public void setRestrictToChangedFiles(boolean restrictToChangedFiles) { this.restrictToChangedFiles = restrictToChangedFiles; } + + public void setAllowHiddenFileAnalysis(boolean allowHiddenFileAnalysis) { + this.allowHiddenFileAnalysis = allowHiddenFileAnalysis; + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 97e449fcb26..c1349872c24 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -62,6 +62,7 @@ public class ProjectFileIndexer { private final FileIndexer fileIndexer; private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; + private final HiddenFilesProjectData hiddenFilesProjectData; private ProgressReport progressReport; @@ -69,7 +70,7 @@ public class ProjectFileIndexer { SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, - ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings, HiddenFilesProjectData hiddenFilesProjectData) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -81,6 +82,7 @@ public class ProjectFileIndexer { this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void index() { @@ -91,10 +93,10 @@ public class ProjectFileIndexer { projectCoverageAndDuplicationExclusions.log(" "); indexModulesRecursively(inputModuleHierarchy.root()); + hiddenFilesProjectData.clearHiddenFilesData(); int totalIndexed = componentStore.inputFiles().size(); - progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - + progressReport.stopAndLogTotalTime(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); } private void indexModulesRecursively(DefaultInputModule module) { @@ -118,15 +120,15 @@ public class ProjectFileIndexer { moduleCoverageAndDuplicationExclusions.log(" "); } List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); projectFilePreprocessor.getTestSourcesByModule(module) - .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); + .ifPresent(tests -> indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { if (!paths.isEmpty()) { StringBuilder sb = new StringBuilder(label); - for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { + for (Iterator<Path> it = paths.iterator(); it.hasNext();) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); if (relativePathToBaseDir.isEmpty()) { @@ -148,12 +150,13 @@ public class ProjectFileIndexer { } } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + private void indexFiles(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); + indexDirectory(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } @@ -163,18 +166,16 @@ public class ProjectFileIndexer { } } - private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + private void indexDirectory(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), - module, moduleExclusionFilters, inputModuleHierarchy, type)); + module, moduleConfig, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); } private static String pluralizeFiles(int count) { return count == 1 ? "file" : "files"; } - - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 033ab56d3d4..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -66,6 +66,7 @@ public class ProjectFilePreprocessor { private final LanguageDetection languageDetection; private final FilePreprocessor filePreprocessor; private final ProjectExclusionFilters projectExclusionFilters; + private final HiddenFilesProjectData hiddenFilesProjectData; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -79,7 +80,7 @@ public class ProjectFilePreprocessor { public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, LanguageDetection languageDetection, FilePreprocessor filePreprocessor, - ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, HiddenFilesProjectData hiddenFilesProjectData) { this.analysisWarnings = analysisWarnings; this.scmConfiguration = scmConfiguration; this.inputModuleHierarchy = inputModuleHierarchy; @@ -92,6 +93,7 @@ public class ProjectFilePreprocessor { this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.ignoreCommand = loadIgnoreCommand(); this.useScmExclusion = ignoreCommand != null; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void execute() { @@ -109,7 +111,7 @@ public class ProjectFilePreprocessor { int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); - progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + progressReport.stopAndLogTotalTime(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); @@ -138,27 +140,31 @@ public class ProjectFilePreprocessor { // Default to index basedir when no sources provided List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + List<Path> processedSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, exclusionCounter); mainSourcesByModule.put(module, processedSources); totalFilesPreprocessed += processedSources.size(); module.getTestDirsOrFiles().ifPresent(tests -> { - List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + List<Path> processedTestSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); testSourcesByModule.put(module, processedTestSources); totalFilesPreprocessed += processedTestSources.size(); }); } - private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, + private List<Path> processModuleSources(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, InputFile.Type type, ExclusionCounter exclusionCounter) { List<Path> processedFiles = new ArrayList<>(); try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -167,12 +173,17 @@ public class ProjectFilePreprocessor { return processedFiles; } - private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + private List<Path> processDirectory(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, Path path, InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java index a08380cf9d8..10d75a4b3c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java @@ -19,11 +19,11 @@ */ package org.sonar.scanner.sensor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor; import org.sonar.api.scanner.sensor.ProjectSensor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.filesystem.MutableFileSystem; @@ -60,7 +60,12 @@ public abstract class AbstractSensorWrapper<G extends ProjectSensor> { if (sensorIsRestricted) { LOGGER.info("Sensor {} is restricted to changed files only", descriptor.name()); } + boolean allowHiddenFileAnalysis = descriptor.isProcessesHiddenFiles(); + if (allowHiddenFileAnalysis) { + LOGGER.debug("Sensor {} is allowed to analyze hidden files", descriptor.name()); + } fileSystem.setRestrictToChangedFiles(sensorIsRestricted); + fileSystem.setAllowHiddenFileAnalysis(allowHiddenFileAnalysis); wrappedSensor.execute(context); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 5f28e7e283e..01b6c0c11cd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -28,7 +28,6 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.cache.ReadCache; import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.config.Configuration; -import org.sonar.api.config.Settings; import org.sonar.scanner.bootstrap.ScannerPluginRepository; import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; @@ -38,11 +37,11 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; - public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, + public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, FileSystem fs, ActiveRules activeRules, DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepository) { - super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, + super(project, config, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); this.module = module; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index 6fb38fa4563..54c86750eaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -66,7 +66,6 @@ public class ProjectSensorContext implements SensorContext { static final NoOpNewAnalysisError NO_OP_NEW_ANALYSIS_ERROR = new NoOpNewAnalysisError(); - private final Settings mutableSettings; private final FileSystem fs; private final ActiveRules activeRules; private final DefaultSensorStorage sensorStorage; @@ -81,15 +80,14 @@ public class ProjectSensorContext implements SensorContext { private final ExecutingSensorContext executingSensorContext; private final ScannerPluginRepository pluginRepo; - public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, - ActiveRules activeRules, - DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, - WriteCache writeCache, ReadCache readCache, - AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, - ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { + public ProjectSensorContext(DefaultInputProject project, Configuration config, FileSystem fs, + ActiveRules activeRules, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + WriteCache writeCache, ReadCache readCache, + AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, + ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { this.project = project; this.config = config; - this.mutableSettings = mutableSettings; this.fs = fs; this.activeRules = activeRules; this.sensorStorage = sensorStorage; @@ -105,7 +103,7 @@ public class ProjectSensorContext implements SensorContext { @Override public Settings settings() { - return mutableSettings; + throw new UnsupportedOperationException("This method is not supported anymore"); } @Override diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java index 0742740bba6..a481f4a54f4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java @@ -22,7 +22,9 @@ package org.sonar.scm.git; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,6 +34,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.diff.RawTextComparator; @@ -238,7 +241,7 @@ public class CompositeBlameCommand extends BlameCommand { break; } linesList.add(new BlameLine() - .date(fileBlame.getCommitDates()[i]) + .date(toDate(fileBlame.getCommitDates()[i])) .revision(fileBlame.getCommitHashes()[i]) .author(fileBlame.getAuthorEmails()[i])); } @@ -251,4 +254,8 @@ public class CompositeBlameCommand extends BlameCommand { } } + private static @Nullable Date toDate(@Nullable Instant commitDate) { + return commitDate != null ? Date.from(commitDate) : null; + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java index 569999a55ab..8f066727e21 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.Date; import java.util.LinkedList; import java.util.List; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -62,6 +63,7 @@ public class NativeGitBlameCommand { private final System2 system; private final ProcessWrapperFactory processWrapperFactory; + private final Consumer<String> stderrConsumer = line -> LOG.debug("[stderr] {}", line); private String gitCommand; @Autowired @@ -85,7 +87,7 @@ public class NativeGitBlameCommand { try { this.gitCommand = locateDefaultGit(); MutableString stdOut = new MutableString(); - this.processWrapperFactory.create(null, l -> stdOut.string = l, gitCommand, "--version").execute(); + this.processWrapperFactory.create(null, l -> stdOut.string = l, stderrConsumer, gitCommand, "--version").execute(); return stdOut.string != null && stdOut.string.startsWith("git version") && isCompatibleGitVersion(stdOut.string); } catch (Exception e) { LOG.debug("Failed to find git native client", e); @@ -109,7 +111,7 @@ public class NativeGitBlameCommand { // To avoid it we use where.exe to find git binary only in PATH. LOG.debug("Looking for git command in the PATH using where.exe (Windows)"); List<String> whereCommandResult = new LinkedList<>(); - this.processWrapperFactory.create(null, whereCommandResult::add, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") + this.processWrapperFactory.create(null, whereCommandResult::add, stderrConsumer, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") .execute(); if (!whereCommandResult.isEmpty()) { @@ -125,6 +127,7 @@ public class NativeGitBlameCommand { var processWrapper = this.processWrapperFactory.create( baseDir, outputProcessor::process, + stderrConsumer, gitCommand, GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), BLAME_COMMAND, diff --git a/sonar-scanner-engine/src/main/resources/logback.xml b/sonar-scanner-engine/src/main/resources/logback.xml index ccd0dfe09b9..ddc2805f08a 100644 --- a/sonar-scanner-engine/src/main/resources/logback.xml +++ b/sonar-scanner-engine/src/main/resources/logback.xml @@ -1,17 +1,8 @@ <?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE configuration> -<!-- This logback configuration is used when the scanner engine is bootstrapped using the SonarScannerCli class. --> +<!-- This logback configuration is used when the scanner engine is bootstrapped using the ScannerMain class. --> <configuration scan="false"> - <import class="ch.qos.logback.core.ConsoleAppender"/> - - <appender name="STDOUT" class="ConsoleAppender"> - <encoder class="org.sonar.scanner.bootstrap.ScannerLogbackEncoder"/> - </appender> - - <root level="info"> - <appender-ref ref="STDOUT"/> - </root> <!-- BeanUtils generate too many DEBUG logs when sonar.verbose is set --> <logger name="org.apache.commons.beanutils.converters" level="WARN"/> @@ -31,4 +22,4 @@ <logger name="nl.altindag.ssl.util.CertificateUtils" level="INFO"/> -</configuration>
\ No newline at end of file +</configuration> diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java index 5091dcf5a3a..f5d88016944 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java @@ -22,12 +22,16 @@ package org.sonar.scanner.externalissue.sarif; import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import org.assertj.core.groups.Tuple; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.slf4j.event.Level; -import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.sarif.pojo.ReportingConfiguration; import org.sonar.sarif.pojo.ReportingDescriptor; import org.sonar.sarif.pojo.Result; @@ -44,12 +48,12 @@ import static org.sonar.sarif.pojo.Result.Level.WARNING; import static org.sonar.scanner.externalissue.sarif.ResultMapper.DEFAULT_IMPACT_SEVERITY; import static org.sonar.scanner.externalissue.sarif.ResultMapper.DEFAULT_SEVERITY; -public class RulesSeverityDetectorTest { +class RulesSeverityDetectorTest { private static final String DRIVER_NAME = "Test"; private static final String RULE_ID = "RULE_ID"; - @org.junit.Rule - public LogTester logTester = new LogTester().setLevel(Level.TRACE); + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); private final Run run = mock(Run.class); private final ReportingDescriptor rule = mock(ReportingDescriptor.class); @@ -59,8 +63,8 @@ public class RulesSeverityDetectorTest { private final ToolComponent extension = mock(ToolComponent.class); private final ReportingConfiguration defaultConfiguration = mock(ReportingConfiguration.class); - @Before - public void setUp() { + @BeforeEach + void setUp() { when(run.getResults()).thenReturn(List.of(result)); when(run.getTool()).thenReturn(tool); when(tool.getDriver()).thenReturn(driver); @@ -68,8 +72,8 @@ public class RulesSeverityDetectorTest { // We keep this test for backward compatibility until we remove the deprecated severity @Test - public void detectRulesSeverities_detectsCorrectlyResultDefinedRuleSeverities() { - Run run = mockResultDefinedRuleSeverities(); + void detectRulesSeverities_detectsCorrectlyResultDefinedRuleSeverities() { + mockResultDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeverities(run, DRIVER_NAME); @@ -78,8 +82,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeveritiesForNewTaxonomy_shouldReturnsEmptyMapAndLogsWarning_whenOnlyResultDefinedRuleSeverities() { - Run run = mockResultDefinedRuleSeverities(); + void detectRulesSeveritiesForNewTaxonomy_shouldReturnsEmptyMapAndLogsWarning_whenOnlyResultDefinedRuleSeverities() { + mockResultDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -88,8 +92,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeverities_detectsCorrectlyDriverDefinedRuleSeverities() { - Run run = mockDriverDefinedRuleSeverities(); + void detectRulesSeverities_detectsCorrectlyDriverDefinedRuleSeverities() { + mockDriverDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -103,9 +107,13 @@ public class RulesSeverityDetectorTest { assertDetectedRuleSeverities(rulesSeveritiesByRuleId, tuple(RULE_ID, WARNING)); } - @Test - public void detectRulesSeverities_detectsCorrectlyExtensionsDefinedRuleSeverities() { - Run run = mockExtensionsDefinedRuleSeverities(); + + + @ParameterizedTest + @NullAndEmptySource + void detectRulesSeverities_detectsCorrectlyExtensionsDefinedRuleSeverities(@Nullable Set<ReportingDescriptor> rules) { + when(driver.getRules()).thenReturn(rules); + mockExtensionsDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -120,8 +128,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeverities_returnsEmptyMapAndLogsWarning_whenUnableToDetectSeverities() { - Run run = mockUnsupportedRuleSeveritiesDefinition(); + void detectRulesSeverities_returnsEmptyMapAndLogsWarning_whenUnableToDetectSeverities() { + mockUnsupportedRuleSeveritiesDefinition(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -135,38 +143,33 @@ public class RulesSeverityDetectorTest { assertDetectedRuleSeverities(rulesSeveritiesByRuleId); } - private Run mockResultDefinedRuleSeverities() { + private void mockResultDefinedRuleSeverities() { when(run.getResults()).thenReturn(List.of(result)); when(result.getLevel()).thenReturn(WARNING); when(result.getRuleId()).thenReturn(RULE_ID); - return run; } - private Run mockDriverDefinedRuleSeverities() { + private void mockDriverDefinedRuleSeverities() { when(driver.getRules()).thenReturn(Set.of(rule)); when(rule.getId()).thenReturn(RULE_ID); when(rule.getDefaultConfiguration()).thenReturn(defaultConfiguration); when(defaultConfiguration.getLevel()).thenReturn(ReportingConfiguration.Level.WARNING); - return run; } - private Run mockExtensionsDefinedRuleSeverities() { - when(driver.getRules()).thenReturn(Set.of()); + private void mockExtensionsDefinedRuleSeverities() { when(tool.getExtensions()).thenReturn(Set.of(extension)); when(extension.getRules()).thenReturn(Set.of(rule)); when(rule.getId()).thenReturn(RULE_ID); when(rule.getDefaultConfiguration()).thenReturn(defaultConfiguration); when(defaultConfiguration.getLevel()).thenReturn(ReportingConfiguration.Level.WARNING); - return run; } - private Run mockUnsupportedRuleSeveritiesDefinition() { + private void mockUnsupportedRuleSeveritiesDefinition() { when(run.getTool()).thenReturn(tool); when(tool.getDriver()).thenReturn(driver); when(driver.getRules()).thenReturn(Set.of()); when(tool.getExtensions()).thenReturn(Set.of(extension)); when(extension.getRules()).thenReturn(Set.of()); - return run; } private void assertNoLogs() { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java index 90ddfadd9f3..164787f8cde 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java @@ -22,19 +22,19 @@ package org.sonar.scanner.externalissue.sarif; import java.util.List; import java.util.Map; import java.util.Set; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.event.Level; import org.sonar.api.batch.sensor.issue.NewExternalIssue; import org.sonar.api.batch.sensor.rule.NewAdHocRule; -import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.sarif.pojo.ReportingDescriptor; import org.sonar.sarif.pojo.Result; import org.sonar.sarif.pojo.Run; @@ -44,13 +44,14 @@ import org.sonar.scanner.externalissue.sarif.RunMapper.RunMapperResult; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.sonar.sarif.pojo.Result.Level.WARNING; -@RunWith(MockitoJUnitRunner.class) -public class RunMapperTest { +@ExtendWith(MockitoExtension.class) +class RunMapperTest { private static final String TEST_DRIVER = "Test driver"; public static final String RULE_ID = "ruleId"; @@ -66,21 +67,21 @@ public class RunMapperTest { @Mock private ReportingDescriptor rule; - @Rule - public LogTester logTester = new LogTester(); + @RegisterExtension + public LogTesterJUnit5 logTester = new LogTesterJUnit5(); @InjectMocks private RunMapper runMapper; - @Before - public void setUp() { - when(run.getTool().getDriver().getName()).thenReturn(TEST_DRIVER); - when(run.getTool().getExtensions()).thenReturn(null); - when(rule.getId()).thenReturn(RULE_ID); + @BeforeEach + void setUp() { + lenient().when(run.getTool().getDriver().getName()).thenReturn(TEST_DRIVER); + lenient().when(run.getTool().getExtensions()).thenReturn(null); + lenient().when(rule.getId()).thenReturn(RULE_ID); } @Test - public void mapRun_shouldMapExternalIssues() { + void mapRun_shouldMapExternalIssues() { Result result1 = mock(Result.class); Result result2 = mock(Result.class); when(run.getResults()).thenReturn(List.of(result1, result2)); @@ -99,7 +100,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldMapExternalRules_whenDriverHasRulesAndNoExtensions() { + void mapRun_shouldMapExternalRules_whenDriverHasRulesAndNoExtensions() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); NewAdHocRule externalRule = mockMappedExternalRule(); @@ -115,7 +116,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldMapExternalRules_whenRulesInExtensions() { + void mapRun_shouldMapExternalRules_whenRulesInExtensions() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of()); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(Set.of(rule)); @@ -134,7 +135,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldNotFail_whenExtensionsDontHaveRules() { + void mapRun_shouldNotFail_whenExtensionsDontHaveRules() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(null); @@ -149,7 +150,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldNotFail_whenExtensionsHaveEmptyRules() { + void mapRun_shouldNotFail_whenExtensionsHaveEmptyRules() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(Set.of()); @@ -164,7 +165,7 @@ public class RunMapperTest { } @Test - public void mapRun_ifRunIsEmpty_returnsEmptyList() { + void mapRun_ifRunIsEmpty_returnsEmptyList() { when(run.getResults()).thenReturn(List.of()); RunMapperResult runMapperResult = runMapper.mapRun(run); @@ -173,7 +174,7 @@ public class RunMapperTest { } @Test - public void mapRun_ifExceptionThrownByResultMapper_logsThemAndContinueProcessing() { + void mapRun_ifExceptionThrownByResultMapper_logsThemAndContinueProcessing() { Result result1 = mock(Result.class); Result result2 = mock(Result.class); when(run.getResults()).thenReturn(List.of(result1, result2)); @@ -194,7 +195,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfToolNotSet() { + void mapRun_failsIfToolNotSet() { when(run.getTool()).thenReturn(null); assertThatIllegalArgumentException() @@ -203,7 +204,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfDriverNotSet() { + void mapRun_failsIfDriverNotSet() { when(run.getTool().getDriver()).thenReturn(null); assertThatIllegalArgumentException() @@ -212,7 +213,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfDriverNameIsNotSet() { + void mapRun_failsIfDriverNameIsNotSet() { when(run.getTool().getDriver().getName()).thenReturn(null); assertThatIllegalArgumentException() @@ -220,6 +221,25 @@ public class RunMapperTest { .withMessage("The run does not have a tool driver name defined."); } + @Test + void mapRun_shouldNotFail_whenDriverRulesNullAndExtensionsRulesNotNull() { + when(run.getTool().getDriver().getRules()).thenReturn(null); + ToolComponent extension = mock(ToolComponent.class); + when(extension.getRules()).thenReturn(Set.of(rule)); + when(run.getTool().getExtensions()).thenReturn(Set.of(extension)); + NewAdHocRule expectedRule = mock(NewAdHocRule.class); + when(ruleMapper.mapRule(rule, TEST_DRIVER, WARNING, WARNING)).thenReturn(expectedRule); + + try (MockedStatic<RulesSeverityDetector> detector = mockStatic(RulesSeverityDetector.class)) { + detector.when(() -> RulesSeverityDetector.detectRulesSeverities(run, TEST_DRIVER)).thenReturn(Map.of(RULE_ID, WARNING)); + detector.when(() -> RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, TEST_DRIVER)).thenReturn(Map.of(RULE_ID, WARNING)); + + RunMapperResult runMapperResult = runMapper.mapRun(run); + assertThat(runMapperResult.getNewAdHocRules()).hasSize(1); + assertThat(runMapperResult.getNewAdHocRules().get(0)).isEqualTo(expectedRule); + } + } + private NewExternalIssue mockMappedExternalIssue(Result result) { NewExternalIssue externalIssue = mock(NewExternalIssue.class); when(result.getRuleId()).thenReturn(RULE_ID); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java index 9193157e213..6615ba4e4e4 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java @@ -29,6 +29,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.SystemUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -161,7 +162,7 @@ class CliCacheServiceTest { WsTestUtil.mockException(scannerWsClient, e); assertThatThrownBy(underTest::cacheCli).isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Unable to load CLI metadata"); + .hasMessageContaining("http error"); verify(telemetryCache).put("scanner.sca.get.cli.success", "false"); } @@ -187,14 +188,18 @@ class CliCacheServiceTest { FileUtils.writeStringToFile(existingFile, fileContent, Charset.defaultCharset()); assertThat(existingFile).exists(); - assertThat(existingFile.canExecute()).isFalse(); + if (!SystemUtils.IS_OS_WINDOWS) { + assertThat(existingFile.canExecute()).isFalse(); + } assertThat(FileUtils.readFileToString(existingFile, Charset.defaultCharset())).isEqualTo(fileContent); underTest.cacheCli(); WsTestUtil.verifyCall(scannerWsClient, CLI_WS_URL); assertThat(existingFile).exists(); - assertThat(existingFile.canExecute()).isFalse(); + if (!SystemUtils.IS_OS_WINDOWS) { + assertThat(existingFile.canExecute()).isFalse(); + } assertThat(FileUtils.readFileToString(existingFile, Charset.defaultCharset())).isEqualTo(fileContent); verify(telemetryCache).put("scanner.sca.get.cli.cache.hit", "true"); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java index 597fafa833c..33cf6c146e6 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java @@ -27,7 +27,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.Optional; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -36,6 +35,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; import org.mockito.MockedStatic; import org.sonar.api.batch.bootstrap.ProjectDefinition; +import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.batch.scm.ScmProvider; import org.sonar.api.platform.Server; @@ -44,6 +44,7 @@ import org.sonar.api.utils.System2; import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scanner.config.DefaultConfiguration; import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; import org.sonar.scanner.scm.ScmConfiguration; import org.sonar.scm.git.GitScmProvider; import org.sonar.scm.git.JGitUtils; @@ -69,6 +70,7 @@ class CliServiceTest { ProcessWrapperFactory processWrapperFactory = mock(ProcessWrapperFactory.class, CALLS_REAL_METHODS); private MockedStatic<JGitUtils> jGitUtilsMock; DefaultConfiguration configuration = mock(DefaultConfiguration.class); + ProjectExclusionFilters projectExclusionFilters = mock(ProjectExclusionFilters.class); private CliService underTest; @@ -86,9 +88,11 @@ class CliServiceTest { jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of("ignored.txt")); when(server.getVersion()).thenReturn("1.0.0"); logTester.setLevel(INFO); - when(configuration.getBoolean("sonar.sca.debug")).thenReturn(Optional.of(true)); + when(projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN)).thenReturn(new String[0]); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[0]); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[0]); - underTest = new CliService(processWrapperFactory, telemetryCache, System2.INSTANCE, server, scmConfiguration); + underTest = new CliService(processWrapperFactory, telemetryCache, System2.INSTANCE, server, scmConfiguration, projectExclusionFilters); } @AfterEach @@ -99,57 +103,55 @@ class CliServiceTest { } @Test - void generateZip_shouldCallProcessCorrectly_andRegisterTelemetry() throws IOException, URISyntaxException { + void generateManifestsArchive_shouldCallProcessCorrectly_andRegisterTelemetry() throws IOException, URISyntaxException { assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); - when(configuration.getProperties()).thenReturn(Map.of("sonar.sca.recursiveManifestSearch", "true", CliService.EXCLUDED_MANIFESTS_PROP_KEY, "foo,bar,baz/**")); - when(configuration.get("sonar.sca.recursiveManifestSearch")).thenReturn(Optional.of("true")); - when(configuration.getStringArray(CliService.EXCLUDED_MANIFESTS_PROP_KEY)).thenReturn(new String[] {"foo", "bar", "baz/**"}); + when(configuration.getProperties()).thenReturn(Map.of(CliService.SCA_EXCLUSIONS_KEY, "foo,bar,baz/**")); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"foo", "bar", "baz/**"}); - File producedZip = underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + File producedArchive = underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); - assertThat(producedZip).exists(); + assertThat(producedArchive).exists(); var expectedArguments = List.of( "projects", "save-lockfiles", - "--zip", - "--zip-filename", - rootInputModule.getWorkDir().resolve("dependency-files.zip").toString(), + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), "--directory", rootInputModule.getBaseDir().toString(), + "--recursive", "--exclude", - "foo,bar,baz/**,ignored.txt,.scannerwork/**", - "--debug"); + "foo,bar,baz/**,ignored.txt,.scannerwork/**"); assertThat(logTester.logs(INFO)) .contains("Arguments Passed In: " + String.join(" ", expectedArguments)) .contains("TIDELIFT_SKIP_UPDATE_CHECK=1") .contains("TIDELIFT_ALLOW_MANIFEST_FAILURES=1") - .contains("TIDELIFT_RECURSIVE_MANIFEST_SEARCH=true") - .contains("Generated manifests zip file: " + producedZip.getName()); + .contains("Generated manifests archive file: " + producedArchive.getName()); assertThat(telemetryCache.getAll()).containsKey("scanner.sca.execution.cli.duration").isNotNull(); assertThat(telemetryCache.getAll()).containsEntry("scanner.sca.execution.cli.success", "true"); } @Test - void generateZip_whenDebugLogLevelAndScaDebugNotEnabled_shouldWriteDebugLogsToDebugStream() throws IOException, URISyntaxException { + void generateManifestsArchive_whenDebugLogLevelAndScaDebugNotEnabled_shouldWriteDebugLogsToDebugStream() throws IOException, URISyntaxException { logTester.setLevel(DEBUG); - when(configuration.getBoolean("sonar.sca.debug")).thenReturn(Optional.of(false)); assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); var expectedArguments = List.of( "projects", "save-lockfiles", - "--zip", - "--zip-filename", - rootInputModule.getWorkDir().resolve("dependency-files.zip").toString(), + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), "--directory", rootInputModule.getBaseDir().toString(), + "--recursive", "--exclude", "ignored.txt,.scannerwork/**", "--debug"); @@ -159,32 +161,30 @@ class CliServiceTest { } @Test - void generateZip_whenScaDebugEnabled_shouldWriteDebugLogsToInfoStream() throws IOException, URISyntaxException { - when(configuration.getBoolean("sonar.sca.debug")).thenReturn(Optional.of(true)); - + void generateManifestsArchive_whenScaDebugEnabled_shouldWriteDebugLogsToInfoStream() throws IOException, URISyntaxException { assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); var expectedArguments = List.of( "projects", "save-lockfiles", - "--zip", - "--zip-filename", - rootInputModule.getWorkDir().resolve("dependency-files.zip").toString(), + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), "--directory", rootInputModule.getBaseDir().toString(), + "--recursive", "--exclude", - "ignored.txt,.scannerwork/**", - "--debug"); + "ignored.txt,.scannerwork/**"); assertThat(logTester.logs(INFO)) .contains("Arguments Passed In: " + String.join(" ", expectedArguments)); } @Test - void generateZip_shouldSendSQEnvVars() throws IOException, URISyntaxException { - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + void generateManifestsArchive_shouldSendSQEnvVars() throws IOException, URISyntaxException { + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); assertThat(logTester.logs(INFO)) .contains("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE=1") @@ -192,20 +192,20 @@ class CliServiceTest { } @Test - void generateZip_includesIgnoredPathsFromGitProvider() throws Exception { - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + void generateManifestsArchive_includesIgnoredPathsFromGitProvider() throws Exception { + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); var expectedArguments = List.of( "projects", "save-lockfiles", - "--zip", - "--zip-filename", - rootInputModule.getWorkDir().resolve("dependency-files.zip").toString(), + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), "--directory", rootInputModule.getBaseDir().toString(), + "--recursive", "--exclude", - "ignored.txt,.scannerwork/**", - "--debug"); + "ignored.txt,.scannerwork/**"); assertThat(logTester.logs(INFO)) .contains("Arguments Passed In: " + String.join(" ", expectedArguments)) @@ -217,87 +217,120 @@ class CliServiceTest { } @Test - void generateZip_withNoScm_doesNotIncludeScmIgnoredPaths() throws Exception { + void generateManifestsArchive_withNoScm_doesNotIncludeScmIgnoredPaths() throws Exception { when(scmConfiguration.provider()).thenReturn(null); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); - assertThat(capturedArgs).contains("--exclude .scannerwork/** --debug"); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); } @Test - void generateZip_withNonGit_doesNotIncludeScmIgnoredPaths() throws Exception { + void generateManifestsArchive_withNonGit_doesNotIncludeScmIgnoredPaths() throws Exception { when(scmProvider.key()).thenReturn("notgit"); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); - assertThat(capturedArgs).contains("--exclude .scannerwork/** --debug"); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); } @Test - void generateZip_withExclusionDisabled_doesNotIncludeScmIgnoredPaths() throws Exception { + void generateManifestsArchive_withScmExclusionDisabled_doesNotIncludeScmIgnoredPaths() throws Exception { when(scmConfiguration.isExclusionDisabled()).thenReturn(true); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); - assertThat(capturedArgs).contains("--exclude .scannerwork/** --debug"); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); } @Test - void generateZip_withNoScmIgnores_doesNotIncludeScmIgnoredPaths() throws Exception { + void generateManifestsArchive_withNoScmIgnores_doesNotIncludeScmIgnoredPaths() throws Exception { jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of()); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); - assertThat(capturedArgs).contains("--exclude .scannerwork/** --debug"); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); } @Test - void generateZip_withExistingExcludedManifests_appendsScmIgnoredPaths() throws Exception { - when(configuration.getStringArray(CliService.EXCLUDED_MANIFESTS_PROP_KEY)).thenReturn(new String[] {"**/test/**"}); + void generateManifestsArchive_withExcludedManifests_appendsScmIgnoredPaths() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test/**"}); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); assertThat(capturedArgs).contains("--exclude **/test/**,ignored.txt,.scannerwork/**"); } @Test - void generateZip_withExcludedManifestsSettingContainingBadCharacters_handlesTheBadCharacters() throws Exception { - when(configuration.getStringArray(CliService.EXCLUDED_MANIFESTS_PROP_KEY)).thenReturn(new String[] { - "**/test/**", "**/path with spaces/**", "**/path,with,commas/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**"}); + void generateManifestsArchive_withExcludedManifestsContainingBadCharacters_handlesTheBadCharacters() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] { + "**/test/**", "**/path with spaces/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**"}); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); String expectedExcludeFlag = """ - --exclude **/test/**,**/path with spaces/**,"**/path,with,commas/**",**/path'with'quotes/**,"**/path""with""double""quotes/**",ignored.txt + --exclude **/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**",ignored.txt """.strip(); + if (SystemUtils.IS_OS_WINDOWS) { + expectedExcludeFlag = """ + --exclude "**/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**",ignored.txt + """.strip(); + } assertThat(capturedArgs).contains(expectedExcludeFlag); } @Test - void generateZip_withScmIgnoresContainingBadCharacters_handlesTheBadCharacters() throws Exception { + void generateManifestsArchive_withExcludedManifestsContainingDupes_dedupes() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test2/**", "**/test1/**"}); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test3/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude **/test1/**,**/test2/**,**/test3/**,ignored.txt,.scannerwork/**"); + } + + @Test + void generateManifestsArchive_withExcludedManifestsAndSonarExcludesContainingDupes_mergesAndDedupes() throws Exception { + when(projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN)).thenReturn(new String[] {"**/test1/**", "**/test4/**"}); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test2/**", "**/test1/**"}); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test3/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude **/test1/**,**/test4/**,**/test2/**,**/test3/**,ignored.txt,.scannerwork/**"); + } + + @Test + void generateManifestsArchive_withScmIgnoresContainingBadCharacters_handlesTheBadCharacters() throws Exception { jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))) - .thenReturn(List.of("**/test/**", "**/path with spaces/**", "**/path,with,commas/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**")); + .thenReturn(List.of("**/test/**", "**/path with spaces/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**")); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); String expectedExcludeFlag = """ - --exclude **/test/**,**/path with spaces/**,"**/path,with,commas/**",**/path'with'quotes/**,"**/path""with""double""quotes/**" + --exclude **/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**" """.strip(); + if (SystemUtils.IS_OS_WINDOWS) { + expectedExcludeFlag = """ + --exclude "**/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**" + """.strip(); + } assertThat(capturedArgs).contains(expectedExcludeFlag); } @Test - void generateZip_withIgnoredDirectories_GlobifiesDirectories() throws Exception { + void generateManifestsArchive_withIgnoredDirectories_GlobifiesDirectories() throws Exception { String ignoredDirectory = "directory1"; Files.createDirectories(rootModuleDir.resolve(ignoredDirectory)); String ignoredFile = "directory2/file.txt"; @@ -306,22 +339,22 @@ class CliServiceTest { Files.createFile(ignoredFilePath); jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of(ignoredDirectory, ignoredFile)); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); assertThat(capturedArgs).contains("--exclude directory1/**,directory2/file.txt"); } @Test - void generateZip_withExternalWorkDir_DoesNotExcludeWorkingDir() throws URISyntaxException, IOException { + void generateManifestsArchive_withExternalWorkDir_DoesNotExcludeWorkingDir() throws URISyntaxException, IOException { Path externalWorkDir = Files.createTempDirectory("externalWorkDir"); try { rootInputModule = new DefaultInputModule(ProjectDefinition.create().setBaseDir(rootModuleDir.toFile()).setWorkDir(externalWorkDir.toFile())); - underTest.generateManifestsZip(rootInputModule, scriptDir(), configuration); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); // externalWorkDir is not present in the exclude flag - assertThat(capturedArgs).contains("--exclude ignored.txt --debug"); + assertThat(capturedArgs).contains("--exclude ignored.txt"); } finally { externalWorkDir.toFile().delete(); } @@ -330,7 +363,7 @@ class CliServiceTest { private URL scriptUrl() { // There is a custom test Bash script available in src/test/resources/org/sonar/scanner/sca that // will serve as our "CLI". This script will output some messages about what arguments were passed - // to it and will try to generate a zip file in the location the process specifies. This allows us + // to it and will try to generate an archive file in the location the process specifies. This allows us // to simulate a real CLI call without needing an OS specific CLI executable to run on a real project. URL scriptUrl = CliServiceTest.class.getResource(SystemUtils.IS_OS_WINDOWS ? "echo_args.bat" : "echo_args.sh"); assertThat(scriptUrl).isNotNull(); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java index aecbc7011a9..ebe6007a1c1 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java @@ -70,14 +70,14 @@ class ScaExecutorTest { File mockManifestZip = Files.newTemporaryFile(); ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); when(cliCacheService.cacheCli()).thenReturn(mockCliFile); - when(cliService.generateManifestsZip(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); when(reportPublisher.getWriter()).thenReturn(mockReportWriter); logTester.setLevel(Level.DEBUG); underTest.execute(root); - verify(cliService).generateManifestsZip(root, mockCliFile, configuration); + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); verify(mockReportWriter).writeScaFile(mockManifestZip); assertThat(logTester.logs(Level.DEBUG)).contains("Zip ready for report: " + mockManifestZip); assertThat(logTester.logs(Level.DEBUG)).contains("Manifest zip written to report"); @@ -87,13 +87,13 @@ class ScaExecutorTest { void execute_whenIOException_shouldHandleException() throws IOException { File mockCliFile = Files.newTemporaryFile(); when(cliCacheService.cacheCli()).thenReturn(mockCliFile); - doThrow(IOException.class).when(cliService).generateManifestsZip(root, mockCliFile, configuration); + doThrow(IOException.class).when(cliService).generateManifestsArchive(root, mockCliFile, configuration); logTester.setLevel(Level.INFO); underTest.execute(root); - verify(cliService).generateManifestsZip(root, mockCliFile, configuration); + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); assertThat(logTester.logs(Level.ERROR)).contains("Error gathering manifests"); } @@ -101,13 +101,13 @@ class ScaExecutorTest { void execute_whenIllegalStateException_shouldHandleException() throws IOException { File mockCliFile = Files.newTemporaryFile(); when(cliCacheService.cacheCli()).thenReturn(mockCliFile); - doThrow(IllegalStateException.class).when(cliService).generateManifestsZip(root, mockCliFile, configuration); + doThrow(IllegalStateException.class).when(cliService).generateManifestsArchive(root, mockCliFile, configuration); logTester.setLevel(Level.INFO); underTest.execute(root); - verify(cliService).generateManifestsZip(root, mockCliFile, configuration); + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); assertThat(logTester.logs(Level.ERROR)).contains("Error gathering manifests"); } @@ -118,7 +118,7 @@ class ScaExecutorTest { underTest.execute(root); - verify(cliService, never()).generateManifestsZip(root, mockCliFile, configuration); + verify(cliService, never()).generateManifestsArchive(root, mockCliFile, configuration); } @Test @@ -150,15 +150,31 @@ class ScaExecutorTest { File mockManifestZip = Files.newTemporaryFile(); ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); when(cliCacheService.cacheCli()).thenReturn(mockCliFile); - when(cliService.generateManifestsZip(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); when(reportPublisher.getWriter()).thenReturn(mockReportWriter); logTester.setLevel(Level.DEBUG); underTest.execute(root); - verify(cliService).generateManifestsZip(root, mockCliFile, configuration); + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); verify(mockReportWriter).writeScaFile(mockManifestZip); assertThat(logTester.logs(Level.DEBUG)).contains("Zip ready for report: " + mockManifestZip); assertThat(logTester.logs(Level.DEBUG)).contains("Manifest zip written to report"); } + + @Test + void execute_printsRuntime() throws IOException { + File mockCliFile = Files.newTemporaryFile(); + File mockManifestZip = Files.newTemporaryFile(); + ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(reportPublisher.getWriter()).thenReturn(mockReportWriter); + + logTester.setLevel(Level.INFO); + + underTest.execute(root); + + assertThat(logTester.logs(Level.INFO)).anyMatch(l -> l.matches("Load SCA project dependencies \\(done\\) \\| time=\\d+ms")); + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java index e598a225b9c..70e7a6b6e53 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java @@ -19,11 +19,9 @@ */ package org.sonar.scanner.sca; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; import org.junit.jupiter.api.Test; import org.sonar.scanner.config.DefaultConfiguration; @@ -36,13 +34,12 @@ class ScaPropertiesTest { private final DefaultConfiguration configuration = mock(DefaultConfiguration.class); @Test - void buildFromScannerProperties_withNoProperties_returnsDefaultMap() { + void buildFromScannerProperties_withNoProperties_returnsEmptyMap() { when(configuration.get(anyString())).thenReturn(Optional.empty()); - var result = ScaProperties.buildFromScannerProperties(configuration, Collections.emptySet()); + var result = ScaProperties.buildFromScannerProperties(configuration); - assertThat(result).containsExactly( - Map.entry("TIDELIFT_RECURSIVE_MANIFEST_SEARCH", "true")); + assertThat(result).isEqualTo(Map.of()); } @Test @@ -51,14 +48,13 @@ class ScaPropertiesTest { inputProperties.put("sonar.sca.pythonBinary", "/usr/bin/python3"); inputProperties.put("sonar.sca.unknownProperty", "value"); inputProperties.put("sonar.somethingElse", "dont-include-non-sca"); - inputProperties.put("sonar.sca.ignoredProperty", "ignore-me"); + inputProperties.put("sonar.sca.recursiveManifestSearch", "ignore-me"); when(configuration.getProperties()).thenReturn(inputProperties); when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); - var result = ScaProperties.buildFromScannerProperties(configuration, Set.of("sonar.sca.ignoredProperty")); + var result = ScaProperties.buildFromScannerProperties(configuration); assertThat(result).containsExactly( - Map.entry("TIDELIFT_RECURSIVE_MANIFEST_SEARCH", "true"), Map.entry("TIDELIFT_PYTHON_BINARY", "/usr/bin/python3"), Map.entry("TIDELIFT_UNKNOWN_PROPERTY", "value")); } @@ -79,7 +75,6 @@ class ScaPropertiesTest { inputProperties.put("sonar.sca.pythonBinary", "/usr/bin/python3"); inputProperties.put("sonar.sca.pythonNoResolve", "true"); inputProperties.put("sonar.sca.pythonResolveLocal", "false"); - inputProperties.put("sonar.sca.recursiveManifestSearch", "true"); when(configuration.getProperties()).thenReturn(inputProperties); when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); @@ -97,37 +92,9 @@ class ScaPropertiesTest { expectedProperties.put("TIDELIFT_PYTHON_BINARY", "/usr/bin/python3"); expectedProperties.put("TIDELIFT_PYTHON_NO_RESOLVE", "true"); expectedProperties.put("TIDELIFT_PYTHON_RESOLVE_LOCAL", "false"); - expectedProperties.put("TIDELIFT_RECURSIVE_MANIFEST_SEARCH", "true"); - var result = ScaProperties.buildFromScannerProperties(configuration, Collections.emptySet()); + var result = ScaProperties.buildFromScannerProperties(configuration); assertThat(result).containsExactlyInAnyOrderEntriesOf(expectedProperties); } - - - @Test - void buildFromScannerProperties_withoutRecursiveModeProp_defaultsRecursiveModeTrue() { - var inputProperties = new HashMap<String, String>(); - when(configuration.getProperties()).thenReturn(inputProperties); - when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); - - var result = ScaProperties.buildFromScannerProperties(configuration, Collections.emptySet()); - - assertThat(result).containsExactly( - Map.entry("TIDELIFT_RECURSIVE_MANIFEST_SEARCH", "true")); - } - - @Test - void buildFromScannerProperties_withRecursiveModeProp_usesPropAsOverride() { - var inputProperties = new HashMap<String, String>(); - inputProperties.put("sonar.sca.recursiveManifestSearch", "false"); - when(configuration.getProperties()).thenReturn(inputProperties); - when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); - - var result = ScaProperties.buildFromScannerProperties(configuration, Collections.emptySet()); - - assertThat(result).containsExactly( - Map.entry("TIDELIFT_RECURSIVE_MANIFEST_SEARCH", "false")); - } - } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java index 21dcf58b114..3e3066e76e2 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java @@ -52,10 +52,9 @@ public class ProjectConfigurationProviderTest { private static final Map<String, String> PROJECT_SERVER_PROPERTIES = Map.of(NON_GLOBAL_KEY_PROPERTIES_1, NON_GLOBAL_VALUE_PROPERTIES_1); private static final Map<String, String> DEFAULT_PROJECT_PROPERTIES = Map.of(DEFAULT_KEY_PROPERTIES_1, DEFAULT_VALUE_1); - private static final Map<String, String> ALL_PROPERTIES_MAP = - Stream.of(GLOBAL_SERVER_PROPERTIES, PROJECT_SERVER_PROPERTIES, DEFAULT_PROJECT_PROPERTIES) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private static final Map<String, String> ALL_PROPERTIES_MAP = Stream.of(GLOBAL_SERVER_PROPERTIES, PROJECT_SERVER_PROPERTIES, DEFAULT_PROJECT_PROPERTIES) + .flatMap(map -> map.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); private static final Map<String, String> PROPERTIES_AFTER_FILTERING = Map.of("aKey", "aValue"); @@ -66,8 +65,6 @@ public class ProjectConfigurationProviderTest { @Mock private GlobalConfiguration globalConfiguration; @Mock - private MutableProjectSettings mutableProjectSettings; - @Mock private DefaultInputProject defaultInputProject; @Mock private SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -75,7 +72,6 @@ public class ProjectConfigurationProviderTest { @InjectMocks private ProjectConfigurationProvider provider; - @Before public void init() { when(globalConfiguration.getDefinitions()).thenReturn(new PropertyDefinitions(System2.INSTANCE)); @@ -89,11 +85,11 @@ public class ProjectConfigurationProviderTest { when(sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(ALL_PROPERTIES_MAP, GLOBAL_SERVER_PROPERTIES)) .thenReturn(PROPERTIES_AFTER_FILTERING); - ProjectConfiguration provide = provider.provide(defaultInputProject, globalConfiguration, globalServerSettings, projectServerSettings, mutableProjectSettings); + ProjectConfiguration provide = provider.provide(defaultInputProject, globalConfiguration, globalServerSettings, projectServerSettings); verify(sonarGlobalPropertiesFilter).enforceOnlyServerSideSonarGlobalPropertiesAreUsed(ALL_PROPERTIES_MAP, GLOBAL_SERVER_PROPERTIES); assertThat(provide.getOriginalProperties()).containsExactlyEntriesOf(PROPERTIES_AFTER_FILTERING); } -}
\ No newline at end of file +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java index 01cb7d1bccf..277fc0dc68a 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java @@ -27,20 +27,26 @@ import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Optional; import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DirectoryFileVisitorTest { @@ -48,33 +54,57 @@ public class DirectoryFileVisitorTest { public static TemporaryFolder temp = new TemporaryFolder(); private final DefaultInputModule module = mock(); + private final ModuleConfiguration moduleConfiguration = mock(); private final ModuleExclusionFilters moduleExclusionFilters = mock(); private final InputModuleHierarchy inputModuleHierarchy = mock(); private final InputFile.Type type = mock(); + private final SonarUserHome sonarUserHome = mock(); + private HiddenFilesProjectData hiddenFilesProjectData; + + @Before + public void before() throws IOException { + Path sonarUserHomePath = temp.newFolder().toPath(); + when(sonarUserHome.getPath()).thenReturn(sonarUserHomePath); + File workDir = temp.newFolder(); + when(module.getWorkDir()).thenReturn(workDir.toPath()); + hiddenFilesProjectData = spy(new HiddenFilesProjectData(sonarUserHome)); + } @Test - public void visit_hidden_file() throws IOException { + public void should_not_visit_hidden_file() throws IOException { + when(moduleConfiguration.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); - File hidden = temp.newFile(".hidden"); - if (SystemUtils.IS_OS_WINDOWS) { - Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); - } - + File hidden = temp.newFile(".hiddenNotVisited"); + setAsHiddenOnWindows(hidden); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); verify(action, never()).execute(any(Path.class)); } @Test + public void should_visit_hidden_file() throws IOException { + when(moduleConfiguration.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(false)); + DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); + + File hidden = temp.newFile(".hiddenVisited"); + setAsHiddenOnWindows(hidden); + + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); + underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); + + verify(action).execute(any(Path.class)); + } + + @Test public void test_visit_file_failed_generic_io_exception() throws IOException { DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); File file = temp.newFile("failed"); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException())); } @@ -84,10 +114,15 @@ public class DirectoryFileVisitorTest { File file = temp.newFile("symlink"); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath())); assertThat(result).isEqualTo(FileVisitResult.CONTINUE); } + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java new file mode 100644 index 00000000000..d5a6e4ff843 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java @@ -0,0 +1,160 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class HiddenFilesProjectDataTest { + + @ClassRule + public static TemporaryFolder temp = new TemporaryFolder(); + + private static final SonarUserHome sonarUserHome = mock(SonarUserHome.class); + private final DefaultInputModule inputModule = mock(DefaultInputModule.class); + private final DefaultInputModule secondInputModule = mock(DefaultInputModule.class); + private HiddenFilesProjectData underTest; + + @BeforeClass + public static void setUp() throws IOException { + File userHomeFolder = temp.newFolder(".userhome"); + setAsHiddenOnWindows(userHomeFolder); + when(sonarUserHome.getPath()).thenReturn(userHomeFolder.toPath()); + } + + @Before + public void before() { + underTest = spy(new HiddenFilesProjectData(sonarUserHome)); + } + + @Test + public void shouldContainNoMarkedHiddenFileOnConstruction() { + assertThat(underTest.hiddenFilesByModule).isEmpty(); + } + + @Test + public void shouldMarkWithCorrectAssociatedInputModule() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isFalse(); + } + + @Test + public void shouldMarkWithCorrectAssociatedInputModuleForTwoDifferentModules() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, secondInputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(2); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isTrue(); + } + + @Test + public void shouldNotShowAsHiddenFileWhenInputModuleIsNotExistingInData() { + Path myFile = Path.of("myFile"); + Path notMarkedFile = Path.of("notMarkedFile"); + underTest.markAsHiddenFile(myFile, inputModule); + + assertThat(underTest.hiddenFilesByModule).isNotEmpty(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(notMarkedFile, secondInputModule)).isFalse(); + } + + @Test + public void shouldClearMap() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, secondInputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(2); + + underTest.clearHiddenFilesData(); + assertThat(underTest.hiddenFilesByModule).isEmpty(); + } + + @Test + public void shouldRemoveVisibilityAfterQuerying() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.hiddenFilesByModule.get(inputModule)).isEmpty(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + } + + @Test + public void shouldOnlyRemoveModuleIfAllFilesAreRemoved() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).isNotEmpty(); + } + + @Test + public void shouldNotFailOnUserPathResolving() throws IOException { + Path expectedPath = sonarUserHome.getPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + assertThat(underTest.getCachedSonarUserHomePath()).isEqualTo(expectedPath); + } + + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java new file mode 100644 index 00000000000..8c111c7ea15 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java @@ -0,0 +1,315 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; +import org.sonar.scanner.scan.ModuleConfiguration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class HiddenFilesVisitorHelperTest { + + @ClassRule + public static TemporaryFolder temp = new TemporaryFolder(); + + private static final SonarUserHome sonarUserHome = mock(SonarUserHome.class); + private static final DefaultInputModule inputModule = mock(DefaultInputModule.class); + + private final ModuleConfiguration moduleConfig = mock(ModuleConfiguration.class); + private final HiddenFilesProjectData hiddenFilesProjectData = spy(new HiddenFilesProjectData(sonarUserHome)); + private HiddenFilesVisitorHelper underTest; + + @BeforeClass + public static void setUp() throws IOException { + File userHomeFolder = temp.newFolder(".userhome"); + setAsHiddenOnWindows(userHomeFolder); + when(sonarUserHome.getPath()).thenReturn(userHomeFolder.toPath()); + + File workDir = temp.newFolder(".sonar"); + setAsHiddenOnWindows(workDir); + when(inputModule.getWorkDir()).thenReturn(workDir.toPath()); + } + + @Before + public void before() { + hiddenFilesProjectData.clearHiddenFilesData(); + underTest = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + } + + @Test + public void verifyDefaultOnConstruction() { + assertThat(underTest.excludeHiddenFiles).isFalse(); + assertThat(underTest.rootHiddenDir).isNull(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToFalseFromConfigurationWhenNotConfigured() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.empty()); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isFalse(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToFalseFromConfigurationWhenDisabled() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(false)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isFalse(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToTrueFromConfigurationWhenEnabled() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isTrue(); + } + + @Test + public void shouldVisitHiddenDirectory() throws IOException { + File hiddenDir = temp.newFolder(".hiddenVisited"); + setAsHiddenOnWindows(hiddenDir); + + boolean visitDir = underTest.shouldVisitDir(hiddenDir.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenDir.toPath()); + verify(underTest).enterHiddenDirectory(hiddenDir.toPath()); + } + + @Test + public void shouldNotVisitHiddenDirectoryWhenHiddenFilesVisitIsExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File hidden = temp.newFolder(".hiddenNotVisited"); + setAsHiddenOnWindows(hidden); + + boolean visitDir = configuredVisitorHelper.shouldVisitDir(hidden.toPath()); + + assertThat(visitDir).isFalse(); + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + verify(configuredVisitorHelper, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldVisitNonHiddenDirectoryWhenHiddenFilesVisitIsExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File nonHiddenFolder = temp.newFolder(); + + boolean visitDir = configuredVisitorHelper.shouldVisitDir(nonHiddenFolder.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + verify(configuredVisitorHelper, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldVisitNonHiddenDirectory() throws IOException { + File nonHiddenFolder = temp.newFolder(); + + boolean visitDir = underTest.shouldVisitDir(nonHiddenFolder.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + assertThat(underTest.excludeHiddenFiles).isFalse(); + } + + @Test + public void shouldNotVisitModuleWorkDir() throws IOException { + Path workingDirectory = inputModule.getWorkDir().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + boolean visitDir = underTest.shouldVisitDir(workingDirectory); + + assertThat(visitDir).isFalse(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldNotVisitSonarUserHome() throws IOException { + Path userHome = sonarUserHome.getPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + boolean visitDir = underTest.shouldVisitDir(userHome); + + assertThat(visitDir).isFalse(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + } + + @Test + public void hiddenFileShouldBeVisited() throws IOException { + File hiddenFile = temp.newFile(".hiddenFileShouldBeVisited"); + setAsHiddenOnWindows(hiddenFile); + + assertThat(underTest.insideHiddenDirectory()).isFalse(); + boolean visitFile = underTest.shouldVisitFile(hiddenFile.toPath()); + + assertThat(visitFile).isTrue(); + verify(hiddenFilesProjectData).markAsHiddenFile(hiddenFile.toPath(), inputModule); + } + + @Test + public void nonHiddenFileShouldBeVisitedInHiddenFolder() throws IOException { + File hidden = temp.newFolder(".hiddenFolder"); + setAsHiddenOnWindows(hidden); + + File nonHiddenFile = temp.newFile(); + + underTest.shouldVisitDir(hidden.toPath()); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + + boolean shouldVisitFile = underTest.shouldVisitFile(nonHiddenFile.toPath()); + + assertThat(shouldVisitFile).isTrue(); + verify(hiddenFilesProjectData).markAsHiddenFile(nonHiddenFile.toPath(), inputModule); + } + + @Test + public void shouldNotSetAsRootHiddenDirectoryWhenAlreadyEnteredHiddenDirectory() throws IOException { + File hidden = temp.newFolder(".outerHiddenFolder"); + File nestedHiddenFolder = temp.newFolder(".outerHiddenFolder", ".nestedHiddenFolder"); + setAsHiddenOnWindows(hidden); + setAsHiddenOnWindows(nestedHiddenFolder); + + underTest.shouldVisitDir(hidden.toPath()); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + + boolean shouldVisitNestedDir = underTest.shouldVisitDir(nestedHiddenFolder.toPath()); + + assertThat(shouldVisitNestedDir).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hidden.toPath()); + verify(underTest).enterHiddenDirectory(nestedHiddenFolder.toPath()); + } + + @Test + public void hiddenFileShouldNotBeVisitedWhenHiddenFileVisitExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File hiddenFile = temp.newFile(".hiddenFileNotVisited"); + setAsHiddenOnWindows(hiddenFile); + + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + + configuredVisitorHelper.shouldVisitFile(hiddenFile.toPath()); + boolean shouldVisitFile = configuredVisitorHelper.shouldVisitFile(hiddenFile.toPath()); + + assertThat(shouldVisitFile).isFalse(); + verify(hiddenFilesProjectData, never()).markAsHiddenFile(hiddenFile.toPath(), inputModule); + } + + @Test + public void shouldCorrectlyExitHiddenFolderOnlyOnHiddenFolderThatEntered() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenRootFolder"); + setAsHiddenOnWindows(hiddenFolder); + + boolean shouldVisitDir = underTest.shouldVisitDir(hiddenFolder.toPath()); + + assertThat(shouldVisitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenFolder.toPath()); + verify(underTest).enterHiddenDirectory(hiddenFolder.toPath()); + + File folder1 = temp.newFolder(".hiddenRootFolder", "myFolderExit"); + File folder2 = temp.newFolder("myFolderExit"); + File folder3 = temp.newFolder(".myFolderExit"); + setAsHiddenOnWindows(folder3); + + underTest.exitDirectory(folder1.toPath()); + underTest.exitDirectory(folder2.toPath()); + underTest.exitDirectory(folder3.toPath()); + + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenFolder.toPath()); + verify(underTest, never()).resetRootHiddenDir(); + + underTest.exitDirectory(hiddenFolder.toPath()); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + assertThat(underTest.rootHiddenDir).isNull(); + verify(underTest).resetRootHiddenDir(); + } + + @Test + public void shouldNotInitiateResetRootDirWhenNotInHiddenDirectory() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenFolderNonRoot"); + setAsHiddenOnWindows(hiddenFolder); + + underTest.exitDirectory(hiddenFolder.toPath()); + + verify(underTest, never()).resetRootHiddenDir(); + } + + @Test + public void filesShouldBeCorrectlyMarkedAsHidden() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenFolderRoot"); + setAsHiddenOnWindows(hiddenFolder); + + File file1 = temp.newFile(); + File file2 = temp.newFile(); + File file3 = temp.newFile(".markedHiddenFile"); + setAsHiddenOnWindows(file3); + File file4 = temp.newFile(); + File file5 = temp.newFile(".markedHiddenFile2"); + setAsHiddenOnWindows(file5); + + underTest.shouldVisitFile(file1.toPath()); + underTest.shouldVisitDir(hiddenFolder.toPath()); + underTest.shouldVisitFile(file2.toPath()); + underTest.shouldVisitFile(file3.toPath()); + underTest.exitDirectory(hiddenFolder.toPath()); + underTest.shouldVisitFile(file4.toPath()); + underTest.shouldVisitFile(file5.toPath()); + + verify(hiddenFilesProjectData, never()).markAsHiddenFile(file1.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file2.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file3.toPath(), inputModule); + verify(hiddenFilesProjectData, never()).markAsHiddenFile(file4.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file5.toPath(), inputModule); + } + + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java index a0031f77633..07f7afec036 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java @@ -19,80 +19,151 @@ */ package org.sonar.scanner.scan.filesystem; -import java.io.IOException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.fs.internal.SensorStrategy; -import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; import org.sonar.scanner.scan.branch.BranchConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class ModuleInputComponentStoreTest { - @Rule - public TemporaryFolder temp = new TemporaryFolder(); +@ExtendWith(MockitoExtension.class) +class ModuleInputComponentStoreTest { + + @TempDir + private File projectBaseDir; + + @Mock + BranchConfiguration branchConfiguration; + + @Mock + SonarRuntime sonarRuntime; + + @Mock + InputComponentStore mockedInputComponentStore; private InputComponentStore componentStore; + private SensorContextTester sensorContextTester; private final String projectKey = "dummy key"; - @Before - public void setUp() throws IOException { - DefaultInputProject root = TestInputFileBuilder.newDefaultInputProject(projectKey, temp.newFolder()); - componentStore = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); + @BeforeEach + void setUp() { + TestInputFileBuilder.newDefaultInputProject(projectKey, projectBaseDir); + File moduleBaseDir = new File(projectBaseDir, "module"); + moduleBaseDir.mkdir(); + sensorContextTester = SensorContextTester.create(moduleBaseDir); + componentStore = spy(new InputComponentStore(branchConfiguration, sonarRuntime)); } @Test - public void should_cache_files_by_filename() { + void should_cache_module_files_by_filename() { ModuleInputComponentStore store = newModuleInputComponentStore(); String filename = "some name"; - InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile1); - InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/" + filename).build(); store.doAdd(inputFile2); - InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "some/path/Dummy.java").build(); + InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "module/some/path/Dummy.java").build(); store.doAdd(dummyInputFile); assertThat(store.getFilesByName(filename)).containsExactlyInAnyOrder(inputFile1, inputFile2); } @Test - public void should_cache_files_by_extension() { + void should_cache_filtered_module_files_by_filename() { + ModuleInputComponentStore store = newModuleInputComponentStore(); + + String filename = "some name"; + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/" + filename).build(); + store.doAdd(inputFile2); + + when(componentStore.getFilesByName(filename)).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByName(filename)).containsOnly(inputFile2); + } + + @Test + void should_cache_module_files_by_filename_global_strategy() { + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), componentStore, new SensorStrategy()); + + String filename = "some name"; + // None in the module + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/" + filename).build(); + + when(componentStore.getFilesByName(filename)).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByName(filename)).containsExactlyInAnyOrder(inputFile1, inputFile2); + } + + @Test + void should_cache_module_files_by_extension() { ModuleInputComponentStore store = newModuleInputComponentStore(); - InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/Program.java").build(); + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "module/some/path/Program.java").build(); store.doAdd(inputFile1); - InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/Utils.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/Utils.java").build(); store.doAdd(inputFile2); - InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "some/path/NotJava.cpp").build(); + InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "module/some/path/NotJava.cpp").build(); store.doAdd(dummyInputFile); assertThat(store.getFilesByExtension("java")).containsExactlyInAnyOrder(inputFile1, inputFile2); } @Test - public void should_not_cache_duplicates() { + void should_cache_filtered_module_files_by_extension() { + ModuleInputComponentStore store = newModuleInputComponentStore(); + + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/NotInModule.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/some/path/Other.java").build(); + store.doAdd(inputFile2); + + when(componentStore.getFilesByExtension("java")).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByExtension("java")).containsOnly(inputFile2); + } + + @Test + void should_cache_module_files_by_extension_global_strategy() { + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), componentStore, new SensorStrategy()); + + // None in the module + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/NotInModule.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "some/path/Other.java").build(); + + when(componentStore.getFilesByExtension("java")).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByExtension("java")).containsExactlyInAnyOrder(inputFile1, inputFile2); + } + + @Test + void should_not_cache_duplicates() { ModuleInputComponentStore store = newModuleInputComponentStore(); String ext = "java"; String filename = "Program." + ext; - InputFile inputFile = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile); store.doAdd(inputFile); store.doAdd(inputFile); @@ -102,12 +173,12 @@ public class ModuleInputComponentStoreTest { } @Test - public void should_get_empty_iterable_on_cache_miss() { + void should_get_empty_iterable_on_cache_miss() { ModuleInputComponentStore store = newModuleInputComponentStore(); String ext = "java"; String filename = "Program." + ext; - InputFile inputFile = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile); assertThat(store.getFilesByName("nonexistent")).isEmpty(); @@ -115,48 +186,42 @@ public class ModuleInputComponentStoreTest { } private ModuleInputComponentStore newModuleInputComponentStore() { - InputModule module = mock(InputModule.class); - when(module.key()).thenReturn("moduleKey"); - return new ModuleInputComponentStore(module, componentStore, mock(SensorStrategy.class)); + SensorStrategy strategy = new SensorStrategy(); + strategy.setGlobal(false); + return new ModuleInputComponentStore(sensorContextTester.module(), componentStore, strategy); } @Test - public void should_find_module_components_with_non_global_strategy() { - InputComponentStore inputComponentStore = mock(InputComponentStore.class); + void should_find_module_components_with_non_global_strategy() { SensorStrategy strategy = new SensorStrategy(); - InputModule module = mock(InputModule.class); - when(module.key()).thenReturn("foo"); - ModuleInputComponentStore store = new ModuleInputComponentStore(module, inputComponentStore, strategy); + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), mockedInputComponentStore, strategy); strategy.setGlobal(false); store.inputFiles(); - verify(inputComponentStore).filesByModule("foo"); + verify(mockedInputComponentStore).filesByModule(sensorContextTester.module().key()); String relativePath = "somepath"; store.inputFile(relativePath); - verify(inputComponentStore).getFile(any(String.class), eq(relativePath)); + verify(mockedInputComponentStore).getFile(any(String.class), eq(relativePath)); store.languages(); - verify(inputComponentStore).languages(any(String.class)); + verify(mockedInputComponentStore).languages(any(String.class)); } @Test - public void should_find_all_components_with_global_strategy() { - InputComponentStore inputComponentStore = mock(InputComponentStore.class); + void should_find_all_components_with_global_strategy() { SensorStrategy strategy = new SensorStrategy(); - ModuleInputComponentStore store = new ModuleInputComponentStore(mock(InputModule.class), inputComponentStore, strategy); - - strategy.setGlobal(true); + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), mockedInputComponentStore, strategy); store.inputFiles(); - verify(inputComponentStore).inputFiles(); + verify(mockedInputComponentStore).inputFiles(); String relativePath = "somepath"; store.inputFile(relativePath); - verify(inputComponentStore).inputFile(relativePath); + verify(mockedInputComponentStore).inputFile(relativePath); store.languages(); - verify(inputComponentStore).languages(); + verify(mockedInputComponentStore).languages(); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java index 31d3312853b..485708c9936 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java @@ -28,6 +28,9 @@ import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; public class MutableFileSystemTest { @@ -44,9 +47,15 @@ public class MutableFileSystemTest { } @Test - public void return_all_files_when_not_restricted() { + public void restriction_and_hidden_file_should_be_disabled_on_default() { + assertThat(underTest.restrictToChangedFiles).isFalse(); + assertThat(underTest.allowHiddenFileAnalysis).isFalse(); + } + + @Test + public void return_all_non_hidden_files_when_not_restricted_and_disabled() { assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); - addFileWithAllStatus(); + addFilesWithAllStatus(); underTest.setRestrictToChangedFiles(false); assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(3); @@ -58,7 +67,7 @@ public class MutableFileSystemTest { @Test public void return_only_changed_files_when_restricted() { assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); - addFileWithAllStatus(); + addFilesWithAllStatus(); underTest.setRestrictToChangedFiles(true); assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(2); @@ -67,19 +76,95 @@ public class MutableFileSystemTest { assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(InputFile.Status.CHANGED)))).isNotNull(); } - private void addFileWithAllStatus() { + @Test + public void return_all_files_when_allowing_hidden_files_analysis() { + assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); + addFilesWithVisibility(); + underTest.setAllowHiddenFileAnalysis(true); + + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(2); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNotNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + } + + @Test + public void return_only_non_hidden_files_when_not_allowing_hidden_files_analysis() { + assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); + addFilesWithVisibility(); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(1); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + } + + @Test + public void hidden_file_predicate_should_preserve_predicate_optimization() { + addFilesWithVisibility(); + var anotherHiddenFile = spy(new TestInputFileBuilder("foo", String.format("src/%s", ".myHiddenFile.txt")) + .setLanguage(LANGUAGE).setStatus(InputFile.Status.ADDED).setHidden(true).build()); + underTest.add(anotherHiddenFile); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + // Verify that predicate optimization is still effective + verify(anotherHiddenFile, never()).isHidden(); + + // This predicate can't be optimized + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(1); + verify(anotherHiddenFile).isHidden(); + } + + @Test + public void hidden_file_predicate_should_be_applied_first_for_non_optimized_predicates() { + // Checking the file type is not very costly, but it is not optimized. In real life, something more costly would be reading the file + // content, for example. + addFilesWithVisibility(); + var anotherHiddenFile = spy(new TestInputFileBuilder("foo", String.format("src/%s", ".myHiddenFile." + LANGUAGE)) + .setLanguage(LANGUAGE).setType(InputFile.Type.MAIN).setStatus(InputFile.Status.ADDED).setHidden(true).build()); + underTest.add(anotherHiddenFile); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFiles(underTest.predicates().hasType(InputFile.Type.MAIN))).hasSize(1); + // Verify that the file type has not been evaluated + verify(anotherHiddenFile, never()).type(); + } + + private void addFilesWithVisibility() { + addFile(true); + addFile(false); + } + + private void addFilesWithAllStatus() { addFile(InputFile.Status.ADDED); addFile(InputFile.Status.CHANGED); addFile(InputFile.Status.SAME); } private void addFile(InputFile.Status status) { - underTest.add(new TestInputFileBuilder("foo", String.format("src/%s", generateFilename(status))) - .setLanguage(LANGUAGE).setStatus(status).build()); + addFile(status, false); + } + + private void addFile(boolean hidden) { + addFile(InputFile.Status.SAME, hidden); + } + + private void addFile(InputFile.Status status, boolean hidden) { + underTest.add(new TestInputFileBuilder("foo", String.format("src/%s", generateFilename(status, hidden))) + .setLanguage(LANGUAGE).setType(InputFile.Type.MAIN).setStatus(status).setHidden(hidden).build()); + } + + private String generateFilename(boolean hidden) { + return generateFilename(InputFile.Status.SAME, hidden); } private String generateFilename(InputFile.Status status) { - return String.format("%s.%s", status.name().toLowerCase(Locale.ROOT), LANGUAGE); + return generateFilename(status, false); + } + + private String generateFilename(InputFile.Status status, boolean hidden) { + return String.format("%s.%s.%s", status.name().toLowerCase(Locale.ROOT), hidden, LANGUAGE); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java index e769e0d6451..355fb2cde0e 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java @@ -19,16 +19,6 @@ */ package org.sonar.scanner.sensor; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.data.MapEntry.entry; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - import java.io.File; import java.util.ArrayList; import java.util.List; @@ -77,6 +67,16 @@ import org.sonar.scanner.repository.ContextPropertiesCache; import org.sonar.scanner.repository.TelemetryCache; import org.sonar.scanner.scan.branch.BranchConfiguration; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.data.MapEntry.entry; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + class DefaultSensorStorageTest { @TempDir @@ -97,7 +97,6 @@ class DefaultSensorStorageTest { public void prepare() { MetricFinder metricFinder = mock(MetricFinder.class); when(metricFinder.<Integer>findByKey(CoreMetrics.NCLOC_KEY)).thenReturn(CoreMetrics.NCLOC); - when(metricFinder.<String>findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION); when(metricFinder.<Integer>findByKey(CoreMetrics.LINES_TO_COVER_KEY)).thenReturn(CoreMetrics.LINES_TO_COVER); settings = new MapSettings(); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java index 4ab9f46fb4a..3a0ff7fb4c2 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java @@ -68,7 +68,7 @@ class ModuleSensorContextTest { @BeforeEach void prepare() { fs = new DefaultFileSystem(temp); - underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); } @@ -104,7 +104,7 @@ class ModuleSensorContextTest { @Test void pull_request_can_skip_unchanged_files() { when(branchConfiguration.isPullRequest()).thenReturn(true); - underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); assertThat(underTest.canSkipUnchangedFiles()).isTrue(); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java index 3c7f3d36793..01c337a5ed0 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java @@ -59,8 +59,8 @@ class ProjectSensorContextTest { private ExecutingSensorContext executingSensorContext = mock(ExecutingSensorContext.class); private ScannerPluginRepository pluginRepository = mock(ScannerPluginRepository.class); - private ProjectSensorContext underTest = new ProjectSensorContext(mock(DefaultInputProject.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, - branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); + private ProjectSensorContext underTest = new ProjectSensorContext(mock(DefaultInputProject.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, + branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); private static final String PLUGIN_KEY = "org.sonarsource.pluginKey"; @@ -69,7 +69,6 @@ class ProjectSensorContextTest { when(executingSensorContext.getSensorExecuting()).thenReturn(new SensorId(PLUGIN_KEY, "sensorName")); } - @Test void addTelemetryProperty_whenTheOrganizationIsSonarSource_mustStoreTheTelemetry() { @@ -77,16 +76,21 @@ class ProjectSensorContextTest { underTest.addTelemetryProperty("key", "value"); - //then verify that the defaultStorage is called with the telemetry property once + // then verify that the defaultStorage is called with the telemetry property once verify(sensorStorage).storeTelemetry("key", "value"); } @Test - void addTelemetryProperty_whenTheOrganizationIsNotSonarSource_mustThrowExcaption() { + void addTelemetryProperty_whenTheOrganizationIsNotSonarSource_mustThrowException() { when(pluginRepository.getPluginInfo(PLUGIN_KEY)).thenReturn(new PluginInfo(PLUGIN_KEY).setOrganizationName("notSonarsource")); assertThrows(IllegalStateException.class, () -> underTest.addTelemetryProperty("key", "value")); verifyNoInteractions(sensorStorage); } + + @Test + void settings_throwsUnsupportedOperationException() { + assertThrows(UnsupportedOperationException.class, () -> underTest.settings()); + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java index fe637015ed7..91f59b964b9 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java @@ -88,7 +88,7 @@ public class ChangedFileTest { secure().next(5), Integer.parseInt(secure().nextNumeric(5)), new SensorStrategy(), - oldRelativePath); + oldRelativePath, false); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java index 383e2a1d643..d8264a5745b 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import org.apache.commons.lang.SystemUtils; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.junit.jupiter.api.Test; @@ -46,7 +47,11 @@ class JGitUtilsTest { // in directory1, the entire directory is ignored without listing each file // in directory2, specific files are ignored, so those files are listed // in directory3, specific files are ignored via a separate .gitignore file - assertThat(result).isEqualTo(List.of("directory1", "directory2/file_a.txt", "directory3/file_b.txt")); + if (SystemUtils.IS_OS_WINDOWS) { + assertThat(result).isEqualTo(List.of("directory1", "directory2\\file_a.txt", "directory3\\file_b.txt")); + } else { + assertThat(result).isEqualTo(List.of("directory1", "directory2/file_a.txt", "directory3/file_b.txt")); + } } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java index 68b7e1b7bd2..23d5e6ca5f0 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java @@ -159,14 +159,14 @@ class NativeGitBlameCommandTest { ProcessWrapperFactory mockFactory = mock(ProcessWrapperFactory.class); ProcessWrapper mockProcess = mock(ProcessWrapper.class); String gitCommand = "git"; - when(mockFactory.create(any(), any(), anyString(), anyString(), anyString(), anyString(), + when(mockFactory.create(any(), any(), any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) - .then(invocation -> mockProcess); + .then(invocation -> mockProcess); NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(gitCommand, System2.INSTANCE, mockFactory); blameCommand.blame(baseDir.toPath(), DUMMY_JAVA); - verify(mockFactory).create(any(), any(), eq(gitCommand), + verify(mockFactory).create(any(), any(), any(), eq(gitCommand), eq(GIT_DIR_FLAG), eq(String.format(GIT_DIR_ARGUMENT, baseDir.toPath())), eq(GIT_DIR_FORCE_FLAG), @@ -238,7 +238,7 @@ class NativeGitBlameCommandTest { "git version 2.25.1.msysgit.2").forEach(output -> { ProcessWrapperFactory mockedCmd = mockGitVersionCommand(output); mockGitWhereOnWindows(mockedCmd); - when(mockedCmd.create(isNull(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { + when(mockedCmd.create(isNull(), any(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept(output); return mock(ProcessWrapper.class); @@ -303,7 +303,7 @@ class NativeGitBlameCommandTest { ProcessWrapper mockProcess = mock(ProcessWrapper.class); mockGitWhereOnWindows(mockFactory); - when(mockFactory.create(isNull(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { + when(mockFactory.create(isNull(), any(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept("git version 2.30.1"); return mockProcess; @@ -346,7 +346,7 @@ class NativeGitBlameCommandTest { } private void mockGitWhereOnWindows(ProcessWrapperFactory processWrapperFactory) { - when(processWrapperFactory.create(isNull(), any(), eq("C:\\Windows\\System32\\where.exe"), eq("$PATH:git.exe"))).then(invocation -> { + when(processWrapperFactory.create(isNull(), any(), any(), eq("C:\\Windows\\System32\\where.exe"), eq("$PATH:git.exe"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept("C:\\mockGit.exe"); return mock(ProcessWrapper.class); @@ -357,7 +357,7 @@ class NativeGitBlameCommandTest { ProcessWrapperFactory mockFactory = mock(ProcessWrapperFactory.class); ProcessWrapper mockProcess = mock(ProcessWrapper.class); - when(mockFactory.create(isNull(), any(), eq("git"), eq("--version"))).then(invocation -> { + when(mockFactory.create(isNull(), any(), any(), eq("git"), eq("--version"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept(commandOutput); return mockProcess; diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat index 5677cf5c437..577375b330d 100644 --- a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat @@ -6,7 +6,7 @@ set "POSITIONAL_ARGS=" :loop if "%~1"=="" goto endloop -if "%~1"=="--zip-filename" ( +if "%~1"=="--xz-filename" ( set "FILENAME=%~2" shift shift diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh index 881be2eaac5..f7feed1f501 100755 --- a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh @@ -6,7 +6,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in - --zip-filename) + --xz-filename) FILENAME="$2" shift shift diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/sonar-project.properties b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/sonar-project.properties new file mode 100644 index 00000000000..d704931fcdc --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/sonar-project.properties @@ -0,0 +1,4 @@ +sonar.projectKey=sample-with-hidden-files +sonar.projectName=Sample with hidden files +sonar.projectVersion=0.1-SNAPSHOT +sonar.sources=xources diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/.xoo new file mode 100644 index 00000000000..9d5005aafea --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/.xoo @@ -0,0 +1 @@ +Some random content
\ No newline at end of file diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/Class.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/Class.xoo new file mode 100644 index 00000000000..fe9d2e54718 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/Class.xoo @@ -0,0 +1,8 @@ +package hello; + +public class ClassOne { + + public static void main(String[] args) { + System.out.println("ClassOne"); + } +} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo new file mode 100644 index 00000000000..9d5005aafea --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo @@ -0,0 +1 @@ +Some random content
\ No newline at end of file diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo new file mode 100644 index 00000000000..fe9d2e54718 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo @@ -0,0 +1,8 @@ +package hello; + +public class ClassOne { + + public static void main(String[] args) { + System.out.println("ClassOne"); + } +} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.xoo new file mode 100644 index 00000000000..9d5005aafea --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/.xoo @@ -0,0 +1 @@ +Some random content
\ No newline at end of file diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/Class.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/Class.xoo new file mode 100644 index 00000000000..fe9d2e54718 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/.hidden/Class.xoo @@ -0,0 +1,8 @@ +package hello; + +public class ClassOne { + + public static void main(String[] args) { + System.out.println("ClassOne"); + } +} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo new file mode 100644 index 00000000000..9d5005aafea --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo @@ -0,0 +1 @@ +Some random content
\ No newline at end of file diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/Class.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/Class.xoo new file mode 100644 index 00000000000..fe9d2e54718 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/Class.xoo @@ -0,0 +1,8 @@ +package hello; + +public class ClassOne { + + public static void main(String[] args) { + System.out.println("ClassOne"); + } +} diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.xoo new file mode 100644 index 00000000000..9d5005aafea --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.xoo @@ -0,0 +1 @@ +Some random content
\ No newline at end of file diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/Class.xoo b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/Class.xoo new file mode 100644 index 00000000000..fe9d2e54718 --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/Class.xoo @@ -0,0 +1,8 @@ +package hello; + +public class ClassOne { + + public static void main(String[] args) { + System.out.println("ClassOne"); + } +} |