diff options
Diffstat (limited to 'sonar-scanner-engine/src')
80 files changed, 4104 insertions, 792 deletions
diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java new file mode 100644 index 00000000000..94ccedfa946 --- /dev/null +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/analysisdata/AnalysisDataIT.java @@ -0,0 +1,68 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.mediumtest.analysisdata; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.core.platform.PluginInfo; +import org.sonar.scanner.mediumtest.AnalysisResult; +import org.sonar.scanner.mediumtest.ScannerMediumTester; +import org.sonar.scanner.protocol.output.ScannerReport; +import org.sonar.xoo.Xoo; +import org.sonar.xoo.XooPlugin; + +public class AnalysisDataIT { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public ScannerMediumTester tester = new ScannerMediumTester() + .registerPlugin(new PluginInfo("xoo").setOrganizationName("SonarSource"), new XooPlugin()) + .addDefaultQProfile("xoo", "Sonar Way"); + + @Test + public void whenScanningWithXoo_thenArchitectureGraphIsInReport() throws Exception { + // given + File projectDir = new File("test-resources/mediumtest/xoo/sample"); + File tmpDir = temp.newFolder(); + FileUtils.copyDirectory(projectDir, tmpDir); + + // when + AnalysisResult result = tester + .newAnalysis(new File(tmpDir, "sonar-project.properties")) + .execute(); + + // then + List<ScannerReport.AnalysisData> analysisData = result.analysisData(); + assertThat(analysisData) + .anyMatch(data -> data.getKey().startsWith("architecture.graph.")); + } +} diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java index adf5b845573..6cfd6ea5f94 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java @@ -92,6 +92,11 @@ class BootstrapMediumIT { } """))); + sonarqube.stubFor(get("/api/features/list") + .willReturn(okJson(""" + [] + """))); + sonarqube.stubFor(get("/api/metrics/search?ps=500&p=1") .willReturn(okJson(""" { @@ -112,7 +117,7 @@ class BootstrapMediumIT { void should_fail_if_invalid_json_input() { var in = new ByteArrayInputStream("}".getBytes()); - var exitCode = ScannerMain.run(in); + var exitCode = ScannerMain.run(in, System.out); assertThat(exitCode).isEqualTo(1); assertThat(logTester.getLogs(Level.ERROR)).hasSize(1); @@ -123,7 +128,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_key() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes())); + {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null key. Value='aValueWithoutKey'"); } @@ -131,7 +136,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -139,7 +144,7 @@ class BootstrapMediumIT { @Test void should_warn_if_not_provided_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -147,7 +152,7 @@ class BootstrapMediumIT { @Test void should_warn_if_duplicate_property_keys() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Duplicated properties. Key='aKey'"); } @@ -155,7 +160,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -163,7 +168,7 @@ class BootstrapMediumIT { @Test void should_warn_if_empty_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{}]}""".getBytes())); + {"scannerProperties": [{}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -224,7 +229,7 @@ class BootstrapMediumIT { } private int runScannerEngine(ScannerProperties scannerProperties) { - return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes())); + return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes()), System.out); } static class ScannerProperties { diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index 0a1ecff943b..babeadf518e 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -28,14 +28,21 @@ import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; +import java.util.Map; import java.util.Random; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.event.Level; import org.sonar.api.CoreProperties; import org.sonar.api.SonarEdition; @@ -49,6 +56,7 @@ import org.sonar.api.utils.System2; import org.sonar.scanner.mediumtest.AnalysisResult; import org.sonar.scanner.mediumtest.ScannerMediumTester; import org.sonar.scanner.mediumtest.ScannerMediumTester.AnalysisBuilder; +import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.xoo.XooPlugin; import org.sonar.xoo.global.DeprecatedGlobalSensor; import org.sonar.xoo.global.GlobalProjectSensor; @@ -59,7 +67,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.tuple; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowableOfType; -import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.slf4j.event.Level.DEBUG; @@ -150,7 +157,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("2 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("2 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(String.join("\n", logTester.logs())).doesNotContain("'src/sample.java' generated metadata"); } @@ -173,7 +180,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("2 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("2 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(logTester.logs()).contains("'src/sample.java' generated metadata with charset 'UTF-8'"); } @@ -200,7 +207,7 @@ class FileSystemMediumIT { .execute(); assertThat(logTester.logs()).containsAnyOf("'src/main/sample.java' indexed with no language", "'src\\main\\sample.java' indexed with no language"); - assertThat(logTester.logs()).contains("3 files indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("3 files indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/main/sample.xoo' generated metadata with charset 'UTF-8'"); assertThat(logTester.logs()).doesNotContain("'src/main/sample.java' generated metadata", "'src\\main\\sample.java' generated metadata"); assertThat(logTester.logs()).doesNotContain("'src/test/sample.java' generated metadata", "'src\\test\\sample.java' generated metadata"); @@ -231,7 +238,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("1 file indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("1 file indexed (done) | time=")); assertThat(logTester.logs()).contains("'src/sample.unknown' indexed with no language"); assertThat(logTester.logs()).contains("'src/sample.unknown' generated metadata with charset 'UTF-8'"); DefaultInputFile inputFile = (DefaultInputFile) result.inputFile("src/sample.unknown"); @@ -456,6 +463,7 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisDoesNotFailOnBrokenSymlink() throws IOException { prepareBrokenSymlinkTestScenario(); @@ -465,6 +473,7 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisWarnsAndIgnoresBrokenSymlink() throws IOException { Path link = prepareBrokenSymlinkTestScenario(); @@ -477,8 +486,8 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) void analysisIgnoresSymbolicLinkWithTargetOutsideBaseDir() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); File srcDir = new File(baseDir, "src"); assertThat(srcDir.mkdir()).isTrue(); @@ -497,23 +506,34 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) + void analysisIgnoresSymbolicLinkWithRelativeTargetOutsideBaseDir() throws IOException { + File srcDir = new File(baseDir, "src"); + assertThat(srcDir.mkdir()).isTrue(); + + File otherDir = createDirectory(temp.toPath().resolve("other_dir")).toFile(); + writeFile(otherDir, "target_outside.xoo"); + + Path linkPath = srcDir.toPath().resolve("target_link"); + Path link = Files.createSymbolicLink(linkPath, Paths.get("../../other_dir/target_outside.xoo")); + + tester.newAnalysis().properties(builder.build()).execute(); + + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file not located in project basedir.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.WARN)).contains(logMessage); + } + + @Test + @DisabledOnOs(OS.WINDOWS) void analysisIgnoresSymbolicLinkWithTargetOutsideModule() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); - File baseDirModuleA = new File(baseDir, "module_a"); - File baseDirModuleB = new File(baseDir, "module_b"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("module_a", "src"); + File srcDirB = createModuleWithSubdirectory("module_b", "src"); File target = writeFile(srcDirA, "target.xoo", "Sample xoo\ncontent"); Path link = Paths.get(srcDirB.getPath(), "target_link.xoo"); Files.createSymbolicLink(link, target.toPath()); - builder = ImmutableMap.<String, String>builder() - .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) - .put("sonar.projectKey", "com.foo.project") - .put("sonar.modules", "module_a,module_b"); + builder.put("sonar.modules", "module_a,module_b"); AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); @@ -524,13 +544,49 @@ class FileSystemMediumIT { } @Test + @DisabledOnOs(OS.WINDOWS) + void analysisIgnoresSymbolicLinkWithRelativeTargetOutsideModule() throws IOException { + File srcA = createModuleWithSubdirectory("module_a", "src"); + File srcB = createModuleWithSubdirectory("module_b", "src"); + + Path target = srcB.toPath().resolve("target.xoo"); + FileUtils.write(target.toFile(), "Sample xoo\ncontent", StandardCharsets.UTF_8); + Path link = srcA.toPath().resolve("target_link"); + Files.createSymbolicLink(link, Paths.get("../../module_b/src/target.xoo")); + + builder.put("sonar.modules", "module_a,module_b"); + + AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); + + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file not located in module basedir.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.INFO)).contains(logMessage); + InputFile fileA = result.inputFile("module_b/src/target.xoo"); + assertThat(fileA).isNotNull(); + } + + @Test + @DisabledOnOs(OS.WINDOWS) + void analysisDoesNotIgnoreSymbolicLinkWithRelativePath() throws IOException { + File src = createModuleWithSubdirectory("module_a", "src"); + Path target = src.toPath().resolve("target.xoo"); + FileUtils.write(target.toFile(), "Sample xoo\ncontent", StandardCharsets.UTF_8); + Path link = src.toPath().resolve("target_link"); + Files.createSymbolicLink(link, Paths.get("target.xoo")); + + builder.put("sonar.modules", "module_a"); + + AnalysisResult result = tester.newAnalysis().properties(builder.build()).execute(); + + InputFile targetFile = result.inputFile("module_a/src/target.xoo"); + assertThat(targetFile).isNotNull(); + String logMessage = String.format("File '%s' is ignored. It is a symbolic link targeting a file that does not exist.", link.toRealPath(LinkOption.NOFOLLOW_LINKS)); + assertThat(logTester.logs(Level.WARN)).doesNotContain(logMessage); + } + + @Test void test_inclusions_on_multi_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "tests"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "tests"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "tests"); + File srcDirB = createModuleWithSubdirectory("moduleB", "tests"); writeFile(srcDirA, "sampleTestA.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sampleTestB.xoo", "Sample xoo\ncontent"); @@ -571,12 +627,8 @@ class FileSystemMediumIT { @Test void test_module_level_inclusions_override_parent_on_multi_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sampleA.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sampleB.xoo", "Sample xoo\ncontent"); @@ -608,12 +660,8 @@ class FileSystemMediumIT { @Test void warn_user_for_outdated_scanner_side_inherited_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -641,12 +689,8 @@ class FileSystemMediumIT { @Test void support_global_server_side_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -671,12 +715,8 @@ class FileSystemMediumIT { @Test void support_global_server_side_global_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -701,12 +741,8 @@ class FileSystemMediumIT { @Test void warn_user_for_outdated_server_side_inherited_exclusions_for_multi_module_project() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -979,12 +1015,8 @@ class FileSystemMediumIT { @Test void log_all_exclusions_properties_per_modules() throws IOException { - File baseDirModuleA = new File(baseDir, "moduleA"); - File baseDirModuleB = new File(baseDir, "moduleB"); - File srcDirA = new File(baseDirModuleA, "src"); - assertThat(srcDirA.mkdirs()).isTrue(); - File srcDirB = new File(baseDirModuleB, "src"); - assertThat(srcDirB.mkdirs()).isTrue(); + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); writeFile(srcDirA, "sample.xoo", "Sample xoo\ncontent"); writeFile(srcDirB, "sample.xoo", "Sample xoo\ncontent"); @@ -1013,7 +1045,7 @@ class FileSystemMediumIT { " Excluded sources for coverage: **/coverage.exclusions", " Excluded sources for duplication: **/cpd.exclusions", "Indexing files of module 'moduleA'", - " Base dir: " + baseDirModuleA.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS), + " Base dir: " + srcDirA.toPath().getParent().toRealPath(LinkOption.NOFOLLOW_LINKS), " Included sources: **/global.inclusions", " Excluded sources: **/global.exclusions, **/global.test.inclusions", " Included tests: **/global.test.inclusions", @@ -1021,7 +1053,7 @@ class FileSystemMediumIT { " Excluded sources for coverage: **/coverage.exclusions", " Excluded sources for duplication: **/cpd.exclusions", "Indexing files of module 'moduleB'", - " Base dir: " + baseDirModuleB.toPath().toRealPath(LinkOption.NOFOLLOW_LINKS), + " Base dir: " + srcDirB.toPath().getParent().toRealPath(LinkOption.NOFOLLOW_LINKS), " Included sources: **/global.inclusions", " Excluded sources: **/global.exclusions, **/global.test.inclusions", " Included tests: **/global.test.inclusions", @@ -1147,7 +1179,7 @@ class FileSystemMediumIT { .build()) .execute(); - assertThat(logTester.logs()).contains("1 file indexed"); + assertThat(logTester.logs()).anyMatch(log -> log.startsWith("1 file indexed (done) | time=")); assertThat(result.inputFile("sample.xoo")).isNotNull(); } @@ -1314,6 +1346,200 @@ class FileSystemMediumIT { .hasMessageEndingWith("Failed to preprocess files"); } + @ParameterizedTest + @ValueSource(booleans = { + true, + false + }) + void shouldScanAndAnalyzeAllHiddenFiles(boolean setHiddenFileScanningExplicitly) throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisBuilder analysisBuilder = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true"); + + if (setHiddenFileScanningExplicitly) { + // default is assumed to be false, here we set it explicitly + analysisBuilder.property("sonar.scanner.excludeHiddenFiles", "false"); + } + + AnalysisResult result = analysisBuilder.execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedIsHidden = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedIsHidden, true); + // we expect the sensor to process all files, regardless of visibility + assertFileIssue(result, filePath, true); + } + assertThat(result.inputFiles()).hasSize(10); + } + + @Test + void shouldScanAllFilesAndOnlyAnalyzeNonHiddenFiles() throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "false") + .execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedHiddenStatus = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedHiddenStatus, true); + // sensor should not process hidden files, we only expect issues on non-hidden files + assertFileIssue(result, filePath, !expectedHiddenStatus); + } + assertThat(result.inputFiles()).hasSize(10); + } + + @ParameterizedTest + @ValueSource(booleans = { + true, + false + }) + void shouldNotScanAndAnalyzeHiddenFilesWhenHiddenFileScanningIsDisabled(boolean sensorHiddenFileProcessingEnabled) throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") + .property("sonar.scanner.excludeHiddenFiles", "true") + // hidden files are not scanned, so issues can't be raised on them regardless if the sensor wants to process them + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", String.valueOf(sensorHiddenFileProcessingEnabled)) + .execute(); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedHiddenStatus = pathToHiddenStatus.getValue(); + assertHiddenFileScan(result, filePath, expectedHiddenStatus, false); + if (!expectedHiddenStatus) { + assertFileIssue(result, filePath, true); + } + } + assertThat(result.inputFiles()).hasSize(1); + } + + @Test + void hiddenFilesAssignedToALanguageShouldNotBePublishedByDefault() throws IOException { + tester + .addRules(new XooRulesDefinition()); + + File srcDir = new File(baseDir, "src"); + assertThat(srcDir.mkdir()).isTrue(); + + File hiddenFile = writeFile(srcDir, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(hiddenFile.toPath()); + File hiddenFileWithoutLanguage = writeFile(srcDir, ".bar", "Sample bar\ncontent"); + setFileAsHiddenOnWindows(hiddenFileWithoutLanguage.toPath()); + writeFile(srcDir, "file.xoo", "Sample xoo\ncontent"); + + AnalysisResult result = tester.newAnalysis() + .properties(builder + .put("sonar.sources", "src") + .build()) + .execute(); + + DefaultInputFile hiddenInputFile = (DefaultInputFile) result.inputFile("src/.xoo"); + + assertThat(hiddenInputFile).isNotNull(); + assertThat(hiddenInputFile.isPublished()).isFalse(); + assertThatThrownBy(() -> result.getReportComponent(hiddenInputFile)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unable to find report for component"); + + DefaultInputFile hiddenInputFileWithoutLanguage = (DefaultInputFile) result.inputFile("src/.bar"); + assertThat(hiddenInputFileWithoutLanguage).isNotNull(); + assertThat(hiddenInputFileWithoutLanguage.isPublished()).isFalse(); + assertThatThrownBy(() -> result.getReportComponent(hiddenInputFileWithoutLanguage)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Unable to find report for component"); + + DefaultInputFile visibleInputFile = (DefaultInputFile) result.inputFile("src/file.xoo"); + assertThat(visibleInputFile).isNotNull(); + assertThat(visibleInputFile.isPublished()).isTrue(); + assertThat(result.getReportComponent(visibleInputFile)).isNotNull(); + } + + @Test + void shouldDetectHiddenFilesFromMultipleModules() throws IOException { + File srcDirA = createModuleWithSubdirectory("moduleA", "src"); + File srcDirB = createModuleWithSubdirectory("moduleB", "src"); + + File fileModuleA = writeFile(srcDirA, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(fileModuleA.toPath()); + File fileModuleB = writeFile(srcDirB, ".xoo", "Sample xoo\ncontent"); + setFileAsHiddenOnWindows(fileModuleB.toPath()); + + AnalysisResult result = tester.newAnalysis() + .properties(ImmutableMap.<String, String>builder() + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.sources", "src") + .put("sonar.modules", "moduleA,moduleB") + .build()) + .execute(); + + assertHiddenFileScan(result, "moduleA/src/.xoo", true, true); + assertHiddenFileScan(result, "moduleB/src/.xoo", true, true); + } + + @Test + void shouldScanAndAnalyzeAllHiddenFilesWithRespectToExclusions() throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + + + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.scm.provider", "xoo") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true") + .property("sonar.exclusions", "**/.nestedHidden/**,**/*.ignore") + .execute(); + + Set<String> excludedFiles = Set.of( + // sonar.exclusions + "xources/.hidden/.nestedHidden/.xoo", + "xources/.hidden/.nestedHidden/Class.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo.ignore", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo", + // scm ignore + "xources/nonHidden/.hiddenInVisibleFolder/.xoo"); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedIsHidden = pathToHiddenStatus.getValue(); + + if (excludedFiles.contains(filePath)) { + assertThat(result.inputFile(filePath)).isNull(); + } else { + assertHiddenFileScan(result, filePath, expectedIsHidden, true); + // we expect the sensor to process all non-excluded files, regardless of visibility + assertFileIssue(result, filePath, true); + } + } + assertThat(result.inputFiles()).hasSize(5); + } + + private File createModuleWithSubdirectory(String moduleName, String subDirName) { + File moduleBaseDir = new File(baseDir, moduleName); + File srcDir = moduleBaseDir.toPath().resolve(subDirName).toFile(); + assertThat(srcDir.mkdirs()).isTrue(); + return srcDir; + } + private static void assertAnalysedFiles(AnalysisResult result, String... files) { assertThat(result.inputFiles().stream().map(InputFile::toString).toList()).contains(files); } @@ -1341,7 +1567,6 @@ class FileSystemMediumIT { } private Path prepareBrokenSymlinkTestScenario() throws IOException { - assumeFalse(SystemUtils.IS_OS_WINDOWS); File srcDir = new File(baseDir, "src"); assertThat(srcDir.mkdir()).isTrue(); @@ -1353,4 +1578,69 @@ class FileSystemMediumIT { return link; } + private Map<String, Boolean> hiddenFileProjectExpectedHiddenStatus() { + return Map.of( + "xources/.hidden/.xoo", true, + "xources/.hidden/Class.xoo", true, + "xources/.hidden/.nestedHidden/.xoo", true, + "xources/.hidden/.nestedHidden/Class.xoo", true, + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", true, + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo", true, + "xources/nonHidden/.xoo", true, + "xources/nonHidden/Class.xoo", false, + "xources/nonHidden/.hiddenInVisibleFolder/.xoo", true, + "xources/nonHidden/.hiddenInVisibleFolder/Class.xoo", true); + } + + private static void prepareHiddenFileProject() throws IOException { + if (!SystemUtils.IS_OS_WINDOWS) { + return; + } + + // On Windows, we need to set the hidden attribute on the file system + Set<String> dirAndFilesToHideOnWindows = Set.of( + "xources/.hidden", + "xources/.hidden/.xoo", + "xources/.hidden/.nestedHidden", + "xources/.hidden/.nestedHidden/.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", + "xources/nonHidden/.xoo", + "xources/nonHidden/.hiddenInVisibleFolder", + "xources/nonHidden/.hiddenInVisibleFolder/.xoo"); + + for (String path : dirAndFilesToHideOnWindows) { + Path pathFromResources = Path.of("test-resources/mediumtest/xoo/sample-with-hidden-files", path); + setFileAsHiddenOnWindows(pathFromResources); + } + } + + private static void setFileAsHiddenOnWindows(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(path, "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } + + private static void assertHiddenFileScan(AnalysisResult result, String filePath, boolean expectedHiddenStatus, boolean hiddenFilesShouldBeScanned) { + InputFile file = result.inputFile(filePath); + + if (!hiddenFilesShouldBeScanned && expectedHiddenStatus) { + assertThat(file).isNull(); + } else { + assertThat(file).withFailMessage(String.format("File \"%s\" was not analyzed", filePath)).isNotNull(); + assertThat(file.isHidden()) + .withFailMessage(String.format("Expected file \"%s\" hidden status to be \"%s\", however was \"%s\"", filePath, expectedHiddenStatus, file.isHidden())) + .isEqualTo(expectedHiddenStatus); + } + } + + private static void assertFileIssue(AnalysisResult result, String filePath, boolean expectToHaveIssue) { + InputFile file = result.inputFile(filePath); + assertThat(file).isNotNull(); + List<ScannerReport.Issue> issues = result.issuesFor(file); + if (expectToHaveIssue) { + assertThat(issues).hasSize(1); + } else { + assertThat(issues).isEmpty(); + } + } } diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/log/LogListenerIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/log/LogListenerIT.java index 59225e021b5..588181a456d 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/log/LogListenerIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/log/LogListenerIT.java @@ -124,7 +124,7 @@ public class LogListenerIT { } Matcher matcher = simpleTimePattern.matcher(msg); - assertThat(matcher.find()).isFalse(); + assertThat(matcher.find()).as("Offending log message: " + msg).isFalse(); } @Test diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java index e1a158bfac9..904e09cc5c5 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/scm/ScmMediumIT.java @@ -241,7 +241,7 @@ public class ScmMediumIT { assertThat(getChangesets(baseDir, NO_BLAME_SCM_ON_SERVER_XOO)).isNull(); // 5 .xoo files + 3 .scm files, but only 4 marked for publishing. 1 file is SAME so not included in the total - assertThat(logTester.logs()).containsSubsequence("8 files indexed"); + assertThat(logTester.logs()).anyMatch(s -> s.startsWith("8 files indexed (done) | time=")); assertThat(logTester.logs()).containsSubsequence("SCM Publisher 4 source files to be analyzed"); assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("SCM Publisher 3/4 source files have been analyzed \\(done\\) \\| time=[0-9]+ms", s))).isTrue(); assertThat(logTester.logs()).containsSubsequence(MISSING_BLAME_INFORMATION_FOR_THE_FOLLOWING_FILES, " * src/no_blame_scm_on_server.xoo"); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 0fe2c3ad479..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,17 +21,21 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.sonar.api.utils.MessageException; @@ -49,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -67,9 +73,11 @@ public class ScannerMain { LOG.info("SonarScanner Engine completed successfully"); return 0; - } catch (Exception e) { - handleException(e); + } catch (Throwable throwable) { + handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -87,30 +95,28 @@ public class ScannerMain { return sb.toString(); } - private static void handleException(Exception e) { - var messageException = unwrapMessageException(e); + private static void handleException(Throwable throwable) { + var messageException = unwrapMessageException(throwable); if (messageException.isPresent()) { // Don't show the stacktrace for a message exception to not pollute the logs if (LoggerFactory.getLogger(ScannerMain.class).isDebugEnabled()) { - LOG.error(messageException.get(), e); + LOG.error(messageException.get(), throwable); } else { LOG.error(messageException.get()); } } else { - LOG.error("Error during SonarScanner Engine execution", e); + LOG.error("Error during SonarScanner Engine execution", throwable); } } - private static Optional<String> unwrapMessageException(Exception t) { - Throwable y = t; - do { - if (y instanceof MessageException messageException) { - return Optional.of(messageException.getMessage()); - } - y = y.getCause(); - } while (y != null); - - return Optional.empty(); + private static Optional<String> unwrapMessageException(@Nullable Throwable throwable) { + if (throwable == null) { + return Optional.empty(); + } else if (throwable instanceof MessageException messageException) { + return Optional.of(messageException.getMessage()); + } else { + return unwrapMessageException(throwable.getCause()); + } } private static @NotNull Map<String, String> parseInputProperties(InputStream in) { @@ -157,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java index 3187c2e9aa3..194c1e17ffa 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginRepository.java @@ -19,6 +19,10 @@ */ package org.sonar.scanner.bootstrap; +import static java.util.stream.Collectors.toMap; +import static org.sonar.api.utils.Preconditions.checkState; +import static org.sonar.core.config.ScannerProperties.PLUGIN_LOADING_OPTIMIZATION_KEY; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -38,10 +42,6 @@ import org.sonar.core.platform.PluginRepository; import org.sonar.core.plugin.PluginType; import org.sonar.scanner.mediumtest.LocalPlugin; -import static java.util.stream.Collectors.toMap; -import static org.sonar.api.utils.Preconditions.checkState; -import static org.sonar.core.config.ScannerProperties.PLUGIN_LOADING_OPTIMIZATION_KEY; - /** * Orchestrates the installation and loading of plugins */ @@ -83,7 +83,7 @@ public class ScannerPluginRepository implements PluginRepository, Startable { // this part is only used by medium tests for (LocalPlugin localPlugin : installer.installLocals()) { ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); + String pluginKey = localPlugin.pluginInfo().getKey(); pluginsByKeys.put(pluginKey, scannerPlugin); pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); } @@ -112,7 +112,7 @@ public class ScannerPluginRepository implements PluginRepository, Startable { // this part is only used by medium tests for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) { ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin(); - String pluginKey = localPlugin.pluginKey(); + String pluginKey = localPlugin.pluginInfo().getKey(); languagePluginsByKeys.put(pluginKey, scannerPlugin); pluginsByKeys.put(pluginKey, scannerPlugin); pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java index 29f1389e20c..133f4387856 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/SpringScannerContainer.java @@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.internal.FileMetadata; import org.sonar.api.batch.rule.CheckFactory; import org.sonar.api.batch.sensor.issue.internal.DefaultNoSonarFilter; -import org.sonar.api.config.PropertyDefinition; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.core.extension.CoreExtensionsInstaller; import org.sonar.core.metric.ScannerMetrics; @@ -87,6 +86,8 @@ import org.sonar.scanner.repository.ProjectRepositoriesProvider; import org.sonar.scanner.repository.QualityProfilesProvider; import org.sonar.scanner.repository.ReferenceBranchSupplier; import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.repository.featureflags.DefaultFeatureFlagsLoader; +import org.sonar.scanner.repository.featureflags.DefaultFeatureFlagsRepository; import org.sonar.scanner.repository.language.DefaultLanguagesLoader; import org.sonar.scanner.repository.language.DefaultLanguagesRepository; import org.sonar.scanner.repository.settings.DefaultProjectSettingsLoader; @@ -98,7 +99,6 @@ import org.sonar.scanner.scan.InputModuleHierarchyProvider; import org.sonar.scanner.scan.InputProjectProvider; import org.sonar.scanner.scan.ModuleIndexer; import org.sonar.scanner.scan.MutableProjectReactorProvider; -import org.sonar.scanner.scan.MutableProjectSettings; import org.sonar.scanner.scan.ProjectBuildersExecutor; import org.sonar.scanner.scan.ProjectConfigurationProvider; import org.sonar.scanner.scan.ProjectLock; @@ -115,6 +115,7 @@ import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.branch.ProjectBranchesProvider; import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem; import org.sonar.scanner.scan.filesystem.FilePreprocessor; +import org.sonar.scanner.scan.filesystem.HiddenFilesProjectData; import org.sonar.scanner.scan.filesystem.InputComponentStore; import org.sonar.scanner.scan.filesystem.LanguageDetection; import org.sonar.scanner.scan.filesystem.MetadataGenerator; @@ -152,25 +153,10 @@ public class SpringScannerContainer extends SpringComponentContainer { @Override protected void doBeforeStart() { - addSuffixesDeprecatedProperties(); addScannerExtensions(); addComponents(); } - private void addSuffixesDeprecatedProperties() { - add( - /* - * This is needed to support properly the deprecated sonar.rpg.suffixes property when the download optimization feature is enabled. - * The value of the property is needed at the preprocessing stage, but being defined by an optional analyzer means that at preprocessing - * it won't be properly available. This will be removed in SQ 11.0 together with the drop of the property from the rpg analyzer. - * See SONAR-21514 - */ - PropertyDefinition.builder("sonar.rpg.file.suffixes") - .deprecatedKey("sonar.rpg.suffixes") - .multiValues(true) - .build()); - } - private void addScannerExtensions() { getParentComponentByType(CoreExtensionsInstaller.class) .install(this, noExtensionFilter(), extension -> getScannerProjectExtensionsFilter().accept(extension)); @@ -214,6 +200,7 @@ public class SpringScannerContainer extends SpringComponentContainer { FilePreprocessor.class, ProjectFilePreprocessor.class, ProjectExclusionFilters.class, + HiddenFilesProjectData.class, // rules new ActiveRulesProvider(), @@ -240,7 +227,6 @@ public class SpringScannerContainer extends SpringComponentContainer { ContextPropertiesCache.class, TelemetryCache.class, - MutableProjectSettings.class, SonarGlobalPropertiesFilter.class, ProjectConfigurationProvider.class, @@ -309,17 +295,20 @@ public class SpringScannerContainer extends SpringComponentContainer { GitlabCi.class, Jenkins.class, SemaphoreCi.class, - TravisCi.class); - - add(GitScmSupport.getObjects()); - add(SvnScmSupport.getObjects()); + TravisCi.class, - add(DefaultProjectSettingsLoader.class, + DefaultProjectSettingsLoader.class, DefaultActiveRulesLoader.class, DefaultQualityProfileLoader.class, DefaultProjectRepositoriesLoader.class, DefaultLanguagesLoader.class, - DefaultLanguagesRepository.class); + DefaultLanguagesRepository.class, + + DefaultFeatureFlagsLoader.class, + DefaultFeatureFlagsRepository.class); + + add(GitScmSupport.getObjects()); + add(SvnScmSupport.getObjects()); } static ExtensionMatcher getScannerProjectExtensionsFilter() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java index 703cd038fd0..d4f22ad9704 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetector.java @@ -97,8 +97,11 @@ public class RulesSeverityDetector { } private static Map<String, Result.Level> getDriverDefinedRuleSeverities(Run run) { - return run.getTool().getDriver().getRules() - .stream() + Set<ReportingDescriptor> rules = run.getTool().getDriver().getRules(); + if (rules == null) { + return emptyMap(); + } + return rules.stream() .filter(RulesSeverityDetector::hasRuleDefinedLevel) .collect(toMap(ReportingDescriptor::getId, x -> Result.Level.valueOf(x.getDefaultConfiguration().getLevel().name()))); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java index 5b9abf383cf..bdf5a9a1114 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/externalissue/sarif/RunMapper.java @@ -83,7 +83,7 @@ public class RunMapper { private List<NewAdHocRule> toNewAdHocRules(Run run, String driverName, Map<String, Result.Level> ruleSeveritiesByRuleId, Map<String, Result.Level> ruleSeveritiesByRuleIdForNewCCT) { - Set<ReportingDescriptor> driverRules = run.getTool().getDriver().getRules(); + Set<ReportingDescriptor> driverRules = Optional.ofNullable(run.getTool().getDriver().getRules()).orElse(Set.of()); Set<ReportingDescriptor> extensionRules = hasExtensions(run.getTool()) ? run.getTool().getExtensions().stream().filter(RunMapper::hasRules).flatMap(extension -> extension.getRules().stream()).collect(toSet()) : Set.of(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java index 088ebfb0052..d9eed8bedc8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/http/ScannerWsClientProvider.java @@ -87,7 +87,7 @@ public class ScannerWsClientProvider { String responseTimeout = defaultIfBlank(scannerProps.property(SONAR_SCANNER_RESPONSE_TIMEOUT), valueOf(DEFAULT_RESPONSE_TIMEOUT)); String envVarToken = defaultIfBlank(system.envVariable(TOKEN_ENV_VARIABLE), null); String token = defaultIfBlank(scannerProps.property(TOKEN_PROPERTY), envVarToken); - String login = defaultIfBlank(scannerProps.property(CoreProperties.LOGIN), token); + String login = defaultIfBlank(token, scannerProps.property(CoreProperties.LOGIN)); boolean skipSystemTrustMaterial = Boolean.parseBoolean(defaultIfBlank(scannerProps.property(SKIP_SYSTEM_TRUST_MATERIAL), "false")); var sslContext = configureSsl(parseSslConfig(scannerProps, sonarUserHome), system, skipSystemTrustMaterial); connectorBuilder diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java index 1e37a715066..df9bfd00b48 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssuePublisher.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @@ -54,6 +55,7 @@ import org.sonar.scanner.report.ReportPublisher; @ThreadSafe public class IssuePublisher { + private static final Set<String> noSonarKeyContains = Set.of("nosonar", "S1291"); private final ActiveRules activeRules; private final IssueFilters filters; private final ReportPublisher reportPublisher; @@ -91,7 +93,7 @@ public class IssuePublisher { return inputComponent.isFile() && textRange != null && ((DefaultInputFile) inputComponent).hasNoSonarAt(textRange.start().line()) - && !StringUtils.containsIgnoreCase(issue.ruleKey().rule(), "nosonar"); + && noSonarKeyContains.stream().noneMatch(k -> StringUtils.containsIgnoreCase(issue.ruleKey().rule(), k)); } public void initAndAddExternalIssue(ExternalIssue issue) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java index 0a3d42979f7..70df99a4f3d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/LocalPlugin.java @@ -25,9 +25,13 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; import org.sonar.scanner.bootstrap.ScannerPlugin; -public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { +public record LocalPlugin(PluginInfo pluginInfo, Plugin pluginInstance, Set<String> requiredForLanguages) { + + public LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) { + this(new PluginInfo(pluginKey).setOrganizationName("SonarSource"), pluginInstance, requiredForLanguages); + } public ScannerPlugin toScannerPlugin() { - return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey)); + return new ScannerPlugin(pluginInfo.getKey(), 1L, PluginType.BUNDLED, pluginInfo); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java new file mode 100644 index 00000000000..8eb338f3fba --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoader.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.sonar.scanner.http.ScannerWsClient; +import org.sonarqube.ws.client.GetRequest; + +public class DefaultFeatureFlagsLoader implements FeatureFlagsLoader { + + private static final String FEATURE_FLAGS_WS_URL = "/api/features/list"; + + private final ScannerWsClient wsClient; + + public DefaultFeatureFlagsLoader(ScannerWsClient wsClient) { + this.wsClient = wsClient; + } + + @Override + public Set<String> load() { + GetRequest getRequest = new GetRequest(FEATURE_FLAGS_WS_URL); + List<String> jsonResponse; + try (Reader reader = wsClient.call(getRequest).contentReader()) { + jsonResponse = new Gson().fromJson(reader, new TypeToken<ArrayList<String>>() { + }.getType()); + } catch (Exception e) { + throw new IllegalStateException("Unable to load feature flags", e); + } + return Set.copyOf(jsonResponse); + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java new file mode 100644 index 00000000000..08b52011ed3 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepository.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import java.util.HashSet; +import java.util.Set; +import org.sonar.api.Startable; + +public class DefaultFeatureFlagsRepository implements FeatureFlagsRepository, Startable { + + private final Set<String> featureFlags = new HashSet<>(); + private final FeatureFlagsLoader featureFlagsLoader; + + public DefaultFeatureFlagsRepository(FeatureFlagsLoader featureFlagsLoader) { + this.featureFlagsLoader = featureFlagsLoader; + } + + @Override + public void start() { + featureFlags.addAll(featureFlagsLoader.load()); + } + + @Override + public void stop() { + // nothing to do + } + + @Override + public boolean isEnabled(String flagName) { + return featureFlags.contains(flagName); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java new file mode 100644 index 00000000000..034b2a7c098 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlag.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +public record FeatureFlag(String flagName, boolean enabled) { +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java new file mode 100644 index 00000000000..71213e86394 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsLoader.java @@ -0,0 +1,28 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import java.util.Set; + +public interface FeatureFlagsLoader { + + Set<String> load(); + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java new file mode 100644 index 00000000000..f354cd1c0f0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/FeatureFlagsRepository.java @@ -0,0 +1,26 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +public interface FeatureFlagsRepository { + + boolean isEnabled(String flagName); + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java new file mode 100644 index 00000000000..4b27773b5c7 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/repository/featureflags/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.scanner.repository.featureflags; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java new file mode 100644 index 00000000000..24db0ddec64 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliCacheService.java @@ -0,0 +1,250 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.internal.apachecommons.lang3.SystemUtils; +import org.sonar.api.utils.System2; +import org.sonar.scanner.bootstrap.SonarUserHome; +import org.sonar.scanner.http.ScannerWsClient; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.WsResponse; + +import static java.lang.String.format; + +/** + * This class is responsible for checking the SQ server for the latest version of the CLI, + * caching the CLI for use across different projects, updating the cached CLI to the latest + * version, and holding on to the cached CLI's file location so that other service classes + * can make use of it. + */ +public class CliCacheService { + protected static final String CLI_WS_URL = "api/v2/sca/clis"; + private static final Logger LOG = LoggerFactory.getLogger(CliCacheService.class); + private final SonarUserHome sonarUserHome; + private final ScannerWsClient wsClient; + private final TelemetryCache telemetryCache; + private final System2 system2; + + public CliCacheService(SonarUserHome sonarUserHome, ScannerWsClient wsClient, TelemetryCache telemetryCache, System2 system2) { + this.sonarUserHome = sonarUserHome; + this.wsClient = wsClient; + this.telemetryCache = telemetryCache; + this.system2 = system2; + } + + static Path newTempFile(Path tempDir) { + try { + return Files.createTempFile(tempDir, "scaFileCache", null); + } catch (IOException e) { + throw new IllegalStateException("Fail to create temp file in " + tempDir, e); + } + } + + static void moveFile(Path sourceFile, Path targetFile) { + try { + Files.move(sourceFile, targetFile, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e1) { + // Check if the file was cached by another process during download + if (!Files.exists(targetFile)) { + LOG.warn("Unable to rename {} to {}", sourceFile, targetFile); + LOG.warn("A copy/delete will be tempted but with no guarantee of atomicity"); + try { + Files.move(sourceFile, targetFile); + } catch (IOException e2) { + throw new IllegalStateException("Fail to move " + sourceFile + " to " + targetFile, e2); + } + } + } + } + + static void mkdir(Path dir) { + try { + Files.createDirectories(dir); + } catch (IOException e) { + throw new IllegalStateException("Fail to create cache directory: " + dir, e); + } + } + + static void downloadBinaryTo(Path downloadLocation, WsResponse response) { + try (InputStream stream = response.contentStream()) { + FileUtils.copyInputStreamToFile(stream, downloadLocation.toFile()); + } catch (IOException e) { + throw new IllegalStateException(format("Fail to download SCA CLI into %s", downloadLocation), e); + } + } + + public File cacheCli() { + boolean success = false; + + var alternateLocation = system2.envVariable("TIDELIFT_CLI_LOCATION"); + if (alternateLocation != null) { + LOG.info("Using alternate location for Tidelift CLI: {}", alternateLocation); + // If the TIDELIFT_CLI_LOCATION environment variable is set, we should use that location + // instead of trying to download the CLI from the server. + File cliFile = new File(alternateLocation); + if (!cliFile.exists()) { + throw new IllegalStateException(format("Alternate location for Tidelift CLI has been set but no file was found at %s", alternateLocation)); + } + return cliFile; + } + + try { + List<CliMetadataResponse> metadataResponses = getLatestMetadata(apiOsName(), apiArch()); + + if (metadataResponses.isEmpty()) { + throw new IllegalStateException(format("Could not find CLI for %s %s", apiOsName(), apiArch())); + } + + // We should only be getting one matching CLI for the OS + Arch combination. + // If we have more than one CLI to choose from then I'm not sure which one to choose. + if (metadataResponses.size() > 1) { + throw new IllegalStateException("Multiple CLI matches found. Unable to correctly cache CLI."); + } + + CliMetadataResponse metadataResponse = metadataResponses.get(0); + String checksum = metadataResponse.sha256(); + // If we have a matching checksum dir with the existing CLI file, then we are up to date. + if (!cachedCliFile(checksum).exists()) { + LOG.debug("SCA CLI update detected"); + downloadCli(metadataResponse.id(), checksum); + telemetryCache.put("scanner.sca.get.cli.cache.hit", "false"); + } else { + telemetryCache.put("scanner.sca.get.cli.cache.hit", "true"); + } + + File cliFile = cachedCliFile(checksum); + success = true; + return cliFile; + } finally { + telemetryCache.put("scanner.sca.get.cli.success", String.valueOf(success)); + } + } + + Path cacheDir() { + return sonarUserHome.getPath().resolve("cache"); + } + + private File cachedCliFile(String checksum) { + return cacheDir().resolve(checksum).resolve(fileName()).toFile(); + } + + private String fileName() { + return system2.isOsWindows() ? "tidelift.exe" : "tidelift"; + } + + private List<CliMetadataResponse> getLatestMetadata(String osName, String arch) { + LOG.info("Requesting CLI for OS {} and arch {}", osName, arch); + GetRequest getRequest = new GetRequest(CLI_WS_URL).setParam("os", osName).setParam("arch", arch); + try (WsResponse response = wsClient.call(getRequest)) { + try (Reader reader = response.contentReader()) { + Type listOfMetadata = new TypeToken<ArrayList<CliMetadataResponse>>() { + }.getType(); + return new Gson().fromJson(reader, listOfMetadata); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private void downloadCli(String id, String checksum) { + LOG.info("Downloading cli {}", id); + long startTime = system2.now(); + boolean success = false; + GetRequest getRequest = new GetRequest(CLI_WS_URL + "/" + id).setHeader("Accept", "application/octet-stream"); + + try (WsResponse response = wsClient.call(getRequest)) { + // Download to a temporary file location in case another process is also trying to + // create the CLI file in the checksum cache directory. Once the file is downloaded to a temporary + // location, do an atomic move to the correct cache location. + Path tempDir = createTempDir(); + Path tempFile = newTempFile(tempDir); + downloadBinaryTo(tempFile, response); + File destinationFile = cachedCliFile(checksum); + // We need to make sure the folder structure exists for the correct cache location before performing the move. + mkdir(destinationFile.toPath().getParent()); + moveFile(tempFile, destinationFile.toPath()); + if (!destinationFile.setExecutable(true, false)) { + throw new IllegalStateException("Unable to mark CLI as executable"); + } + success = true; + } catch (Exception e) { + throw new IllegalStateException("Unable to download CLI executable", e); + } finally { + telemetryCache.put("scanner.sca.download.cli.duration", String.valueOf(system2.now() - startTime)); + telemetryCache.put("scanner.sca.download.cli.success", String.valueOf(success)); + } + } + + String apiOsName() { + // We don't want to send the raw OS name because there could be too many combinations of the OS name + // to reliably match up with the correct CLI needed to be downloaded. Instead, we send a subset of + // OS names that should match to the correct CLI here. + if (system2.isOsWindows()) { + return "windows"; + } else if (system2.isOsMac()) { + return "mac"; + } else { + return "linux"; + } + } + + String apiArch() { + return SystemUtils.OS_ARCH.toLowerCase(Locale.ENGLISH); + } + + Path createTempDir() { + Path dir = sonarUserHome.getPath().resolve("_tmp"); + try { + if (Files.exists(dir)) { + return dir; + } else { + return Files.createDirectory(dir); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory at " + dir, e); + } + } + + private record CliMetadataResponse( + String id, + String filename, + String sha256, + String os, + String arch) { + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java new file mode 100644 index 00000000000..6b3418a8f6a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java @@ -0,0 +1,214 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.platform.Server; +import org.sonar.api.utils.System2; +import org.sonar.core.util.ProcessWrapperFactory; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scm.git.JGitUtils; + +/** + * The CliService class is meant to serve as the main entrypoint for any commands + * that should be executed by the CLI. It will handle manages the external process, + * raising any errors that happen while running a command, and passing back the + * data generated by the command to the caller. + */ +public class CliService { + private static final Logger LOG = LoggerFactory.getLogger(CliService.class); + public static final String SCA_EXCLUSIONS_KEY = "sonar.sca.exclusions"; + public static final String LEGACY_SCA_EXCLUSIONS_KEY = "sonar.sca.excludedManifests"; + + private final ProcessWrapperFactory processWrapperFactory; + private final TelemetryCache telemetryCache; + private final System2 system2; + private final Server server; + private final ScmConfiguration scmConfiguration; + private final ProjectExclusionFilters projectExclusionFilters; + + public CliService(ProcessWrapperFactory processWrapperFactory, TelemetryCache telemetryCache, System2 system2, Server server, ScmConfiguration scmConfiguration, + ProjectExclusionFilters projectExclusionFilters) { + this.processWrapperFactory = processWrapperFactory; + this.telemetryCache = telemetryCache; + this.system2 = system2; + this.server = server; + this.scmConfiguration = scmConfiguration; + this.projectExclusionFilters = projectExclusionFilters; + } + + public File generateManifestsArchive(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { + long startTime = system2.now(); + boolean success = false; + try { + String archiveName = "dependency-files.tar.xz"; + Path archivePath = module.getWorkDir().resolve(archiveName); + List<String> args = new ArrayList<>(); + args.add(cliExecutable.getAbsolutePath()); + args.add("projects"); + args.add("save-lockfiles"); + args.add("--xz"); + args.add("--xz-filename"); + args.add(archivePath.toAbsolutePath().toString()); + args.add("--directory"); + args.add(module.getBaseDir().toString()); + args.add("--recursive"); + + String excludeFlag = getExcludeFlag(module, configuration); + if (excludeFlag != null) { + args.add("--exclude"); + args.add(excludeFlag); + } + + if (LOG.isDebugEnabled()) { + LOG.info("Setting CLI to debug mode"); + args.add("--debug"); + } + + Map<String, String> envProperties = new HashMap<>(); + // sending this will tell the CLI to skip checking for the latest available version on startup + envProperties.put("TIDELIFT_SKIP_UPDATE_CHECK", "1"); + envProperties.put("TIDELIFT_ALLOW_MANIFEST_FAILURES", "1"); + envProperties.put("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE", "1"); + envProperties.put("TIDELIFT_CLI_SQ_SERVER_VERSION", server.getVersion()); + envProperties.putAll(ScaProperties.buildFromScannerProperties(configuration)); + + LOG.info("Running command: {}", args); + LOG.info("Environment properties: {}", envProperties); + + Consumer<String> logConsumer = LOG.atLevel(Level.INFO)::log; + processWrapperFactory.create(module.getWorkDir(), logConsumer, logConsumer, envProperties, args.toArray(new String[0])).execute(); + LOG.info("Generated manifests archive file: {}", archiveName); + success = true; + return archivePath.toFile(); + } finally { + telemetryCache.put("scanner.sca.execution.cli.duration", String.valueOf(system2.now() - startTime)); + telemetryCache.put("scanner.sca.execution.cli.success", String.valueOf(success)); + } + } + + private @Nullable String getExcludeFlag(DefaultInputModule module, DefaultConfiguration configuration) throws IOException { + List<String> configExcludedPaths = getConfigExcludedPaths(configuration, projectExclusionFilters); + List<String> scmIgnoredPaths = getScmIgnoredPaths(module); + + ArrayList<String> mergedExclusionPaths = new ArrayList<>(); + mergedExclusionPaths.addAll(configExcludedPaths); + mergedExclusionPaths.addAll(scmIgnoredPaths); + + String workDirExcludedPath = getWorkDirExcludedPath(module); + if (workDirExcludedPath != null) { + mergedExclusionPaths.add(workDirExcludedPath); + } + + if (mergedExclusionPaths.isEmpty()) { + return null; + } + + // wrap each exclusion path in quotes to handle commas in file paths + return toCsvString(mergedExclusionPaths); + } + + private static List<String> getConfigExcludedPaths(DefaultConfiguration configuration, ProjectExclusionFilters projectExclusionFilters) { + String[] sonarExclusions = projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN); + String[] scaExclusions = configuration.getStringArray(SCA_EXCLUSIONS_KEY); + String[] scaExclusionsLegacy = configuration.getStringArray(LEGACY_SCA_EXCLUSIONS_KEY); + + return Stream.of(sonarExclusions, scaExclusions, scaExclusionsLegacy) + .flatMap(Arrays::stream) + .distinct() + .toList(); + } + + private List<String> getScmIgnoredPaths(DefaultInputModule module) { + var scmProvider = scmConfiguration.provider(); + // Only Git is supported at this time + if (scmProvider == null || scmProvider.key() == null || !scmProvider.key().equals("git")) { + return List.of(); + } + + if (scmConfiguration.isExclusionDisabled()) { + // The user has opted out of using the SCM exclusion rules + return List.of(); + } + + Path baseDirPath = module.getBaseDir(); + List<String> scmIgnoredPaths = JGitUtils.getAllIgnoredPaths(baseDirPath); + if (scmIgnoredPaths.isEmpty()) { + return List.of(); + } + return scmIgnoredPaths.stream() + .map(ignoredPathRel -> { + + boolean isDirectory = false; + try { + isDirectory = Files.isDirectory(baseDirPath.resolve(ignoredPathRel.replace("/", File.separator))); + } catch (java.nio.file.InvalidPathException e) { + // if it's not a valid path, it's not a directory so we can just pass to the Tidelift CLI + } + // Directories need to get turned into a glob for the Tidelift CLI + return isDirectory ? (ignoredPathRel + "/**") : ignoredPathRel; + }) + .toList(); + } + + private static String getWorkDirExcludedPath(DefaultInputModule module) { + Path baseDir = module.getBaseDir().toAbsolutePath().normalize(); + Path workDir = module.getWorkDir().toAbsolutePath().normalize(); + + if (workDir.startsWith(baseDir)) { + // workDir is inside baseDir, so return the relative path as a glob + Path relativeWorkDir = baseDir.relativize(workDir); + return relativeWorkDir + "/**"; + } + + return null; + } + + private static String toCsvString(List<String> values) throws IOException { + StringWriter sw = new StringWriter(); + try (CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT)) { + printer.printRecord(values); + } + // trim to remove the trailing newline + return sw.toString().trim(); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java new file mode 100644 index 00000000000..143e144c2dc --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java @@ -0,0 +1,92 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.report.ReportPublisher; +import org.sonar.scanner.repository.featureflags.FeatureFlagsRepository; + +/** + * The ScaExecutor class is the main entrypoint for generating manifest dependency + * data during a Sonar scan and passing that data in the report so that it can + * be analyzed further by SQ server. + */ +public class ScaExecutor { + private static final Logger LOG = LoggerFactory.getLogger(ScaExecutor.class); + private static final String SCA_FEATURE_NAME = "sca"; + + private final CliCacheService cliCacheService; + private final CliService cliService; + private final ReportPublisher reportPublisher; + private final FeatureFlagsRepository featureFlagsRepository; + private final DefaultConfiguration configuration; + + public ScaExecutor(CliCacheService cliCacheService, CliService cliService, ReportPublisher reportPublisher, FeatureFlagsRepository featureFlagsRepository, + DefaultConfiguration configuration) { + this.cliCacheService = cliCacheService; + this.cliService = cliService; + this.reportPublisher = reportPublisher; + this.featureFlagsRepository = featureFlagsRepository; + this.configuration = configuration; + } + + public void execute(DefaultInputModule root) { + // Global feature flag + if (!featureFlagsRepository.isEnabled(SCA_FEATURE_NAME)) { + LOG.info("Dependency analysis skipped"); + return; + } + + // Project or scanner level feature flag + if (!configuration.getBoolean("sonar.sca.enabled").orElse(true)) { + LOG.info("Dependency analysis disabled for this project"); + return; + } + + var stopwatch = new StopWatch(); + stopwatch.start(); + LOG.info("Checking for latest CLI"); + File cliFile = cliCacheService.cacheCli(); + + LOG.info("Collecting manifests for the dependency analysis..."); + if (cliFile.exists()) { + try { + File generatedZip = cliService.generateManifestsArchive(root, cliFile, configuration); + LOG.debug("Zip ready for report: {}", generatedZip); + reportPublisher.getWriter().writeScaFile(generatedZip); + LOG.debug("Manifest zip written to report"); + } catch (IOException | IllegalStateException e) { + LOG.error("Error gathering manifests", e); + } finally { + stopwatch.stop(); + if (LOG.isInfoEnabled()) { + LOG.info("Load SCA project dependencies (done) | time={}ms", stopwatch.getTime(TimeUnit.MILLISECONDS)); + } + } + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java new file mode 100644 index 00000000000..a697aef3e20 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaProperties.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.sonar.scanner.config.DefaultConfiguration; + +public class ScaProperties { + private static final Pattern sonarScaPropertyRegex = Pattern.compile("^sonar\\.sca\\.([a-zA-Z]+)$"); + private static final String SONAR_SCA_PREFIX = "sonar.sca."; + private static final Set<String> IGNORED_PROPERTIES = Set.of( + // sonar.sca.exclusions is a special case which we handle when building --exclude + "sonar.sca.exclusions", + // excludedManifests is a special case which we handle when building --exclude + "sonar.sca.excludedManifests", + // keep recursive enabled to better match sonar-scanner behavior + "sonar.sca.recursiveManifestSearch"); + + private ScaProperties() { + } + + /** + * Build a map of environment variables from the sonar.sca.* properties in the configuration. + * The environment variable names are derived from the property names by removing the sonar.sca. prefix + * and converting to upper snake case to be used with the Tidelift CLI with the value from the configuration. + * <p> + * Examples: + * <br> + * { "sonar.sca.propertyName" : "value" } becomes { "TIDELIFT_PROPERTY_NAME" : "value" } + * <br> + * { "sonar.someOtherProperty" : "value" } returns an empty map + * + * @param configuration the scanner configuration possibly containing sonar.sca.* properties + * @return a map of Tidelift CLI compatible environment variable names to their configuration values + */ + public static Map<String, String> buildFromScannerProperties(DefaultConfiguration configuration) { + HashMap<String, String> props = new HashMap<>(configuration.getProperties()); + + return props + .entrySet() + .stream() + .filter(entry -> entry.getKey().startsWith(SONAR_SCA_PREFIX)) + .filter(entry -> !IGNORED_PROPERTIES.contains(entry.getKey())) + .collect(Collectors.toMap(entry -> convertPropToEnvVariable(entry.getKey()), Map.Entry::getValue)); + } + + // convert sonar.sca.* to TIDELIFT_* and convert from camelCase to UPPER_SNAKE_CASE + private static String convertPropToEnvVariable(String propertyName) { + var regexMatcher = sonarScaPropertyRegex.matcher(propertyName); + + if (regexMatcher.matches() && regexMatcher.groupCount() == 1) { + var tideliftNamespace = "TIDELIFT_"; + var convertedPropertyName = PropertyNamingStrategies.UpperSnakeCaseStrategy.INSTANCE.translate(regexMatcher.group(1)); + + return tideliftNamespace + convertedPropertyName; + } + + return propertyName; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java new file mode 100644 index 00000000000..b0f34909c27 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.scanner.sca; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java deleted file mode 100644 index 15912f8a510..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableModuleSettings.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import jakarta.annotation.Priority; -import org.sonar.api.config.internal.Settings; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ModuleConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(1) -public class MutableModuleSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableModuleSettings(ModuleConfiguration config) { - super(config.getDefinitions(), config.getEncryption()); - addProperties(config.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java deleted file mode 100644 index df24cbe81e5..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/MutableProjectSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.scan; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import org.sonar.api.config.internal.Settings; -import org.sonar.scanner.bootstrap.GlobalConfiguration; - -import jakarta.annotation.Priority; - -import static java.util.Objects.requireNonNull; - -/** - * @deprecated since 6.5 {@link ProjectConfiguration} used to be mutable, so keep a mutable copy for backward compatibility. - */ -@Deprecated -@Priority(2) -public class MutableProjectSettings extends Settings { - - private final Map<String, String> properties = new HashMap<>(); - - public MutableProjectSettings(GlobalConfiguration globalConfig) { - super(globalConfig.getDefinitions(), globalConfig.getEncryption()); - addProperties(globalConfig.getProperties()); - } - - public void complete(ProjectConfiguration projectConfig) { - addProperties(projectConfig.getProperties()); - } - - @Override - protected Optional<String> get(String key) { - return Optional.ofNullable(properties.get(key)); - } - - @Override - protected void set(String key, String value) { - properties.put( - requireNonNull(key, "key can't be null"), - requireNonNull(value, "value can't be null").trim()); - } - - @Override - protected void remove(String key) { - properties.remove(key); - } - - @Override - public Map<String, String> getProperties() { - return properties; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java index c12ec245924..e5543d4f9c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectConfigurationProvider.java @@ -26,7 +26,6 @@ import org.sonar.scanner.bootstrap.GlobalConfiguration; import org.sonar.scanner.bootstrap.GlobalServerSettings; import org.springframework.context.annotation.Bean; - public class ProjectConfigurationProvider { private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -37,7 +36,7 @@ public class ProjectConfigurationProvider { @Bean("ProjectConfiguration") public ProjectConfiguration provide(DefaultInputProject project, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, - ProjectServerSettings projectServerSettings, MutableProjectSettings projectSettings) { + ProjectServerSettings projectServerSettings) { Map<String, String> settings = new LinkedHashMap<>(); settings.putAll(globalServerSettings.properties()); settings.putAll(projectServerSettings.properties()); @@ -45,10 +44,7 @@ public class ProjectConfigurationProvider { settings = sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(settings, globalServerSettings.properties()); - ProjectConfiguration projectConfig = new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); - projectSettings.complete(projectConfig); - return projectConfig; + return new ProjectConfiguration(globalConfig.getDefinitions(), globalConfig.getEncryption(), settings); } - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java index 4315c762481..8ddb889912d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringModuleScanContainer.java @@ -54,7 +54,6 @@ public class SpringModuleScanContainer extends SpringComponentContainer { add( module.definition(), module, - MutableModuleSettings.class, SonarGlobalPropertiesFilter.class, ModuleConfigurationProvider.class, @@ -68,8 +67,7 @@ public class SpringModuleScanContainer extends SpringComponentContainer { ModuleSensorOptimizer.class, ModuleSensorContext.class, - ModuleSensorExtensionDictionary.class - ); + ModuleSensorExtensionDictionary.class); } private void addExtensions() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java index ef57ea5a076..ead791bdeaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/SpringProjectScanContainer.java @@ -19,9 +19,9 @@ */ package org.sonar.scanner.scan; +import jakarta.annotation.Priority; import java.util.Collection; import java.util.Set; -import jakarta.annotation.Priority; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.Plugin; @@ -52,6 +52,9 @@ import org.sonar.scanner.postjob.PostJobsExecutor; import org.sonar.scanner.qualitygate.QualityGateCheck; import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.rule.QProfileVerifier; +import org.sonar.scanner.sca.CliCacheService; +import org.sonar.scanner.sca.CliService; +import org.sonar.scanner.sca.ScaExecutor; import org.sonar.scanner.scan.filesystem.FileIndexer; import org.sonar.scanner.scan.filesystem.InputFileFilterRepository; import org.sonar.scanner.scan.filesystem.LanguageDetection; @@ -131,7 +134,12 @@ public class SpringProjectScanContainer extends SpringComponentContainer { // file system InputFileFilterRepository.class, FileIndexer.class, - ProjectFileIndexer.class); + ProjectFileIndexer.class, + + // SCA + CliService.class, + CliCacheService.class, + ScaExecutor.class); } static ExtensionMatcher getScannerProjectExtensionsFilter() { @@ -172,6 +180,9 @@ public class SpringProjectScanContainer extends SpringComponentContainer { LOG.info("------------- Run sensors on project"); getComponentByType(ProjectSensorsExecutor.class).execute(); + LOG.info("------------- Gather SCA dependencies on project"); + getComponentByType(ScaExecutor.class).execute(tree.root()); + getComponentByType(ScmPublisher.class).publish(); getComponentByType(CpdExecutor.class).execute(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java index 314e923ce71..242bc015574 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitor.java @@ -24,17 +24,15 @@ import java.nio.file.AccessDeniedException; import java.nio.file.FileSystemLoopException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; -import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.DosFileAttributes; -import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; public class DirectoryFileVisitor implements FileVisitor<Path> { @@ -43,27 +41,31 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { private final FileVisitAction fileVisitAction; private final DefaultInputModule module; private final ModuleExclusionFilters moduleExclusionFilters; - private final InputModuleHierarchy inputModuleHierarchy; private final InputFile.Type type; + private final HiddenFilesVisitorHelper hiddenFilesVisitorHelper; - DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, - InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) { + DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + InputModuleHierarchy inputModuleHierarchy, InputFile.Type type, HiddenFilesProjectData hiddenFilesProjectData) { this.fileVisitAction = fileVisitAction; this.module = module; this.moduleExclusionFilters = moduleExclusionFilters; this.inputModuleHierarchy = inputModuleHierarchy; this.type = type; + this.hiddenFilesVisitorHelper = new HiddenFilesVisitorHelper(hiddenFilesProjectData, module, moduleConfig); } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE; + if (hiddenFilesVisitorHelper.shouldVisitDir(dir)) { + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!Files.isHidden(file)) { + if (hiddenFilesVisitorHelper.shouldVisitFile(file)) { fileVisitAction.execute(file); } return FileVisitResult.CONTINUE; @@ -129,25 +131,12 @@ public class DirectoryFileVisitor implements FileVisitor<Path> { @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + hiddenFilesVisitorHelper.exitDirectory(dir); return FileVisitResult.CONTINUE; } - private static boolean isHidden(Path path) throws IOException { - if (SystemUtils.IS_OS_WINDOWS) { - try { - DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - return dosFileAttributes.isHidden(); - } catch (UnsupportedOperationException e) { - return path.toFile().isHidden(); - } - } else { - return Files.isHidden(path); - } - } - @FunctionalInterface interface FileVisitAction { void execute(Path file) throws IOException; } } - diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 7f31c949132..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -63,12 +63,13 @@ public class FileIndexer { private final ModuleRelativePathWarner moduleRelativePathWarner; private final InputFileFilterRepository inputFileFilterRepository; private final Languages languages; + private final HiddenFilesProjectData hiddenFilesProjectData; public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader, MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, - InputFileFilterRepository inputFileFilterRepository, Languages languages) { + InputFileFilterRepository inputFileFilterRepository, Languages languages, HiddenFilesProjectData hiddenFilesProjectData) { this.project = project; this.scannerComponentIdGenerator = scannerComponentIdGenerator; this.componentStore = componentStore; @@ -83,15 +84,18 @@ public class FileIndexer { this.moduleRelativePathWarner = moduleRelativePathWarner; this.inputFileFilterRepository = inputFileFilterRepository; this.languages = languages; + this.hiddenFilesProjectData = hiddenFilesProjectData; } - void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, - Type type, ProgressReport progressReport) { + void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile, Type type, + ProgressReport progressReport) { Path projectRelativePath = project.getBaseDir().relativize(sourceFile); Path moduleRelativePath = module.getBaseDir().relativize(sourceFile); // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, @@ -102,11 +106,12 @@ public class FileIndexer { language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy, - scmChangedFiles.getOldRelativeFilePath(sourceFile)); + scmChangedFiles.getOldRelativeFilePath(sourceFile), + isHidden); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()), f -> f.setStatus(statusDetection.findStatusFromScm(f))); - if (language != null && isPublishAllFiles(language.key())) { + if (!isHidden && language != null && isPublishAllFiles(language.key())) { inputFile.setPublished(true); } if (!accept(inputFile)) { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java index 544fe46c43b..a87c5f11fc9 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FilePreprocessor.java @@ -147,22 +147,35 @@ public class FilePreprocessor { return true; } - Path target = Files.readSymbolicLink(absolutePath); - if (!Files.exists(target)) { + Optional<Path> target = resolvePathToTarget(absolutePath); + if (target.isEmpty() || !Files.exists(target.get())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file that does not exist.", absolutePath); return false; } - if (!target.startsWith(project.getBaseDir())) { + if (!target.get().startsWith(project.getBaseDir())) { LOG.warn("File '{}' is ignored. It is a symbolic link targeting a file not located in project basedir.", absolutePath); return false; } - if (!target.startsWith(moduleBaseDirectory)) { + if (!target.get().startsWith(moduleBaseDirectory)) { LOG.info("File '{}' is ignored. It is a symbolic link targeting a file not located in module basedir.", absolutePath); return false; } return true; } + + private static Optional<Path> resolvePathToTarget(Path symbolicLinkAbsolutePath) throws IOException { + Path target = Files.readSymbolicLink(symbolicLinkAbsolutePath); + if (target.isAbsolute()) { + return Optional.of(target); + } + + try { + return Optional.of(symbolicLinkAbsolutePath.getParent().resolve(target).toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize()); + } catch (IOException e) { + return Optional.empty(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java new file mode 100644 index 00000000000..d779a054455 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +public class HiddenFilesProjectData { + + final Map<DefaultInputModule, Set<Path>> hiddenFilesByModule = new HashMap<>(); + private final SonarUserHome sonarUserHome; + private Path cachedSonarUserHomePath; + + public HiddenFilesProjectData(SonarUserHome sonarUserHome) { + this.sonarUserHome = sonarUserHome; + } + + public void markAsHiddenFile(Path file, DefaultInputModule module) { + hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); + } + + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { + Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); + } + return false; + } + + public Path getCachedSonarUserHomePath() throws IOException { + if (cachedSonarUserHomePath == null) { + cachedSonarUserHomePath = resolveRealPath(sonarUserHome.getPath()); + } + return cachedSonarUserHomePath; + } + + public void clearHiddenFilesData() { + // Allowing the GC to collect the map, should only be done after all indexing is complete + hiddenFilesByModule.clear(); + } + + public Path resolveRealPath(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + return path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + } + return path; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java new file mode 100644 index 00000000000..607a859ef44 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelper.java @@ -0,0 +1,112 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.DosFileAttributes; +import org.apache.commons.lang3.SystemUtils; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.scan.ModuleConfiguration; + +public class HiddenFilesVisitorHelper { + + private static final String EXCLUDE_HIDDEN_FILES_PROPERTY = "sonar.scanner.excludeHiddenFiles"; + private final HiddenFilesProjectData hiddenFilesProjectData; + private final DefaultInputModule module; + final boolean excludeHiddenFiles; + private Path moduleWorkDir; + Path rootHiddenDir; + + public HiddenFilesVisitorHelper(HiddenFilesProjectData hiddenFilesProjectData, DefaultInputModule module, ModuleConfiguration moduleConfig) { + this.hiddenFilesProjectData = hiddenFilesProjectData; + this.module = module; + this.excludeHiddenFiles = moduleConfig.getBoolean(EXCLUDE_HIDDEN_FILES_PROPERTY).orElse(false); + } + + public boolean shouldVisitDir(Path path) throws IOException { + boolean isHidden = isHiddenDir(path); + + if (isHidden && (excludeHiddenFiles || isExcludedHiddenDirectory(path))) { + return false; + } + if (isHidden) { + enterHiddenDirectory(path); + } + return true; + } + + private boolean isExcludedHiddenDirectory(Path path) throws IOException { + return getCachedModuleWorkDir().equals(path) || hiddenFilesProjectData.getCachedSonarUserHomePath().equals(path); + } + + void enterHiddenDirectory(Path dir) { + if (!insideHiddenDirectory()) { + rootHiddenDir = dir; + } + } + + public void exitDirectory(Path path) { + if (insideHiddenDirectory() && rootHiddenDir.equals(path)) { + resetRootHiddenDir(); + } + } + + void resetRootHiddenDir() { + this.rootHiddenDir = null; + } + + public boolean shouldVisitFile(Path path) throws IOException { + boolean isHidden = insideHiddenDirectory() || Files.isHidden(path); + + if (!excludeHiddenFiles && isHidden) { + hiddenFilesProjectData.markAsHiddenFile(path, module); + } + + return !excludeHiddenFiles || !isHidden; + } + + private Path getCachedModuleWorkDir() throws IOException { + if (moduleWorkDir == null) { + moduleWorkDir = hiddenFilesProjectData.resolveRealPath(module.getWorkDir()); + } + return moduleWorkDir; + } + + // visible for testing + boolean insideHiddenDirectory() { + return rootHiddenDir != null; + } + + protected static boolean isHiddenDir(Path path) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + try { + DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + return dosFileAttributes.isHidden(); + } catch (UnsupportedOperationException e) { + return path.toFile().isHidden(); + } + } else { + return Files.isHidden(path); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java index 6ef26dafd07..68b6d1db580 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java @@ -19,12 +19,15 @@ */ package org.sonar.scanner.scan.filesystem; +import java.util.Set; import java.util.SortedSet; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.sonar.api.batch.ScannerSide; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputModule; -import org.sonar.api.batch.fs.internal.SensorStrategy; import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.SensorStrategy; @ScannerSide public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @@ -73,11 +76,29 @@ public class ModuleInputComponentStore extends DefaultFileSystem.Cache { @Override public Iterable<InputFile> getFilesByName(String filename) { - return inputComponentStore.getFilesByName(filename); + Iterable<InputFile> allFilesByName = inputComponentStore.getFilesByName(filename); + if (strategy.isGlobal()) { + return allFilesByName; + } + + return filterByModule(allFilesByName); } @Override public Iterable<InputFile> getFilesByExtension(String extension) { - return inputComponentStore.getFilesByExtension(extension); + Iterable<InputFile> allFilesByExtension = inputComponentStore.getFilesByExtension(extension); + if (strategy.isGlobal()) { + return allFilesByExtension; + } + + return filterByModule(allFilesByExtension); + } + + private Iterable<InputFile> filterByModule(Iterable<InputFile> projectInputFiles) { + Set<InputFile> projectInputFilesSet = StreamSupport.stream(projectInputFiles.spliterator(), false) + .collect(Collectors.toSet()); + return StreamSupport.stream(inputComponentStore.filesByModule(moduleKey).spliterator(), false) + .filter(projectInputFilesSet::contains) + .toList(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java index 5daa384d3ac..9c969f6ae20 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MutableFileSystem.java @@ -25,35 +25,54 @@ import org.sonar.api.batch.fs.FilePredicates; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.predicates.ChangedFilePredicate; +import org.sonar.api.batch.fs.internal.predicates.NonHiddenFilesPredicate; public class MutableFileSystem extends DefaultFileSystem { - private boolean restrictToChangedFiles = false; + + boolean restrictToChangedFiles = false; + boolean allowHiddenFileAnalysis = false; public MutableFileSystem(Path baseDir, Cache cache, FilePredicates filePredicates) { super(baseDir, cache, filePredicates); } - public MutableFileSystem(Path baseDir) { + MutableFileSystem(Path baseDir) { super(baseDir); } @Override public Iterable<InputFile> inputFiles(FilePredicate requestPredicate) { - if (restrictToChangedFiles) { - return super.inputFiles(new ChangedFilePredicate(requestPredicate)); - } - return super.inputFiles(requestPredicate); + return super.inputFiles(applyAdditionalPredicate(requestPredicate)); } @Override public InputFile inputFile(FilePredicate requestPredicate) { + return super.inputFile(applyAdditionalPredicate(requestPredicate)); + } + + private FilePredicate applyAdditionalPredicate(FilePredicate requestPredicate) { + return applyHiddenFilePredicate(applyChangedFilePredicate(requestPredicate)); + } + + private FilePredicate applyHiddenFilePredicate(FilePredicate predicate) { + if (allowHiddenFileAnalysis) { + return predicate; + } + return predicates().and(new NonHiddenFilesPredicate(), predicate); + } + + private FilePredicate applyChangedFilePredicate(FilePredicate predicate) { if (restrictToChangedFiles) { - return super.inputFile(new ChangedFilePredicate(requestPredicate)); + return predicates().and(new ChangedFilePredicate(), predicate); } - return super.inputFile(requestPredicate); + return predicate; } public void setRestrictToChangedFiles(boolean restrictToChangedFiles) { this.restrictToChangedFiles = restrictToChangedFiles; } + + public void setAllowHiddenFileAnalysis(boolean allowHiddenFileAnalysis) { + this.allowHiddenFileAnalysis = allowHiddenFileAnalysis; + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java index 97e449fcb26..c1349872c24 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFileIndexer.java @@ -62,6 +62,7 @@ public class ProjectFileIndexer { private final FileIndexer fileIndexer; private final ProjectFilePreprocessor projectFilePreprocessor; private final AnalysisWarnings analysisWarnings; + private final HiddenFilesProjectData hiddenFilesProjectData; private ProgressReport progressReport; @@ -69,7 +70,7 @@ public class ProjectFileIndexer { SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, - ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) { + ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings, HiddenFilesProjectData hiddenFilesProjectData) { this.componentStore = componentStore; this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.inputModuleHierarchy = inputModuleHierarchy; @@ -81,6 +82,7 @@ public class ProjectFileIndexer { this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions; this.projectFilePreprocessor = projectFilePreprocessor; this.analysisWarnings = analysisWarnings; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void index() { @@ -91,10 +93,10 @@ public class ProjectFileIndexer { projectCoverageAndDuplicationExclusions.log(" "); indexModulesRecursively(inputModuleHierarchy.root()); + hiddenFilesProjectData.clearHiddenFilesData(); int totalIndexed = componentStore.inputFiles().size(); - progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); - + progressReport.stopAndLogTotalTime(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed"); } private void indexModulesRecursively(DefaultInputModule module) { @@ -118,15 +120,15 @@ public class ProjectFileIndexer { moduleCoverageAndDuplicationExclusions.log(" "); } List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module); - indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); + indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN); projectFilePreprocessor.getTestSourcesByModule(module) - .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); + .ifPresent(tests -> indexFiles(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST)); } private static void logPaths(String label, Path baseDir, List<Path> paths) { if (!paths.isEmpty()) { StringBuilder sb = new StringBuilder(label); - for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) { + for (Iterator<Path> it = paths.iterator(); it.hasNext();) { Path file = it.next(); Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file); if (relativePathToBaseDir.isEmpty()) { @@ -148,12 +150,13 @@ public class ProjectFileIndexer { } } - private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, + private void indexFiles(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, + ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type) { try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); + indexDirectory(module, moduleConfig, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type); } else { fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport); } @@ -163,18 +166,16 @@ public class ProjectFileIndexer { } } - private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, + private void indexDirectory(DefaultInputModule module, ModuleConfiguration moduleConfig, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type) throws IOException { Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport), - module, moduleExclusionFilters, inputModuleHierarchy, type)); + module, moduleConfig, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); } private static String pluralizeFiles(int count) { return count == 1 ? "file" : "files"; } - - } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 033ab56d3d4..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -66,6 +66,7 @@ public class ProjectFilePreprocessor { private final LanguageDetection languageDetection; private final FilePreprocessor filePreprocessor; private final ProjectExclusionFilters projectExclusionFilters; + private final HiddenFilesProjectData hiddenFilesProjectData; private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -79,7 +80,7 @@ public class ProjectFilePreprocessor { public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy, GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings, LanguageDetection languageDetection, FilePreprocessor filePreprocessor, - ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) { + ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, HiddenFilesProjectData hiddenFilesProjectData) { this.analysisWarnings = analysisWarnings; this.scmConfiguration = scmConfiguration; this.inputModuleHierarchy = inputModuleHierarchy; @@ -92,6 +93,7 @@ public class ProjectFilePreprocessor { this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter; this.ignoreCommand = loadIgnoreCommand(); this.useScmExclusion = ignoreCommand != null; + this.hiddenFilesProjectData = hiddenFilesProjectData; } public void execute() { @@ -109,7 +111,7 @@ public class ProjectFilePreprocessor { int totalLanguagesDetected = languageDetection.getDetectedLanguages().size(); - progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), + progressReport.stopAndLogTotalTime(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected), pluralizeWithCount("preprocessed file", totalFilesPreprocessed))); int excludedFileByPatternCount = exclusionCounter.getByPatternsCount(); @@ -138,27 +140,31 @@ public class ProjectFilePreprocessor { // Default to index basedir when no sources provided List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles() .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath())); - List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, + List<Path> processedSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN, exclusionCounter); mainSourcesByModule.put(module, processedSources); totalFilesPreprocessed += processedSources.size(); module.getTestDirsOrFiles().ifPresent(tests -> { - List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); + List<Path> processedTestSources = processModuleSources(module, moduleConfig, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter); testSourcesByModule.put(module, processedTestSources); totalFilesPreprocessed += processedTestSources.size(); }); } - private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, + private List<Path> processModuleSources(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources, InputFile.Type type, ExclusionCounter exclusionCounter) { List<Path> processedFiles = new ArrayList<>(); try { for (Path dirOrFile : sources) { if (dirOrFile.toFile().isDirectory()) { - processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); + processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -167,12 +173,17 @@ public class ProjectFilePreprocessor { return processedFiles; } - private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path, + private List<Path> processDirectory(DefaultInputModule module, ModuleConfiguration moduleConfiguration, ModuleExclusionFilters moduleExclusionFilters, Path path, InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java index a08380cf9d8..10d75a4b3c5 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/AbstractSensorWrapper.java @@ -19,11 +19,11 @@ */ package org.sonar.scanner.sensor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor; import org.sonar.api.scanner.sensor.ProjectSensor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.sonar.scanner.scan.branch.BranchConfiguration; import org.sonar.scanner.scan.branch.BranchType; import org.sonar.scanner.scan.filesystem.MutableFileSystem; @@ -60,7 +60,12 @@ public abstract class AbstractSensorWrapper<G extends ProjectSensor> { if (sensorIsRestricted) { LOGGER.info("Sensor {} is restricted to changed files only", descriptor.name()); } + boolean allowHiddenFileAnalysis = descriptor.isProcessesHiddenFiles(); + if (allowHiddenFileAnalysis) { + LOGGER.debug("Sensor {} is allowed to analyze hidden files", descriptor.name()); + } fileSystem.setRestrictToChangedFiles(sensorIsRestricted); + fileSystem.setAllowHiddenFileAnalysis(allowHiddenFileAnalysis); wrappedSensor.execute(context); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java index 5fa6e33aac6..0ff5109124d 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/DefaultSensorStorage.java @@ -19,6 +19,16 @@ */ package org.sonar.scanner.sensor; +import static java.lang.Math.max; +import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DATA_KEY; +import static org.sonar.api.measures.CoreMetrics.LINES_KEY; +import static org.sonar.api.measures.CoreMetrics.PUBLIC_DOCUMENTED_API_DENSITY_KEY; +import static org.sonar.api.measures.CoreMetrics.TEST_SUCCESS_DENSITY_KEY; +import static org.sonar.api.utils.Preconditions.checkArgument; + +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.InputStream; import java.io.Serializable; import java.util.HashSet; import java.util.List; @@ -28,6 +38,7 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.fs.InputComponent; @@ -80,12 +91,6 @@ import org.sonar.scanner.repository.ContextPropertiesCache; import org.sonar.scanner.repository.TelemetryCache; import org.sonar.scanner.scan.branch.BranchConfiguration; -import static java.lang.Math.max; -import static org.sonar.api.measures.CoreMetrics.COMMENT_LINES_DATA_KEY; -import static org.sonar.api.measures.CoreMetrics.LINES_KEY; -import static org.sonar.api.measures.CoreMetrics.PUBLIC_DOCUMENTED_API_DENSITY_KEY; -import static org.sonar.api.measures.CoreMetrics.TEST_SUCCESS_DENSITY_KEY; - public class DefaultSensorStorage implements SensorStorage { private static final Logger LOG = LoggerFactory.getLogger(DefaultSensorStorage.class); @@ -122,6 +127,7 @@ public class DefaultSensorStorage implements SensorStorage { private final ScannerMetrics scannerMetrics; private final BranchConfiguration branchConfiguration; private final Set<String> alreadyLogged = new HashSet<>(); + private final Set<String> alreadyAddedData = new HashSet<>(); public DefaultSensorStorage(MetricFinder metricFinder, IssuePublisher moduleIssues, Configuration settings, ReportPublisher reportPublisher, SonarCpdBlockIndex index, ContextPropertiesCache contextPropertiesCache, TelemetryCache telemetryCache, ScannerMetrics scannerMetrics, BranchConfiguration branchConfiguration) { @@ -472,4 +478,23 @@ public class DefaultSensorStorage implements SensorStorage { writer.writeComponentSignificantCode(componentRef, protobuf); } + + public void storeAnalysisData(String key, String mimeType, InputStream data) { + checkArgument(!StringUtils.isBlank(key), "Key must not be null"); + checkArgument(!alreadyAddedData.contains(key), "A data with this key already exists"); + checkArgument(!StringUtils.isBlank(mimeType), "MimeType must not be null"); + checkArgument(data != null, "Data must not be null"); + alreadyAddedData.add(key); + try (data) { + ScannerReport.AnalysisData analysisData = ScannerReport.AnalysisData.newBuilder() + .setKey(key) + .setMimeType(mimeType) + .setData(ByteString.readFrom(data)) + .build(); + ScannerReportWriter writer = reportPublisher.getWriter(); + writer.appendAnalysisData(analysisData); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read data InputStream", e); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java index 5f28e7e283e..01b6c0c11cd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ModuleSensorContext.java @@ -28,7 +28,6 @@ import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.cache.ReadCache; import org.sonar.api.batch.sensor.cache.WriteCache; import org.sonar.api.config.Configuration; -import org.sonar.api.config.Settings; import org.sonar.scanner.bootstrap.ScannerPluginRepository; import org.sonar.scanner.cache.AnalysisCacheEnabled; import org.sonar.scanner.scan.branch.BranchConfiguration; @@ -38,11 +37,11 @@ public class ModuleSensorContext extends ProjectSensorContext { private final InputModule module; - public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, Settings mutableModuleSettings, FileSystem fs, ActiveRules activeRules, + public ModuleSensorContext(DefaultInputProject project, InputModule module, Configuration config, FileSystem fs, ActiveRules activeRules, DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, WriteCache writeCache, ReadCache readCache, AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepository) { - super(project, config, mutableModuleSettings, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, + super(project, config, fs, activeRules, sensorStorage, sonarRuntime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); this.module = module; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java index bac06a38645..54c86750eaf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sensor/ProjectSensorContext.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.sensor; +import java.io.InputStream; import java.io.Serializable; import javax.annotation.concurrent.ThreadSafe; import org.sonar.api.SonarRuntime; @@ -65,7 +66,6 @@ public class ProjectSensorContext implements SensorContext { static final NoOpNewAnalysisError NO_OP_NEW_ANALYSIS_ERROR = new NoOpNewAnalysisError(); - private final Settings mutableSettings; private final FileSystem fs; private final ActiveRules activeRules; private final DefaultSensorStorage sensorStorage; @@ -80,15 +80,14 @@ public class ProjectSensorContext implements SensorContext { private final ExecutingSensorContext executingSensorContext; private final ScannerPluginRepository pluginRepo; - public ProjectSensorContext(DefaultInputProject project, Configuration config, Settings mutableSettings, FileSystem fs, - ActiveRules activeRules, - DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, - WriteCache writeCache, ReadCache readCache, - AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, - ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { + public ProjectSensorContext(DefaultInputProject project, Configuration config, FileSystem fs, + ActiveRules activeRules, + DefaultSensorStorage sensorStorage, SonarRuntime sonarRuntime, BranchConfiguration branchConfiguration, + WriteCache writeCache, ReadCache readCache, + AnalysisCacheEnabled analysisCacheEnabled, UnchangedFilesHandler unchangedFilesHandler, + ExecutingSensorContext executingSensorContext, ScannerPluginRepository pluginRepo) { this.project = project; this.config = config; - this.mutableSettings = mutableSettings; this.fs = fs; this.activeRules = activeRules; this.sensorStorage = sensorStorage; @@ -104,7 +103,7 @@ public class ProjectSensorContext implements SensorContext { @Override public Settings settings() { - return mutableSettings; + throw new UnsupportedOperationException("This method is not supported anymore"); } @Override @@ -233,6 +232,15 @@ public class ProjectSensorContext implements SensorContext { } @Override + public void addAnalysisData(String key, String mimeType, InputStream data) { + if (isSonarSourcePlugin()) { + this.sensorStorage.storeAnalysisData(key, mimeType, data); + } else { + throw new IllegalStateException("Analysis data can only be added by SonarSource plugins"); + } + } + + @Override public NewSignificantCode newSignificantCode() { return new DefaultSignificantCode(sensorStorage); } @@ -250,4 +258,5 @@ public class ProjectSensorContext implements SensorContext { } return false; } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java index 0742740bba6..a481f4a54f4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java @@ -22,7 +22,9 @@ package org.sonar.scm.git; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,6 +34,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.diff.RawTextComparator; @@ -238,7 +241,7 @@ public class CompositeBlameCommand extends BlameCommand { break; } linesList.add(new BlameLine() - .date(fileBlame.getCommitDates()[i]) + .date(toDate(fileBlame.getCommitDates()[i])) .revision(fileBlame.getCommitHashes()[i]) .author(fileBlame.getAuthorEmails()[i])); } @@ -251,4 +254,8 @@ public class CompositeBlameCommand extends BlameCommand { } } + private static @Nullable Date toDate(@Nullable Instant commitDate) { + return commitDate != null ? Date.from(commitDate) : null; + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java index 162e7f71eff..d8aef57bc2f 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java @@ -22,6 +22,7 @@ package org.sonar.scm.git; import java.util.Arrays; import java.util.List; import org.eclipse.jgit.util.FS; +import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scm.git.strategy.DefaultBlameStrategy; public final class GitScmSupport { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java index 972a8ce8da3..bc38a55e619 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java @@ -21,6 +21,9 @@ package org.sonar.scm.git; import java.io.IOException; import java.nio.file.Path; +import java.util.List; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; @@ -41,4 +44,27 @@ public class JGitUtils { throw new IllegalStateException("Unable to open Git repository", e); } } + + // Return a list of scm ignored paths relative to the baseDir. + public static List<String> getAllIgnoredPaths(Path baseDir) { + try (Repository repo = buildRepository(baseDir)) { + Path workTreePath = repo.getWorkTree().toPath(); + Path baseDirAbs = baseDir.toAbsolutePath().normalize(); + + try (Git git = new Git(repo)) { + return git.status().call().getIgnoredNotInIndex().stream() + // Convert to absolute path + .map(filePathStr -> workTreePath.resolve(filePathStr).normalize()) + // Exclude any outside of the baseDir + .filter(filePath -> filePath.startsWith(baseDirAbs)) + // Make path relative to the baseDir + .map(baseDir::relativize) + .map(Path::toString) + .sorted() + .toList(); + } catch (GitAPIException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java index cac4370e09e..8f066727e21 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.Date; import java.util.LinkedList; import java.util.List; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.sonar.api.batch.scm.BlameLine; import org.sonar.api.utils.System2; import org.sonar.api.utils.Version; +import org.sonar.core.util.ProcessWrapperFactory; import org.springframework.beans.factory.annotation.Autowired; import static java.util.Collections.emptyList; @@ -61,6 +63,7 @@ public class NativeGitBlameCommand { private final System2 system; private final ProcessWrapperFactory processWrapperFactory; + private final Consumer<String> stderrConsumer = line -> LOG.debug("[stderr] {}", line); private String gitCommand; @Autowired @@ -84,7 +87,7 @@ public class NativeGitBlameCommand { try { this.gitCommand = locateDefaultGit(); MutableString stdOut = new MutableString(); - this.processWrapperFactory.create(null, l -> stdOut.string = l, gitCommand, "--version").execute(); + this.processWrapperFactory.create(null, l -> stdOut.string = l, stderrConsumer, gitCommand, "--version").execute(); return stdOut.string != null && stdOut.string.startsWith("git version") && isCompatibleGitVersion(stdOut.string); } catch (Exception e) { LOG.debug("Failed to find git native client", e); @@ -108,7 +111,7 @@ public class NativeGitBlameCommand { // To avoid it we use where.exe to find git binary only in PATH. LOG.debug("Looking for git command in the PATH using where.exe (Windows)"); List<String> whereCommandResult = new LinkedList<>(); - this.processWrapperFactory.create(null, whereCommandResult::add, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") + this.processWrapperFactory.create(null, whereCommandResult::add, stderrConsumer, "C:\\Windows\\System32\\where.exe", "$PATH:git.exe") .execute(); if (!whereCommandResult.isEmpty()) { @@ -119,18 +122,19 @@ public class NativeGitBlameCommand { throw new IllegalStateException("git.exe not found in PATH. PATH value was: " + system.property("PATH")); } - public List<BlameLine> blame(Path baseDir, String fileName) throws Exception { + public List<BlameLine> blame(Path baseDir, String fileName) throws IOException { BlameOutputProcessor outputProcessor = new BlameOutputProcessor(); - try { - this.processWrapperFactory.create( - baseDir, - outputProcessor::process, - gitCommand, - GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), - BLAME_COMMAND, - BLAME_LINE_PORCELAIN_FLAG, IGNORE_WHITESPACES, FILENAME_SEPARATOR_FLAG, fileName) - .execute(); - } catch (UncommittedLineException e) { + var processWrapper = this.processWrapperFactory.create( + baseDir, + outputProcessor::process, + stderrConsumer, + gitCommand, + GIT_DIR_FLAG, String.format(GIT_DIR_ARGUMENT, baseDir), GIT_DIR_FORCE_FLAG, baseDir.toString(), + BLAME_COMMAND, + BLAME_LINE_PORCELAIN_FLAG, IGNORE_WHITESPACES, FILENAME_SEPARATOR_FLAG, fileName); + outputProcessor.setProcessWrapper(processWrapper); + processWrapper.execute(); + if (outputProcessor.hasEncounteredUncommittedLine()) { LOG.debug("Unable to blame file '{}' - it has uncommitted changes", fileName); return emptyList(); } @@ -142,6 +146,8 @@ public class NativeGitBlameCommand { private String sha1 = null; private String committerTime = null; private String authorMail = null; + private ProcessWrapperFactory.ProcessWrapper processWrapper = null; + private volatile boolean encounteredUncommittedLine = false; public List<BlameLine> getBlameLines() { return blameLines; @@ -160,11 +166,16 @@ public class NativeGitBlameCommand { authorMail = matcher.group(1); } if (authorMail.equals("not.committed.yet")) { - throw new UncommittedLineException(); + encounteredUncommittedLine = true; + processWrapper.destroy(); } } } + public boolean hasEncounteredUncommittedLine() { + return encounteredUncommittedLine; + } + private void saveEntry() { checkState(authorMail != null, "Did not find an author email for an entry"); checkState(committerTime != null, "Did not find a committer time for an entry"); @@ -181,6 +192,10 @@ public class NativeGitBlameCommand { sha1 = null; committerTime = null; } + + public void setProcessWrapper(ProcessWrapperFactory.ProcessWrapper processWrapper) { + this.processWrapper = processWrapper; + } } private static boolean isCompatibleGitVersion(String gitVersionCommandOutput) { @@ -207,7 +222,4 @@ public class NativeGitBlameCommand { String string; } - private static class UncommittedLineException extends RuntimeException { - - } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java deleted file mode 100644 index 9fa97ea1cab..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ProcessWrapperFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scm.git; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.Scanner; -import java.util.function.Consumer; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.lang.String.format; -import static java.lang.String.join; -import static java.nio.charset.StandardCharsets.UTF_8; - -public class ProcessWrapperFactory { - private static final Logger LOG = LoggerFactory.getLogger(ProcessWrapperFactory.class); - - public ProcessWrapperFactory() { - // nothing to do - } - - public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, String... command) { - return new ProcessWrapper(baseDir, stdOutLineConsumer, Map.of(), command); - } - - public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) { - return new ProcessWrapper(baseDir, stdOutLineConsumer, envVariables, command); - } - - static class ProcessWrapper { - - private final Path baseDir; - private final Consumer<String> stdOutLineConsumer; - private final String[] command; - private final Map<String, String> envVariables = new HashMap<>(); - - ProcessWrapper(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) { - this.baseDir = baseDir; - this.stdOutLineConsumer = stdOutLineConsumer; - this.envVariables.putAll(envVariables); - this.command = command; - } - - public void execute() throws IOException { - ProcessBuilder pb = new ProcessBuilder() - .command(command) - .directory(baseDir != null ? baseDir.toFile() : null); - envVariables.forEach(pb.environment()::put); - - Process p = pb.start(); - try { - processInputStream(p.getInputStream(), stdOutLineConsumer); - - processInputStream(p.getErrorStream(), line -> { - if (!line.isBlank()) { - LOG.debug(line); - } - }); - - int exit = p.waitFor(); - if (exit != 0) { - throw new IllegalStateException(format("Command execution exited with code: %d", exit)); - } - } catch (InterruptedException e) { - LOG.warn(format("Command [%s] interrupted", join(" ", command)), e); - Thread.currentThread().interrupt(); - } finally { - p.destroy(); - } - } - - private static void processInputStream(InputStream inputStream, Consumer<String> stringConsumer) { - try (Scanner scanner = new Scanner(new InputStreamReader(inputStream, UTF_8))) { - scanner.useDelimiter("\n"); - while (scanner.hasNext()) { - stringConsumer.accept(scanner.next()); - } - } - } - } - -} diff --git a/sonar-scanner-engine/src/main/resources/logback.xml b/sonar-scanner-engine/src/main/resources/logback.xml index ccd0dfe09b9..ddc2805f08a 100644 --- a/sonar-scanner-engine/src/main/resources/logback.xml +++ b/sonar-scanner-engine/src/main/resources/logback.xml @@ -1,17 +1,8 @@ <?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE configuration> -<!-- This logback configuration is used when the scanner engine is bootstrapped using the SonarScannerCli class. --> +<!-- This logback configuration is used when the scanner engine is bootstrapped using the ScannerMain class. --> <configuration scan="false"> - <import class="ch.qos.logback.core.ConsoleAppender"/> - - <appender name="STDOUT" class="ConsoleAppender"> - <encoder class="org.sonar.scanner.bootstrap.ScannerLogbackEncoder"/> - </appender> - - <root level="info"> - <appender-ref ref="STDOUT"/> - </root> <!-- BeanUtils generate too many DEBUG logs when sonar.verbose is set --> <logger name="org.apache.commons.beanutils.converters" level="WARN"/> @@ -31,4 +22,4 @@ <logger name="nl.altindag.ssl.util.CertificateUtils" level="INFO"/> -</configuration>
\ No newline at end of file +</configuration> diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java index 5091dcf5a3a..f5d88016944 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RulesSeverityDetectorTest.java @@ -22,12 +22,16 @@ package org.sonar.scanner.externalissue.sarif; import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import org.assertj.core.groups.Tuple; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.slf4j.event.Level; -import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.sarif.pojo.ReportingConfiguration; import org.sonar.sarif.pojo.ReportingDescriptor; import org.sonar.sarif.pojo.Result; @@ -44,12 +48,12 @@ import static org.sonar.sarif.pojo.Result.Level.WARNING; import static org.sonar.scanner.externalissue.sarif.ResultMapper.DEFAULT_IMPACT_SEVERITY; import static org.sonar.scanner.externalissue.sarif.ResultMapper.DEFAULT_SEVERITY; -public class RulesSeverityDetectorTest { +class RulesSeverityDetectorTest { private static final String DRIVER_NAME = "Test"; private static final String RULE_ID = "RULE_ID"; - @org.junit.Rule - public LogTester logTester = new LogTester().setLevel(Level.TRACE); + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); private final Run run = mock(Run.class); private final ReportingDescriptor rule = mock(ReportingDescriptor.class); @@ -59,8 +63,8 @@ public class RulesSeverityDetectorTest { private final ToolComponent extension = mock(ToolComponent.class); private final ReportingConfiguration defaultConfiguration = mock(ReportingConfiguration.class); - @Before - public void setUp() { + @BeforeEach + void setUp() { when(run.getResults()).thenReturn(List.of(result)); when(run.getTool()).thenReturn(tool); when(tool.getDriver()).thenReturn(driver); @@ -68,8 +72,8 @@ public class RulesSeverityDetectorTest { // We keep this test for backward compatibility until we remove the deprecated severity @Test - public void detectRulesSeverities_detectsCorrectlyResultDefinedRuleSeverities() { - Run run = mockResultDefinedRuleSeverities(); + void detectRulesSeverities_detectsCorrectlyResultDefinedRuleSeverities() { + mockResultDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeverities(run, DRIVER_NAME); @@ -78,8 +82,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeveritiesForNewTaxonomy_shouldReturnsEmptyMapAndLogsWarning_whenOnlyResultDefinedRuleSeverities() { - Run run = mockResultDefinedRuleSeverities(); + void detectRulesSeveritiesForNewTaxonomy_shouldReturnsEmptyMapAndLogsWarning_whenOnlyResultDefinedRuleSeverities() { + mockResultDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -88,8 +92,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeverities_detectsCorrectlyDriverDefinedRuleSeverities() { - Run run = mockDriverDefinedRuleSeverities(); + void detectRulesSeverities_detectsCorrectlyDriverDefinedRuleSeverities() { + mockDriverDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -103,9 +107,13 @@ public class RulesSeverityDetectorTest { assertDetectedRuleSeverities(rulesSeveritiesByRuleId, tuple(RULE_ID, WARNING)); } - @Test - public void detectRulesSeverities_detectsCorrectlyExtensionsDefinedRuleSeverities() { - Run run = mockExtensionsDefinedRuleSeverities(); + + + @ParameterizedTest + @NullAndEmptySource + void detectRulesSeverities_detectsCorrectlyExtensionsDefinedRuleSeverities(@Nullable Set<ReportingDescriptor> rules) { + when(driver.getRules()).thenReturn(rules); + mockExtensionsDefinedRuleSeverities(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -120,8 +128,8 @@ public class RulesSeverityDetectorTest { } @Test - public void detectRulesSeverities_returnsEmptyMapAndLogsWarning_whenUnableToDetectSeverities() { - Run run = mockUnsupportedRuleSeveritiesDefinition(); + void detectRulesSeverities_returnsEmptyMapAndLogsWarning_whenUnableToDetectSeverities() { + mockUnsupportedRuleSeveritiesDefinition(); Map<String, Result.Level> rulesSeveritiesByRuleId = RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, DRIVER_NAME); @@ -135,38 +143,33 @@ public class RulesSeverityDetectorTest { assertDetectedRuleSeverities(rulesSeveritiesByRuleId); } - private Run mockResultDefinedRuleSeverities() { + private void mockResultDefinedRuleSeverities() { when(run.getResults()).thenReturn(List.of(result)); when(result.getLevel()).thenReturn(WARNING); when(result.getRuleId()).thenReturn(RULE_ID); - return run; } - private Run mockDriverDefinedRuleSeverities() { + private void mockDriverDefinedRuleSeverities() { when(driver.getRules()).thenReturn(Set.of(rule)); when(rule.getId()).thenReturn(RULE_ID); when(rule.getDefaultConfiguration()).thenReturn(defaultConfiguration); when(defaultConfiguration.getLevel()).thenReturn(ReportingConfiguration.Level.WARNING); - return run; } - private Run mockExtensionsDefinedRuleSeverities() { - when(driver.getRules()).thenReturn(Set.of()); + private void mockExtensionsDefinedRuleSeverities() { when(tool.getExtensions()).thenReturn(Set.of(extension)); when(extension.getRules()).thenReturn(Set.of(rule)); when(rule.getId()).thenReturn(RULE_ID); when(rule.getDefaultConfiguration()).thenReturn(defaultConfiguration); when(defaultConfiguration.getLevel()).thenReturn(ReportingConfiguration.Level.WARNING); - return run; } - private Run mockUnsupportedRuleSeveritiesDefinition() { + private void mockUnsupportedRuleSeveritiesDefinition() { when(run.getTool()).thenReturn(tool); when(tool.getDriver()).thenReturn(driver); when(driver.getRules()).thenReturn(Set.of()); when(tool.getExtensions()).thenReturn(Set.of(extension)); when(extension.getRules()).thenReturn(Set.of()); - return run; } private void assertNoLogs() { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java index 90ddfadd9f3..164787f8cde 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/externalissue/sarif/RunMapperTest.java @@ -22,19 +22,19 @@ package org.sonar.scanner.externalissue.sarif; import java.util.List; import java.util.Map; import java.util.Set; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.event.Level; import org.sonar.api.batch.sensor.issue.NewExternalIssue; import org.sonar.api.batch.sensor.rule.NewAdHocRule; -import org.sonar.api.testfixtures.log.LogTester; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.sarif.pojo.ReportingDescriptor; import org.sonar.sarif.pojo.Result; import org.sonar.sarif.pojo.Run; @@ -44,13 +44,14 @@ import org.sonar.scanner.externalissue.sarif.RunMapper.RunMapperResult; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.sonar.sarif.pojo.Result.Level.WARNING; -@RunWith(MockitoJUnitRunner.class) -public class RunMapperTest { +@ExtendWith(MockitoExtension.class) +class RunMapperTest { private static final String TEST_DRIVER = "Test driver"; public static final String RULE_ID = "ruleId"; @@ -66,21 +67,21 @@ public class RunMapperTest { @Mock private ReportingDescriptor rule; - @Rule - public LogTester logTester = new LogTester(); + @RegisterExtension + public LogTesterJUnit5 logTester = new LogTesterJUnit5(); @InjectMocks private RunMapper runMapper; - @Before - public void setUp() { - when(run.getTool().getDriver().getName()).thenReturn(TEST_DRIVER); - when(run.getTool().getExtensions()).thenReturn(null); - when(rule.getId()).thenReturn(RULE_ID); + @BeforeEach + void setUp() { + lenient().when(run.getTool().getDriver().getName()).thenReturn(TEST_DRIVER); + lenient().when(run.getTool().getExtensions()).thenReturn(null); + lenient().when(rule.getId()).thenReturn(RULE_ID); } @Test - public void mapRun_shouldMapExternalIssues() { + void mapRun_shouldMapExternalIssues() { Result result1 = mock(Result.class); Result result2 = mock(Result.class); when(run.getResults()).thenReturn(List.of(result1, result2)); @@ -99,7 +100,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldMapExternalRules_whenDriverHasRulesAndNoExtensions() { + void mapRun_shouldMapExternalRules_whenDriverHasRulesAndNoExtensions() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); NewAdHocRule externalRule = mockMappedExternalRule(); @@ -115,7 +116,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldMapExternalRules_whenRulesInExtensions() { + void mapRun_shouldMapExternalRules_whenRulesInExtensions() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of()); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(Set.of(rule)); @@ -134,7 +135,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldNotFail_whenExtensionsDontHaveRules() { + void mapRun_shouldNotFail_whenExtensionsDontHaveRules() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(null); @@ -149,7 +150,7 @@ public class RunMapperTest { } @Test - public void mapRun_shouldNotFail_whenExtensionsHaveEmptyRules() { + void mapRun_shouldNotFail_whenExtensionsHaveEmptyRules() { when(run.getTool().getDriver().getRules()).thenReturn(Set.of(rule)); ToolComponent extension = mock(ToolComponent.class); when(extension.getRules()).thenReturn(Set.of()); @@ -164,7 +165,7 @@ public class RunMapperTest { } @Test - public void mapRun_ifRunIsEmpty_returnsEmptyList() { + void mapRun_ifRunIsEmpty_returnsEmptyList() { when(run.getResults()).thenReturn(List.of()); RunMapperResult runMapperResult = runMapper.mapRun(run); @@ -173,7 +174,7 @@ public class RunMapperTest { } @Test - public void mapRun_ifExceptionThrownByResultMapper_logsThemAndContinueProcessing() { + void mapRun_ifExceptionThrownByResultMapper_logsThemAndContinueProcessing() { Result result1 = mock(Result.class); Result result2 = mock(Result.class); when(run.getResults()).thenReturn(List.of(result1, result2)); @@ -194,7 +195,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfToolNotSet() { + void mapRun_failsIfToolNotSet() { when(run.getTool()).thenReturn(null); assertThatIllegalArgumentException() @@ -203,7 +204,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfDriverNotSet() { + void mapRun_failsIfDriverNotSet() { when(run.getTool().getDriver()).thenReturn(null); assertThatIllegalArgumentException() @@ -212,7 +213,7 @@ public class RunMapperTest { } @Test - public void mapRun_failsIfDriverNameIsNotSet() { + void mapRun_failsIfDriverNameIsNotSet() { when(run.getTool().getDriver().getName()).thenReturn(null); assertThatIllegalArgumentException() @@ -220,6 +221,25 @@ public class RunMapperTest { .withMessage("The run does not have a tool driver name defined."); } + @Test + void mapRun_shouldNotFail_whenDriverRulesNullAndExtensionsRulesNotNull() { + when(run.getTool().getDriver().getRules()).thenReturn(null); + ToolComponent extension = mock(ToolComponent.class); + when(extension.getRules()).thenReturn(Set.of(rule)); + when(run.getTool().getExtensions()).thenReturn(Set.of(extension)); + NewAdHocRule expectedRule = mock(NewAdHocRule.class); + when(ruleMapper.mapRule(rule, TEST_DRIVER, WARNING, WARNING)).thenReturn(expectedRule); + + try (MockedStatic<RulesSeverityDetector> detector = mockStatic(RulesSeverityDetector.class)) { + detector.when(() -> RulesSeverityDetector.detectRulesSeverities(run, TEST_DRIVER)).thenReturn(Map.of(RULE_ID, WARNING)); + detector.when(() -> RulesSeverityDetector.detectRulesSeveritiesForNewTaxonomy(run, TEST_DRIVER)).thenReturn(Map.of(RULE_ID, WARNING)); + + RunMapperResult runMapperResult = runMapper.mapRun(run); + assertThat(runMapperResult.getNewAdHocRules()).hasSize(1); + assertThat(runMapperResult.getNewAdHocRules().get(0)).isEqualTo(expectedRule); + } + } + private NewExternalIssue mockMappedExternalIssue(Result result) { NewExternalIssue externalIssue = mock(NewExternalIssue.class); when(result.getRuleId()).thenReturn(RULE_ID); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/issue/IssuePublisherTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/issue/IssuePublisherTest.java index 5a124b74214..24ce368bd7e 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/issue/IssuePublisherTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/issue/IssuePublisherTest.java @@ -19,14 +19,15 @@ */ package org.sonar.scanner.issue; -import java.io.IOException; +import java.io.File; import java.util.Collections; import java.util.HashSet; import java.util.List; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.junit.MockitoJUnitRunner; @@ -65,14 +66,13 @@ import static org.sonar.api.issue.impact.SoftwareQuality.MAINTAINABILITY; import static org.sonar.api.issue.impact.SoftwareQuality.RELIABILITY; @RunWith(MockitoJUnitRunner.class) -public class IssuePublisherTest { +class IssuePublisherTest { private static final RuleKey JAVA_RULE_KEY = RuleKey.of("java", "AvoidCycle"); - private static final RuleKey NOSONAR_RULE_KEY = RuleKey.of("java", "NoSonarCheck"); private DefaultInputProject project; - @Rule - public TemporaryFolder temp = new TemporaryFolder(); + @TempDir + public File temp; public IssueFilters filters = mock(IssueFilters.class); private final ActiveRulesBuilder activeRulesBuilder = new ActiveRulesBuilder(); @@ -80,12 +80,12 @@ public class IssuePublisherTest { private final DefaultInputFile file = new TestInputFileBuilder("foo", "src/Foo.php").initMetadata("Foo\nBar\nBiz\n").build(); private final ReportPublisher reportPublisher = mock(ReportPublisher.class, RETURNS_DEEP_STUBS); - @Before - public void prepare() throws IOException { + @BeforeEach + void prepare() { project = new DefaultInputProject(ProjectDefinition.create() .setKey("foo") - .setBaseDir(temp.newFolder()) - .setWorkDir(temp.newFolder())); + .setBaseDir(temp) + .setWorkDir(temp)); activeRulesBuilder.addRule(new NewActiveRule.Builder() .setRuleKey(JAVA_RULE_KEY) @@ -96,12 +96,12 @@ public class IssuePublisherTest { } @Test - public void ignore_null_active_rule() { - RuleKey INACTIVE_RULE_KEY = RuleKey.of("repo", "inactive"); + void ignore_null_active_rule() { + RuleKey inactiveRuleKey = RuleKey.of("repo", "inactive"); initModuleIssues(); DefaultIssue issue = new DefaultIssue(project) .at(new DefaultIssueLocation().on(file).at(file.selectLine(3)).message("Foo")) - .forRule(INACTIVE_RULE_KEY); + .forRule(inactiveRuleKey); boolean added = moduleIssues.initAndAddIssue(issue); assertThat(added).isFalse(); @@ -109,7 +109,7 @@ public class IssuePublisherTest { } @Test - public void ignore_null_rule_of_active_rule() { + void ignore_null_rule_of_active_rule() { initModuleIssues(); DefaultIssue issue = new DefaultIssue(project) @@ -122,7 +122,7 @@ public class IssuePublisherTest { } @Test - public void add_issue_to_cache() { + void add_issue_to_cache() { initModuleIssues(); final String ruleDescriptionContextKey = "spring"; @@ -156,7 +156,7 @@ public class IssuePublisherTest { } @Test - public void add_issue_flows_to_cache() { + void add_issue_flows_to_cache() { initModuleIssues(); DefaultMessageFormatting messageFormatting = new DefaultMessageFormatting().start(0).end(4).type(CODE); @@ -198,7 +198,7 @@ public class IssuePublisherTest { } @Test - public void add_external_issue_to_cache() { + void add_external_issue_to_cache() { initModuleIssues(); DefaultExternalIssue issue = new DefaultExternalIssue(project) @@ -215,7 +215,7 @@ public class IssuePublisherTest { } @Test - public void initAndAddExternalIssue_whenImpactAndCleanCodeAttributeProvided_shouldPopulateReportFields() { + void initAndAddExternalIssue_whenImpactAndCleanCodeAttributeProvided_shouldPopulateReportFields() { initModuleIssues(); DefaultExternalIssue issue = new DefaultExternalIssue(project) @@ -234,7 +234,7 @@ public class IssuePublisherTest { } @Test - public void dont_store_severity_if_no_severity_override_on_issue() { + void dont_store_severity_if_no_severity_override_on_issue() { initModuleIssues(); DefaultIssue issue = new DefaultIssue(project) @@ -250,7 +250,7 @@ public class IssuePublisherTest { } @Test - public void filter_issue() { + void filter_issue() { DefaultIssue issue = new DefaultIssue(project) .at(new DefaultIssueLocation().on(file).at(file.selectLine(3)).message("")) .forRule(JAVA_RULE_KEY); @@ -264,7 +264,7 @@ public class IssuePublisherTest { } @Test - public void should_ignore_lines_commented_with_nosonar() { + void should_ignore_lines_commented_with_nosonar() { initModuleIssues(); DefaultIssue issue = new DefaultIssue(project) @@ -279,11 +279,13 @@ public class IssuePublisherTest { verifyNoInteractions(reportPublisher); } - @Test - public void should_accept_issues_on_no_sonar_rules() { + @ParameterizedTest + @ValueSource(strings = {"NoSonarCheck", "S1291", "S1291Check"}) + void should_accept_issues_on_no_sonar_rules(String noSonarRule) { + RuleKey noSonarRuleKey = RuleKey.of("java", noSonarRule); // The "No Sonar" rule logs violations on the lines that are flagged with "NOSONAR" !! activeRulesBuilder.addRule(new NewActiveRule.Builder() - .setRuleKey(NOSONAR_RULE_KEY) + .setRuleKey(noSonarRuleKey) .setSeverity(Severity.INFO) .setQProfileKey("qp-1") .build()); @@ -293,7 +295,7 @@ public class IssuePublisherTest { DefaultIssue issue = new DefaultIssue(project) .at(new DefaultIssueLocation().on(file).at(file.selectLine(3)).message("")) - .forRule(NOSONAR_RULE_KEY); + .forRule(noSonarRuleKey); when(filters.accept(any(InputComponent.class), any(ScannerReport.Issue.class), anyString())).thenReturn(true); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoaderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoaderTest.java new file mode 100644 index 00000000000..59a73e86f16 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsLoaderTest.java @@ -0,0 +1,85 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import com.google.gson.Gson; +import java.io.Reader; +import java.io.StringReader; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.sonar.api.utils.MessageException; +import org.sonar.scanner.WsTestUtil; +import org.sonar.scanner.http.DefaultScannerWsClient; +import org.sonar.scanner.scan.branch.BranchConfiguration; +import wiremock.org.apache.hc.core5.http.HttpException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatException; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +class DefaultFeatureFlagsLoaderTest { + + private DefaultFeatureFlagsLoader loader; + private DefaultScannerWsClient wsClient; + + @BeforeEach + void setUp() { + wsClient = mock(DefaultScannerWsClient.class); + BranchConfiguration branchConfig = mock(BranchConfiguration.class); + when(branchConfig.isPullRequest()).thenReturn(false); + loader = new DefaultFeatureFlagsLoader(wsClient); + } + + @Test + void load_shouldRequestFeatureFlagsAndParseResponse() { + WsTestUtil.mockReader(wsClient, "/api/features/list", response()); + + Set<String> features = loader.load(); + assertThat(features).containsExactlyInAnyOrder("feature1", "feature2"); + + WsTestUtil.verifyCall(wsClient, "/api/features/list"); + + verifyNoMoreInteractions(wsClient); + } + + @Test + void load_whenHasSomeError_shouldThrowIllegalStateException() { + when(wsClient.call(any())).thenThrow(MessageException.of("You're not authorized")); + + assertThatException().isThrownBy(loader::load) + .isInstanceOf(IllegalStateException.class) + .withMessage("Unable to load feature flags"); + } + + private Reader response() { + return toReader(List.of("feature1", "feature2")); + } + + private static Reader toReader(List<String> featureFlags) { + String json = new Gson().toJson(featureFlags); + return new StringReader(json); + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepositoryTest.java new file mode 100644 index 00000000000..e043fcfdd2f --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/repository/featureflags/DefaultFeatureFlagsRepositoryTest.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.repository.featureflags; + +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DefaultFeatureFlagsRepositoryTest { + + private DefaultFeatureFlagsRepository underTest; + + @BeforeEach + void prepare() { + var loader = mock(FeatureFlagsLoader.class); + when(loader.load()).thenReturn(Set.of("feature1")); + underTest = new DefaultFeatureFlagsRepository(loader); + } + + @Test + void start_shouldReturnFlagStatus() { + underTest.start(); + + assertThat(underTest.isEnabled("feature1")).isTrue(); + assertThat(underTest.isEnabled("feature2")).isFalse(); + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java new file mode 100644 index 00000000000..6615ba4e4e4 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliCacheServiceTest.java @@ -0,0 +1,307 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.SystemUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.event.Level; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.api.utils.System2; +import org.sonar.scanner.WsTestUtil; +import org.sonar.scanner.bootstrap.SonarUserHome; +import org.sonar.scanner.http.DefaultScannerWsClient; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonarqube.ws.client.HttpException; +import org.sonarqube.ws.client.WsResponse; + +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sonar.scanner.sca.CliCacheService.CLI_WS_URL; + +@ExtendWith(MockitoExtension.class) +class CliCacheServiceTest { + @Mock + private SonarUserHome sonarUserHome; + @Mock + private DefaultScannerWsClient scannerWsClient; + @Mock + private System2 system2; + @Mock + private TelemetryCache telemetryCache; + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); + @TempDir + public Path cacheDir; + + private CliCacheService underTest; + + @BeforeEach + void setup() { + lenient().when(sonarUserHome.getPath()).thenReturn(cacheDir); + lenient().when(telemetryCache.put(any(), any())).thenReturn(telemetryCache); + + underTest = new CliCacheService(sonarUserHome, scannerWsClient, telemetryCache, system2); + } + + @Test + void cacheCli_shouldDownloadCli_whenCacheDoesNotExist() { + String checksum = "checksum"; + String id = "tidelift"; + WsTestUtil.mockReader(scannerWsClient, CLI_WS_URL, new StringReader(""" + [ + { + "id": "%s", + "filename": "tidelift_darwin", + "sha256": "%s", + "os": "mac", + "arch": "x64_86" + } + ]""".formatted(id, checksum))); + + WsTestUtil.mockStream(scannerWsClient, CLI_WS_URL + "/" + id, new ByteArrayInputStream("cli content".getBytes())); + + assertThat(cacheDir).isEmptyDirectory(); + + File generatedFile = underTest.cacheCli(); + + assertThat(generatedFile).exists().isExecutable(); + assertThat(cacheDir.resolve("cache").resolve(checksum)).exists().isNotEmptyDirectory(); + + verify(telemetryCache).put(eq("scanner.sca.download.cli.duration"), any()); + verify(telemetryCache).put("scanner.sca.download.cli.success", "true"); + verify(telemetryCache).put("scanner.sca.get.cli.cache.hit", "false"); + verify(telemetryCache).put("scanner.sca.get.cli.success", "true"); + } + + @Test + void cacheCli_shouldThrowException_whenMultipleMetadatas() { + WsTestUtil.mockReader(scannerWsClient, CLI_WS_URL, new StringReader(""" + [ + { + "id": "tidelift", + "filename": "tidelift_darwin", + "sha256": "1", + "os": "mac", + "arch": "x64_86" + }, + { + "id": "tidelift_other", + "filename": "tidelift", + "sha256": "2", + "os": "mac", + "arch": "x64_86" + } + ]""")); + + assertThatThrownBy(underTest::cacheCli).isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Multiple CLI matches found. Unable to correctly cache CLI."); + + verify(telemetryCache).put("scanner.sca.get.cli.success", "false"); + + } + + @Test + void cacheCli_shouldThrowException_whenNoMetadata() { + WsTestUtil.mockReader(scannerWsClient, CLI_WS_URL, new StringReader("[]")); + + assertThatThrownBy(underTest::cacheCli).isInstanceOf(IllegalStateException.class) + .hasMessageMatching("Could not find CLI for .+ .+"); + + verify(telemetryCache).put("scanner.sca.get.cli.success", "false"); + + } + + @Test + void cacheCli_shouldThrowException_whenServerError() { + HttpException http = new HttpException("url", 500, "some error message"); + IllegalStateException e = new IllegalStateException("http error", http); + WsTestUtil.mockException(scannerWsClient, e); + + assertThatThrownBy(underTest::cacheCli).isInstanceOf(IllegalStateException.class) + .hasMessageContaining("http error"); + + verify(telemetryCache).put("scanner.sca.get.cli.success", "false"); + } + + @Test + void cacheCli_shouldNotOverwrite_whenCachedFileExists() throws IOException { + String checksum = "checksum"; + WsTestUtil.mockReader(scannerWsClient, CLI_WS_URL, new StringReader(""" + [ + { + "id": "tidelift", + "filename": "tidelift_darwin", + "sha256": "%s", + "os": "mac", + "arch": "x64_86" + } + ]""".formatted(checksum))); + when(system2.isOsWindows()).thenReturn(false); + + String fileContent = "test content"; + File existingFile = underTest.cacheDir().resolve(checksum).resolve("tidelift").toFile(); + FileUtils.createParentDirectories(existingFile); + FileUtils.writeStringToFile(existingFile, fileContent, Charset.defaultCharset()); + + assertThat(existingFile).exists(); + if (!SystemUtils.IS_OS_WINDOWS) { + assertThat(existingFile.canExecute()).isFalse(); + } + assertThat(FileUtils.readFileToString(existingFile, Charset.defaultCharset())).isEqualTo(fileContent); + + underTest.cacheCli(); + + WsTestUtil.verifyCall(scannerWsClient, CLI_WS_URL); + assertThat(existingFile).exists(); + if (!SystemUtils.IS_OS_WINDOWS) { + assertThat(existingFile.canExecute()).isFalse(); + } + assertThat(FileUtils.readFileToString(existingFile, Charset.defaultCharset())).isEqualTo(fileContent); + + verify(telemetryCache).put("scanner.sca.get.cli.cache.hit", "true"); + verify(telemetryCache).put("scanner.sca.get.cli.success", "true"); + } + + @Test + void cacheCli_shouldAllowLocationOverride(@TempDir Path tempDir) throws IOException { + File alternateCliFile = tempDir.resolve("alternate_cli").toFile(); + FileUtils.writeStringToFile(alternateCliFile, "alternate cli content", Charset.defaultCharset()); + when(system2.envVariable("TIDELIFT_CLI_LOCATION")).thenReturn(alternateCliFile.getAbsolutePath()); + + var returnedFile = underTest.cacheCli(); + + assertThat(returnedFile.getAbsolutePath()).isEqualTo(alternateCliFile.getAbsolutePath()); + assertThat(logTester.logs(Level.INFO)).contains("Using alternate location for Tidelift CLI: " + alternateCliFile.getAbsolutePath()); + verify(scannerWsClient, never()).call(any()); + } + + @Test + void cacheCli_whenOverrideDoesntExist_shouldRaiseError() { + var location = "incorrect_location"; + when(system2.envVariable("TIDELIFT_CLI_LOCATION")).thenReturn(location); + + assertThatThrownBy(underTest::cacheCli).isInstanceOf(IllegalStateException.class) + .hasMessageMatching("Alternate location for Tidelift CLI has been set but no file was found at " + location); + + assertThat(logTester.logs(Level.INFO)).contains("Using alternate location for Tidelift CLI: " + location); + verify(scannerWsClient, never()).call(any()); + } + + @Test + void apiOsName_shouldReturnApiCompatibleName() { + when(system2.isOsWindows()).thenReturn(true); + when(system2.isOsMac()).thenReturn(false); + assertThat(underTest.apiOsName()).isEqualTo("windows"); + reset(system2); + + when(system2.isOsWindows()).thenReturn(false); + when(system2.isOsMac()).thenReturn(true); + assertThat(underTest.apiOsName()).isEqualTo("mac"); + + reset(system2); + when(system2.isOsWindows()).thenReturn(false); + when(system2.isOsMac()).thenReturn(false); + assertThat(underTest.apiOsName()).isEqualTo("linux"); + } + + @Test + void createTempDir_shouldReturnExistingDir() throws IOException { + Path dir = sonarUserHome.getPath().resolve("_tmp"); + Files.createDirectory(dir); + + assertThat(underTest.createTempDir()).isEqualTo(dir); + } + + @Test + void createTempDir_shouldHandleIOException() { + try (MockedStatic<Files> mockFilesClass = mockStatic(Files.class)) { + mockFilesClass.when(() -> Files.createDirectory(any(Path.class))).thenThrow(IOException.class); + + Path expectedDir = sonarUserHome.getPath().resolve("_tmp"); + assertThatThrownBy(underTest::createTempDir).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(format("Unable to create temp directory at %s", expectedDir)); + } + } + + @Test + void moveFile_shouldHandleIOException(@TempDir Path sourceFile, @TempDir Path targetFile) { + try (MockedStatic<Files> mockFilesClass = mockStatic(Files.class)) { + mockFilesClass.when(() -> Files.move(sourceFile, targetFile, StandardCopyOption.ATOMIC_MOVE)).thenThrow(IOException.class); + mockFilesClass.when(() -> Files.move(sourceFile, targetFile)).thenThrow(IOException.class); + + assertThatThrownBy(() -> CliCacheService.moveFile(sourceFile, targetFile)).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(format("Fail to move %s to %s", sourceFile, targetFile)); + + assertThat(logTester.logs(Level.WARN)).contains(format("Unable to rename %s to %s", sourceFile, targetFile)); + assertThat(logTester.logs(Level.WARN)).contains("A copy/delete will be tempted but with no guarantee of atomicity"); + } + } + + @Test + void mkdir_shouldHandleIOException(@TempDir Path dir) { + try (MockedStatic<Files> mockFilesClass = mockStatic(Files.class)) { + mockFilesClass.when(() -> Files.createDirectories(dir)).thenThrow(IOException.class); + + assertThatThrownBy(() -> CliCacheService.mkdir(dir)).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(format("Fail to create cache directory: %s", dir)); + } + } + + @Test + void downloadBinaryTo_shouldHandleIOException(@TempDir Path downloadLocation) { + WsResponse mockResponse = mock(WsResponse.class); + InputStream mockStream = mock(InputStream.class); + when(mockResponse.contentStream()).thenReturn(mockStream); + + try (MockedStatic<FileUtils> mockFileUtils = mockStatic(FileUtils.class)) { + mockFileUtils.when(() -> FileUtils.copyInputStreamToFile(mockStream, downloadLocation.toFile())).thenThrow(IOException.class); + + assertThatThrownBy(() -> CliCacheService.downloadBinaryTo(downloadLocation, mockResponse)).isInstanceOf(IllegalStateException.class) + .hasMessageContaining(format("Fail to download SCA CLI into %s", downloadLocation)); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java new file mode 100644 index 00000000000..33cf6c146e6 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/CliServiceTest.java @@ -0,0 +1,376 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.SystemUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; +import org.sonar.api.batch.bootstrap.ProjectDefinition; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.platform.Server; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.api.utils.System2; +import org.sonar.core.util.ProcessWrapperFactory; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.repository.TelemetryCache; +import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters; +import org.sonar.scanner.scm.ScmConfiguration; +import org.sonar.scm.git.GitScmProvider; +import org.sonar.scm.git.JGitUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.slf4j.event.Level.DEBUG; +import static org.slf4j.event.Level.INFO; + +class CliServiceTest { + private TelemetryCache telemetryCache; + private DefaultInputModule rootInputModule; + private final Server server = mock(Server.class); + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); + @TempDir + Path rootModuleDir; + private final ScmConfiguration scmConfiguration = mock(ScmConfiguration.class); + private final ScmProvider scmProvider = mock(GitScmProvider.class); + ProcessWrapperFactory processWrapperFactory = mock(ProcessWrapperFactory.class, CALLS_REAL_METHODS); + private MockedStatic<JGitUtils> jGitUtilsMock; + DefaultConfiguration configuration = mock(DefaultConfiguration.class); + ProjectExclusionFilters projectExclusionFilters = mock(ProjectExclusionFilters.class); + + private CliService underTest; + + @BeforeEach + void setup() throws IOException { + telemetryCache = new TelemetryCache(); + Path workDir = rootModuleDir.resolve(".scannerwork"); + Files.createDirectories(workDir); + rootInputModule = new DefaultInputModule( + ProjectDefinition.create().setBaseDir(rootModuleDir.toFile()).setWorkDir(workDir.toFile())); + when(scmConfiguration.provider()).thenReturn(scmProvider); + when(scmProvider.key()).thenReturn("git"); + when(scmConfiguration.isExclusionDisabled()).thenReturn(false); + jGitUtilsMock = org.mockito.Mockito.mockStatic(JGitUtils.class); + jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of("ignored.txt")); + when(server.getVersion()).thenReturn("1.0.0"); + logTester.setLevel(INFO); + when(projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN)).thenReturn(new String[0]); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[0]); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[0]); + + underTest = new CliService(processWrapperFactory, telemetryCache, System2.INSTANCE, server, scmConfiguration, projectExclusionFilters); + } + + @AfterEach + void teardown() { + if (jGitUtilsMock != null) { + jGitUtilsMock.close(); + } + } + + @Test + void generateManifestsArchive_shouldCallProcessCorrectly_andRegisterTelemetry() throws IOException, URISyntaxException { + assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); + + when(configuration.getProperties()).thenReturn(Map.of(CliService.SCA_EXCLUSIONS_KEY, "foo,bar,baz/**")); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"foo", "bar", "baz/**"}); + + File producedArchive = underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + assertThat(producedArchive).exists(); + + var expectedArguments = List.of( + "projects", + "save-lockfiles", + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), + "--directory", + rootInputModule.getBaseDir().toString(), + "--recursive", + "--exclude", + "foo,bar,baz/**,ignored.txt,.scannerwork/**"); + + assertThat(logTester.logs(INFO)) + .contains("Arguments Passed In: " + String.join(" ", expectedArguments)) + .contains("TIDELIFT_SKIP_UPDATE_CHECK=1") + .contains("TIDELIFT_ALLOW_MANIFEST_FAILURES=1") + .contains("Generated manifests archive file: " + producedArchive.getName()); + + assertThat(telemetryCache.getAll()).containsKey("scanner.sca.execution.cli.duration").isNotNull(); + assertThat(telemetryCache.getAll()).containsEntry("scanner.sca.execution.cli.success", "true"); + } + + @Test + void generateManifestsArchive_whenDebugLogLevelAndScaDebugNotEnabled_shouldWriteDebugLogsToDebugStream() throws IOException, URISyntaxException { + logTester.setLevel(DEBUG); + + assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + var expectedArguments = List.of( + "projects", + "save-lockfiles", + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), + "--directory", + rootInputModule.getBaseDir().toString(), + "--recursive", + "--exclude", + "ignored.txt,.scannerwork/**", + "--debug"); + + assertThat(logTester.logs(INFO)) + .contains("Arguments Passed In: " + String.join(" ", expectedArguments)); + } + + @Test + void generateManifestsArchive_whenScaDebugEnabled_shouldWriteDebugLogsToInfoStream() throws IOException, URISyntaxException { + assertThat(rootModuleDir.resolve("test_file").toFile().createNewFile()).isTrue(); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + var expectedArguments = List.of( + "projects", + "save-lockfiles", + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), + "--directory", + rootInputModule.getBaseDir().toString(), + "--recursive", + "--exclude", + "ignored.txt,.scannerwork/**"); + + assertThat(logTester.logs(INFO)) + .contains("Arguments Passed In: " + String.join(" ", expectedArguments)); + } + + @Test + void generateManifestsArchive_shouldSendSQEnvVars() throws IOException, URISyntaxException { + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + assertThat(logTester.logs(INFO)) + .contains("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE=1") + .contains("TIDELIFT_CLI_SQ_SERVER_VERSION=1.0.0"); + } + + @Test + void generateManifestsArchive_includesIgnoredPathsFromGitProvider() throws Exception { + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + var expectedArguments = List.of( + "projects", + "save-lockfiles", + "--xz", + "--xz-filename", + rootInputModule.getWorkDir().resolve("dependency-files.tar.xz").toString(), + "--directory", + rootInputModule.getBaseDir().toString(), + "--recursive", + "--exclude", + "ignored.txt,.scannerwork/**"); + + assertThat(logTester.logs(INFO)) + .contains("Arguments Passed In: " + String.join(" ", expectedArguments)) + .contains("TIDELIFT_SKIP_UPDATE_CHECK=1") + .contains("TIDELIFT_ALLOW_MANIFEST_FAILURES=1") + .contains("TIDELIFT_CLI_INSIDE_SCANNER_ENGINE=1") + .contains("TIDELIFT_CLI_SQ_SERVER_VERSION=1.0.0"); + + } + + @Test + void generateManifestsArchive_withNoScm_doesNotIncludeScmIgnoredPaths() throws Exception { + when(scmConfiguration.provider()).thenReturn(null); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); + } + + @Test + void generateManifestsArchive_withNonGit_doesNotIncludeScmIgnoredPaths() throws Exception { + when(scmProvider.key()).thenReturn("notgit"); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); + } + + @Test + void generateManifestsArchive_withScmExclusionDisabled_doesNotIncludeScmIgnoredPaths() throws Exception { + when(scmConfiguration.isExclusionDisabled()).thenReturn(true); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); + } + + @Test + void generateManifestsArchive_withNoScmIgnores_doesNotIncludeScmIgnoredPaths() throws Exception { + jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of()); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude .scannerwork/**"); + } + + @Test + void generateManifestsArchive_withExcludedManifests_appendsScmIgnoredPaths() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude **/test/**,ignored.txt,.scannerwork/**"); + } + + @Test + void generateManifestsArchive_withExcludedManifestsContainingBadCharacters_handlesTheBadCharacters() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] { + "**/test/**", "**/path with spaces/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + + String expectedExcludeFlag = """ + --exclude **/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**",ignored.txt + """.strip(); + if (SystemUtils.IS_OS_WINDOWS) { + expectedExcludeFlag = """ + --exclude "**/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**",ignored.txt + """.strip(); + } + assertThat(capturedArgs).contains(expectedExcludeFlag); + } + + @Test + void generateManifestsArchive_withExcludedManifestsContainingDupes_dedupes() throws Exception { + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test2/**", "**/test1/**"}); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test3/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude **/test1/**,**/test2/**,**/test3/**,ignored.txt,.scannerwork/**"); + } + + @Test + void generateManifestsArchive_withExcludedManifestsAndSonarExcludesContainingDupes_mergesAndDedupes() throws Exception { + when(projectExclusionFilters.getExclusionsConfig(InputFile.Type.MAIN)).thenReturn(new String[] {"**/test1/**", "**/test4/**"}); + when(configuration.getStringArray(CliService.SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test2/**", "**/test1/**"}); + when(configuration.getStringArray(CliService.LEGACY_SCA_EXCLUSIONS_KEY)).thenReturn(new String[] {"**/test1/**", "**/test3/**"}); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude **/test1/**,**/test4/**,**/test2/**,**/test3/**,ignored.txt,.scannerwork/**"); + } + + @Test + void generateManifestsArchive_withScmIgnoresContainingBadCharacters_handlesTheBadCharacters() throws Exception { + jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))) + .thenReturn(List.of("**/test/**", "**/path with spaces/**", "**/path'with'quotes/**", "**/path\"with\"double\"quotes/**")); + + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + + String expectedExcludeFlag = """ + --exclude **/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**" + """.strip(); + if (SystemUtils.IS_OS_WINDOWS) { + expectedExcludeFlag = """ + --exclude "**/test/**,**/path with spaces/**,**/path'with'quotes/**,"**/path""with""double""quotes/**" + """.strip(); + } + assertThat(capturedArgs).contains(expectedExcludeFlag); + } + + @Test + void generateManifestsArchive_withIgnoredDirectories_GlobifiesDirectories() throws Exception { + String ignoredDirectory = "directory1"; + Files.createDirectories(rootModuleDir.resolve(ignoredDirectory)); + String ignoredFile = "directory2/file.txt"; + Path ignoredFilePath = rootModuleDir.resolve(ignoredFile); + Files.createDirectories(ignoredFilePath.getParent()); + Files.createFile(ignoredFilePath); + + jGitUtilsMock.when(() -> JGitUtils.getAllIgnoredPaths(any(Path.class))).thenReturn(List.of(ignoredDirectory, ignoredFile)); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + assertThat(capturedArgs).contains("--exclude directory1/**,directory2/file.txt"); + } + + @Test + void generateManifestsArchive_withExternalWorkDir_DoesNotExcludeWorkingDir() throws URISyntaxException, IOException { + Path externalWorkDir = Files.createTempDirectory("externalWorkDir"); + try { + rootInputModule = new DefaultInputModule(ProjectDefinition.create().setBaseDir(rootModuleDir.toFile()).setWorkDir(externalWorkDir.toFile())); + underTest.generateManifestsArchive(rootInputModule, scriptDir(), configuration); + String capturedArgs = logTester.logs().stream().filter(log -> log.contains("Arguments Passed In:")).findFirst().get(); + + // externalWorkDir is not present in the exclude flag + assertThat(capturedArgs).contains("--exclude ignored.txt"); + } finally { + externalWorkDir.toFile().delete(); + } + } + + private URL scriptUrl() { + // There is a custom test Bash script available in src/test/resources/org/sonar/scanner/sca that + // will serve as our "CLI". This script will output some messages about what arguments were passed + // to it and will try to generate an archive file in the location the process specifies. This allows us + // to simulate a real CLI call without needing an OS specific CLI executable to run on a real project. + URL scriptUrl = CliServiceTest.class.getResource(SystemUtils.IS_OS_WINDOWS ? "echo_args.bat" : "echo_args.sh"); + assertThat(scriptUrl).isNotNull(); + return scriptUrl; + } + + private File scriptDir() throws URISyntaxException { + return new File(scriptUrl().toURI()); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java new file mode 100644 index 00000000000..ebe6007a1c1 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaExecutorTest.java @@ -0,0 +1,180 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.assertj.core.util.Files; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.event.Level; +import org.sonar.api.batch.bootstrap.ProjectDefinition; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; +import org.sonar.scanner.config.DefaultConfiguration; +import org.sonar.scanner.protocol.output.ScannerReportWriter; +import org.sonar.scanner.report.ReportPublisher; +import org.sonar.scanner.repository.featureflags.FeatureFlagsRepository; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class ScaExecutorTest { + private final CliService cliService = mock(CliService.class); + private final CliCacheService cliCacheService = mock(CliCacheService.class); + private final ReportPublisher reportPublisher = mock(ReportPublisher.class); + private final FeatureFlagsRepository featureFlagsRepository = mock(FeatureFlagsRepository.class); + private final DefaultConfiguration configuration = mock(DefaultConfiguration.class); + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); + private final ScaExecutor underTest = new ScaExecutor(cliCacheService, cliService, reportPublisher, featureFlagsRepository, configuration); + @TempDir + File rootModuleDir; + private DefaultInputModule root; + + @BeforeEach + void before() { + when(featureFlagsRepository.isEnabled("sca")).thenReturn(true); + root = new DefaultInputModule( + ProjectDefinition.create().setBaseDir(rootModuleDir).setWorkDir(rootModuleDir.toPath().getRoot().toFile())); + } + + @Test + void execute_shouldCallCliAndPublisher() throws IOException { + File mockCliFile = Files.newTemporaryFile(); + File mockManifestZip = Files.newTemporaryFile(); + ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(reportPublisher.getWriter()).thenReturn(mockReportWriter); + + logTester.setLevel(Level.DEBUG); + + underTest.execute(root); + + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); + verify(mockReportWriter).writeScaFile(mockManifestZip); + assertThat(logTester.logs(Level.DEBUG)).contains("Zip ready for report: " + mockManifestZip); + assertThat(logTester.logs(Level.DEBUG)).contains("Manifest zip written to report"); + } + + @Test + void execute_whenIOException_shouldHandleException() throws IOException { + File mockCliFile = Files.newTemporaryFile(); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + doThrow(IOException.class).when(cliService).generateManifestsArchive(root, mockCliFile, configuration); + + logTester.setLevel(Level.INFO); + + underTest.execute(root); + + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); + assertThat(logTester.logs(Level.ERROR)).contains("Error gathering manifests"); + } + + @Test + void execute_whenIllegalStateException_shouldHandleException() throws IOException { + File mockCliFile = Files.newTemporaryFile(); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + doThrow(IllegalStateException.class).when(cliService).generateManifestsArchive(root, mockCliFile, configuration); + + logTester.setLevel(Level.INFO); + + underTest.execute(root); + + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); + assertThat(logTester.logs(Level.ERROR)).contains("Error gathering manifests"); + } + + @Test + void execute_whenNoCliFound_shouldSkipAnalysis() throws IOException { + File mockCliFile = new File(""); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + + underTest.execute(root); + + verify(cliService, never()).generateManifestsArchive(root, mockCliFile, configuration); + } + + @Test + void execute_whenGlobalFeatureDisabled_skips() { + when(featureFlagsRepository.isEnabled("sca")).thenReturn(false); + logTester.setLevel(Level.DEBUG); + + underTest.execute(root); + + assertThat(logTester.logs()).contains("Dependency analysis skipped"); + verifyNoInteractions(cliService, cliCacheService); + } + + @Test + void execute_whenProjectPropertyDisabled_skips() { + when(configuration.getBoolean("sonar.sca.enabled")).thenReturn(Optional.of(false)); + logTester.setLevel(Level.DEBUG); + + underTest.execute(root); + + assertThat(logTester.logs()).contains("Dependency analysis disabled for this project"); + verifyNoInteractions(cliService, cliCacheService); + } + + @Test + void execute_whenProjectPropertyExplicitlyEnabled_CallsCli() throws IOException { + when(configuration.getBoolean("sonar.sca.enabled")).thenReturn(Optional.of(true)); + File mockCliFile = Files.newTemporaryFile(); + File mockManifestZip = Files.newTemporaryFile(); + ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(reportPublisher.getWriter()).thenReturn(mockReportWriter); + logTester.setLevel(Level.DEBUG); + + underTest.execute(root); + + verify(cliService).generateManifestsArchive(root, mockCliFile, configuration); + verify(mockReportWriter).writeScaFile(mockManifestZip); + assertThat(logTester.logs(Level.DEBUG)).contains("Zip ready for report: " + mockManifestZip); + assertThat(logTester.logs(Level.DEBUG)).contains("Manifest zip written to report"); + } + + @Test + void execute_printsRuntime() throws IOException { + File mockCliFile = Files.newTemporaryFile(); + File mockManifestZip = Files.newTemporaryFile(); + ScannerReportWriter mockReportWriter = mock(ScannerReportWriter.class); + when(cliCacheService.cacheCli()).thenReturn(mockCliFile); + when(cliService.generateManifestsArchive(root, mockCliFile, configuration)).thenReturn(mockManifestZip); + when(reportPublisher.getWriter()).thenReturn(mockReportWriter); + + logTester.setLevel(Level.INFO); + + underTest.execute(root); + + assertThat(logTester.logs(Level.INFO)).anyMatch(l -> l.matches("Load SCA project dependencies \\(done\\) \\| time=\\d+ms")); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java new file mode 100644 index 00000000000..70e7a6b6e53 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sca/ScaPropertiesTest.java @@ -0,0 +1,100 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.sca; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.sonar.scanner.config.DefaultConfiguration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class ScaPropertiesTest { + private final DefaultConfiguration configuration = mock(DefaultConfiguration.class); + + @Test + void buildFromScannerProperties_withNoProperties_returnsEmptyMap() { + when(configuration.get(anyString())).thenReturn(Optional.empty()); + + var result = ScaProperties.buildFromScannerProperties(configuration); + + assertThat(result).isEqualTo(Map.of()); + } + + @Test + void buildFromScannerProperties_withUnmappedProperties_ignoresProperties() { + var inputProperties = new HashMap<String, String>(); + inputProperties.put("sonar.sca.pythonBinary", "/usr/bin/python3"); + inputProperties.put("sonar.sca.unknownProperty", "value"); + inputProperties.put("sonar.somethingElse", "dont-include-non-sca"); + inputProperties.put("sonar.sca.recursiveManifestSearch", "ignore-me"); + when(configuration.getProperties()).thenReturn(inputProperties); + when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); + + var result = ScaProperties.buildFromScannerProperties(configuration); + + assertThat(result).containsExactly( + Map.entry("TIDELIFT_PYTHON_BINARY", "/usr/bin/python3"), + Map.entry("TIDELIFT_UNKNOWN_PROPERTY", "value")); + } + + @Test + void buildFromScannerProperties_withLotsOfProperties_mapsAllProperties() { + var inputProperties = new HashMap<String, String>(); + inputProperties.put("sonar.sca.goNoResolve", "true"); + inputProperties.put("sonar.sca.gradleConfigurationPattern", "pattern"); + inputProperties.put("sonar.sca.gradleNoResolve", "false"); + inputProperties.put("sonar.sca.mavenForceDepPlugin", "plugin"); + inputProperties.put("sonar.sca.mavenNoResolve", "true"); + inputProperties.put("sonar.sca.mavenIgnoreWrapper", "false"); + inputProperties.put("sonar.sca.mavenOptions", "-DskipTests"); + inputProperties.put("sonar.sca.npmEnableScripts", "true"); + inputProperties.put("sonar.sca.npmNoResolve", "true"); + inputProperties.put("sonar.sca.nugetNoResolve", "false"); + inputProperties.put("sonar.sca.pythonBinary", "/usr/bin/python3"); + inputProperties.put("sonar.sca.pythonNoResolve", "true"); + inputProperties.put("sonar.sca.pythonResolveLocal", "false"); + when(configuration.getProperties()).thenReturn(inputProperties); + when(configuration.get(anyString())).thenAnswer(i -> Optional.ofNullable(inputProperties.get(i.getArgument(0, String.class)))); + + var expectedProperties = new HashMap<String, String>(); + expectedProperties.put("TIDELIFT_GO_NO_RESOLVE", "true"); + expectedProperties.put("TIDELIFT_GRADLE_CONFIGURATION_PATTERN", "pattern"); + expectedProperties.put("TIDELIFT_GRADLE_NO_RESOLVE", "false"); + expectedProperties.put("TIDELIFT_MAVEN_FORCE_DEP_PLUGIN", "plugin"); + expectedProperties.put("TIDELIFT_MAVEN_NO_RESOLVE", "true"); + expectedProperties.put("TIDELIFT_MAVEN_IGNORE_WRAPPER", "false"); + expectedProperties.put("TIDELIFT_MAVEN_OPTIONS", "-DskipTests"); + expectedProperties.put("TIDELIFT_NPM_ENABLE_SCRIPTS", "true"); + expectedProperties.put("TIDELIFT_NPM_NO_RESOLVE", "true"); + expectedProperties.put("TIDELIFT_NUGET_NO_RESOLVE", "false"); + expectedProperties.put("TIDELIFT_PYTHON_BINARY", "/usr/bin/python3"); + expectedProperties.put("TIDELIFT_PYTHON_NO_RESOLVE", "true"); + expectedProperties.put("TIDELIFT_PYTHON_RESOLVE_LOCAL", "false"); + + var result = ScaProperties.buildFromScannerProperties(configuration); + + assertThat(result).containsExactlyInAnyOrderEntriesOf(expectedProperties); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java index 21dcf58b114..3e3066e76e2 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ProjectConfigurationProviderTest.java @@ -52,10 +52,9 @@ public class ProjectConfigurationProviderTest { private static final Map<String, String> PROJECT_SERVER_PROPERTIES = Map.of(NON_GLOBAL_KEY_PROPERTIES_1, NON_GLOBAL_VALUE_PROPERTIES_1); private static final Map<String, String> DEFAULT_PROJECT_PROPERTIES = Map.of(DEFAULT_KEY_PROPERTIES_1, DEFAULT_VALUE_1); - private static final Map<String, String> ALL_PROPERTIES_MAP = - Stream.of(GLOBAL_SERVER_PROPERTIES, PROJECT_SERVER_PROPERTIES, DEFAULT_PROJECT_PROPERTIES) - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private static final Map<String, String> ALL_PROPERTIES_MAP = Stream.of(GLOBAL_SERVER_PROPERTIES, PROJECT_SERVER_PROPERTIES, DEFAULT_PROJECT_PROPERTIES) + .flatMap(map -> map.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); private static final Map<String, String> PROPERTIES_AFTER_FILTERING = Map.of("aKey", "aValue"); @@ -66,8 +65,6 @@ public class ProjectConfigurationProviderTest { @Mock private GlobalConfiguration globalConfiguration; @Mock - private MutableProjectSettings mutableProjectSettings; - @Mock private DefaultInputProject defaultInputProject; @Mock private SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter; @@ -75,7 +72,6 @@ public class ProjectConfigurationProviderTest { @InjectMocks private ProjectConfigurationProvider provider; - @Before public void init() { when(globalConfiguration.getDefinitions()).thenReturn(new PropertyDefinitions(System2.INSTANCE)); @@ -89,11 +85,11 @@ public class ProjectConfigurationProviderTest { when(sonarGlobalPropertiesFilter.enforceOnlyServerSideSonarGlobalPropertiesAreUsed(ALL_PROPERTIES_MAP, GLOBAL_SERVER_PROPERTIES)) .thenReturn(PROPERTIES_AFTER_FILTERING); - ProjectConfiguration provide = provider.provide(defaultInputProject, globalConfiguration, globalServerSettings, projectServerSettings, mutableProjectSettings); + ProjectConfiguration provide = provider.provide(defaultInputProject, globalConfiguration, globalServerSettings, projectServerSettings); verify(sonarGlobalPropertiesFilter).enforceOnlyServerSideSonarGlobalPropertiesAreUsed(ALL_PROPERTIES_MAP, GLOBAL_SERVER_PROPERTIES); assertThat(provide.getOriginalProperties()).containsExactlyEntriesOf(PROPERTIES_AFTER_FILTERING); } -}
\ No newline at end of file +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java index 01cb7d1bccf..277fc0dc68a 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java @@ -27,20 +27,26 @@ import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Optional; import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; import org.sonar.scanner.fs.InputModuleHierarchy; +import org.sonar.scanner.scan.ModuleConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DirectoryFileVisitorTest { @@ -48,33 +54,57 @@ public class DirectoryFileVisitorTest { public static TemporaryFolder temp = new TemporaryFolder(); private final DefaultInputModule module = mock(); + private final ModuleConfiguration moduleConfiguration = mock(); private final ModuleExclusionFilters moduleExclusionFilters = mock(); private final InputModuleHierarchy inputModuleHierarchy = mock(); private final InputFile.Type type = mock(); + private final SonarUserHome sonarUserHome = mock(); + private HiddenFilesProjectData hiddenFilesProjectData; + + @Before + public void before() throws IOException { + Path sonarUserHomePath = temp.newFolder().toPath(); + when(sonarUserHome.getPath()).thenReturn(sonarUserHomePath); + File workDir = temp.newFolder(); + when(module.getWorkDir()).thenReturn(workDir.toPath()); + hiddenFilesProjectData = spy(new HiddenFilesProjectData(sonarUserHome)); + } @Test - public void visit_hidden_file() throws IOException { + public void should_not_visit_hidden_file() throws IOException { + when(moduleConfiguration.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); - File hidden = temp.newFile(".hidden"); - if (SystemUtils.IS_OS_WINDOWS) { - Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); - } - + File hidden = temp.newFile(".hiddenNotVisited"); + setAsHiddenOnWindows(hidden); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); verify(action, never()).execute(any(Path.class)); } @Test + public void should_visit_hidden_file() throws IOException { + when(moduleConfiguration.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(false)); + DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); + + File hidden = temp.newFile(".hiddenVisited"); + setAsHiddenOnWindows(hidden); + + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); + underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class)); + + verify(action).execute(any(Path.class)); + } + + @Test public void test_visit_file_failed_generic_io_exception() throws IOException { DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class); File file = temp.newFile("failed"); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException())); } @@ -84,10 +114,15 @@ public class DirectoryFileVisitorTest { File file = temp.newFile("symlink"); - DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type); + DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData); FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath())); assertThat(result).isEqualTo(FileVisitResult.CONTINUE); } + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java new file mode 100644 index 00000000000..d5a6e4ff843 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java @@ -0,0 +1,160 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class HiddenFilesProjectDataTest { + + @ClassRule + public static TemporaryFolder temp = new TemporaryFolder(); + + private static final SonarUserHome sonarUserHome = mock(SonarUserHome.class); + private final DefaultInputModule inputModule = mock(DefaultInputModule.class); + private final DefaultInputModule secondInputModule = mock(DefaultInputModule.class); + private HiddenFilesProjectData underTest; + + @BeforeClass + public static void setUp() throws IOException { + File userHomeFolder = temp.newFolder(".userhome"); + setAsHiddenOnWindows(userHomeFolder); + when(sonarUserHome.getPath()).thenReturn(userHomeFolder.toPath()); + } + + @Before + public void before() { + underTest = spy(new HiddenFilesProjectData(sonarUserHome)); + } + + @Test + public void shouldContainNoMarkedHiddenFileOnConstruction() { + assertThat(underTest.hiddenFilesByModule).isEmpty(); + } + + @Test + public void shouldMarkWithCorrectAssociatedInputModule() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isFalse(); + } + + @Test + public void shouldMarkWithCorrectAssociatedInputModuleForTwoDifferentModules() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, secondInputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(2); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isTrue(); + } + + @Test + public void shouldNotShowAsHiddenFileWhenInputModuleIsNotExistingInData() { + Path myFile = Path.of("myFile"); + Path notMarkedFile = Path.of("notMarkedFile"); + underTest.markAsHiddenFile(myFile, inputModule); + + assertThat(underTest.hiddenFilesByModule).isNotEmpty(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(notMarkedFile, secondInputModule)).isFalse(); + } + + @Test + public void shouldClearMap() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, secondInputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(2); + + underTest.clearHiddenFilesData(); + assertThat(underTest.hiddenFilesByModule).isEmpty(); + } + + @Test + public void shouldRemoveVisibilityAfterQuerying() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.hiddenFilesByModule.get(inputModule)).isEmpty(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + } + + @Test + public void shouldOnlyRemoveModuleIfAllFilesAreRemoved() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).isNotEmpty(); + } + + @Test + public void shouldNotFailOnUserPathResolving() throws IOException { + Path expectedPath = sonarUserHome.getPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + assertThat(underTest.getCachedSonarUserHomePath()).isEqualTo(expectedPath); + } + + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java new file mode 100644 index 00000000000..8c111c7ea15 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesVisitorHelperTest.java @@ -0,0 +1,315 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.scan.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.scanner.bootstrap.SonarUserHome; +import org.sonar.scanner.scan.ModuleConfiguration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class HiddenFilesVisitorHelperTest { + + @ClassRule + public static TemporaryFolder temp = new TemporaryFolder(); + + private static final SonarUserHome sonarUserHome = mock(SonarUserHome.class); + private static final DefaultInputModule inputModule = mock(DefaultInputModule.class); + + private final ModuleConfiguration moduleConfig = mock(ModuleConfiguration.class); + private final HiddenFilesProjectData hiddenFilesProjectData = spy(new HiddenFilesProjectData(sonarUserHome)); + private HiddenFilesVisitorHelper underTest; + + @BeforeClass + public static void setUp() throws IOException { + File userHomeFolder = temp.newFolder(".userhome"); + setAsHiddenOnWindows(userHomeFolder); + when(sonarUserHome.getPath()).thenReturn(userHomeFolder.toPath()); + + File workDir = temp.newFolder(".sonar"); + setAsHiddenOnWindows(workDir); + when(inputModule.getWorkDir()).thenReturn(workDir.toPath()); + } + + @Before + public void before() { + hiddenFilesProjectData.clearHiddenFilesData(); + underTest = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + } + + @Test + public void verifyDefaultOnConstruction() { + assertThat(underTest.excludeHiddenFiles).isFalse(); + assertThat(underTest.rootHiddenDir).isNull(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToFalseFromConfigurationWhenNotConfigured() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.empty()); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isFalse(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToFalseFromConfigurationWhenDisabled() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(false)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isFalse(); + } + + @Test + public void excludeHiddenFilesShouldBeSetToTrueFromConfigurationWhenEnabled() { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + assertThat(configuredVisitorHelper.excludeHiddenFiles).isTrue(); + } + + @Test + public void shouldVisitHiddenDirectory() throws IOException { + File hiddenDir = temp.newFolder(".hiddenVisited"); + setAsHiddenOnWindows(hiddenDir); + + boolean visitDir = underTest.shouldVisitDir(hiddenDir.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenDir.toPath()); + verify(underTest).enterHiddenDirectory(hiddenDir.toPath()); + } + + @Test + public void shouldNotVisitHiddenDirectoryWhenHiddenFilesVisitIsExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File hidden = temp.newFolder(".hiddenNotVisited"); + setAsHiddenOnWindows(hidden); + + boolean visitDir = configuredVisitorHelper.shouldVisitDir(hidden.toPath()); + + assertThat(visitDir).isFalse(); + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + verify(configuredVisitorHelper, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldVisitNonHiddenDirectoryWhenHiddenFilesVisitIsExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File nonHiddenFolder = temp.newFolder(); + + boolean visitDir = configuredVisitorHelper.shouldVisitDir(nonHiddenFolder.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + verify(configuredVisitorHelper, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldVisitNonHiddenDirectory() throws IOException { + File nonHiddenFolder = temp.newFolder(); + + boolean visitDir = underTest.shouldVisitDir(nonHiddenFolder.toPath()); + + assertThat(visitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + assertThat(underTest.excludeHiddenFiles).isFalse(); + } + + @Test + public void shouldNotVisitModuleWorkDir() throws IOException { + Path workingDirectory = inputModule.getWorkDir().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + boolean visitDir = underTest.shouldVisitDir(workingDirectory); + + assertThat(visitDir).isFalse(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + } + + @Test + public void shouldNotVisitSonarUserHome() throws IOException { + Path userHome = sonarUserHome.getPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); + boolean visitDir = underTest.shouldVisitDir(userHome); + + assertThat(visitDir).isFalse(); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + verify(underTest, never()).enterHiddenDirectory(any()); + } + + @Test + public void hiddenFileShouldBeVisited() throws IOException { + File hiddenFile = temp.newFile(".hiddenFileShouldBeVisited"); + setAsHiddenOnWindows(hiddenFile); + + assertThat(underTest.insideHiddenDirectory()).isFalse(); + boolean visitFile = underTest.shouldVisitFile(hiddenFile.toPath()); + + assertThat(visitFile).isTrue(); + verify(hiddenFilesProjectData).markAsHiddenFile(hiddenFile.toPath(), inputModule); + } + + @Test + public void nonHiddenFileShouldBeVisitedInHiddenFolder() throws IOException { + File hidden = temp.newFolder(".hiddenFolder"); + setAsHiddenOnWindows(hidden); + + File nonHiddenFile = temp.newFile(); + + underTest.shouldVisitDir(hidden.toPath()); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + + boolean shouldVisitFile = underTest.shouldVisitFile(nonHiddenFile.toPath()); + + assertThat(shouldVisitFile).isTrue(); + verify(hiddenFilesProjectData).markAsHiddenFile(nonHiddenFile.toPath(), inputModule); + } + + @Test + public void shouldNotSetAsRootHiddenDirectoryWhenAlreadyEnteredHiddenDirectory() throws IOException { + File hidden = temp.newFolder(".outerHiddenFolder"); + File nestedHiddenFolder = temp.newFolder(".outerHiddenFolder", ".nestedHiddenFolder"); + setAsHiddenOnWindows(hidden); + setAsHiddenOnWindows(nestedHiddenFolder); + + underTest.shouldVisitDir(hidden.toPath()); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + + boolean shouldVisitNestedDir = underTest.shouldVisitDir(nestedHiddenFolder.toPath()); + + assertThat(shouldVisitNestedDir).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hidden.toPath()); + verify(underTest).enterHiddenDirectory(nestedHiddenFolder.toPath()); + } + + @Test + public void hiddenFileShouldNotBeVisitedWhenHiddenFileVisitExcluded() throws IOException { + when(moduleConfig.getBoolean("sonar.scanner.excludeHiddenFiles")).thenReturn(Optional.of(true)); + HiddenFilesVisitorHelper configuredVisitorHelper = spy(new HiddenFilesVisitorHelper(hiddenFilesProjectData, inputModule, moduleConfig)); + + File hiddenFile = temp.newFile(".hiddenFileNotVisited"); + setAsHiddenOnWindows(hiddenFile); + + assertThat(configuredVisitorHelper.insideHiddenDirectory()).isFalse(); + + configuredVisitorHelper.shouldVisitFile(hiddenFile.toPath()); + boolean shouldVisitFile = configuredVisitorHelper.shouldVisitFile(hiddenFile.toPath()); + + assertThat(shouldVisitFile).isFalse(); + verify(hiddenFilesProjectData, never()).markAsHiddenFile(hiddenFile.toPath(), inputModule); + } + + @Test + public void shouldCorrectlyExitHiddenFolderOnlyOnHiddenFolderThatEntered() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenRootFolder"); + setAsHiddenOnWindows(hiddenFolder); + + boolean shouldVisitDir = underTest.shouldVisitDir(hiddenFolder.toPath()); + + assertThat(shouldVisitDir).isTrue(); + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenFolder.toPath()); + verify(underTest).enterHiddenDirectory(hiddenFolder.toPath()); + + File folder1 = temp.newFolder(".hiddenRootFolder", "myFolderExit"); + File folder2 = temp.newFolder("myFolderExit"); + File folder3 = temp.newFolder(".myFolderExit"); + setAsHiddenOnWindows(folder3); + + underTest.exitDirectory(folder1.toPath()); + underTest.exitDirectory(folder2.toPath()); + underTest.exitDirectory(folder3.toPath()); + + assertThat(underTest.insideHiddenDirectory()).isTrue(); + assertThat(underTest.rootHiddenDir).isEqualTo(hiddenFolder.toPath()); + verify(underTest, never()).resetRootHiddenDir(); + + underTest.exitDirectory(hiddenFolder.toPath()); + assertThat(underTest.insideHiddenDirectory()).isFalse(); + assertThat(underTest.rootHiddenDir).isNull(); + verify(underTest).resetRootHiddenDir(); + } + + @Test + public void shouldNotInitiateResetRootDirWhenNotInHiddenDirectory() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenFolderNonRoot"); + setAsHiddenOnWindows(hiddenFolder); + + underTest.exitDirectory(hiddenFolder.toPath()); + + verify(underTest, never()).resetRootHiddenDir(); + } + + @Test + public void filesShouldBeCorrectlyMarkedAsHidden() throws IOException { + File hiddenFolder = temp.newFolder(".hiddenFolderRoot"); + setAsHiddenOnWindows(hiddenFolder); + + File file1 = temp.newFile(); + File file2 = temp.newFile(); + File file3 = temp.newFile(".markedHiddenFile"); + setAsHiddenOnWindows(file3); + File file4 = temp.newFile(); + File file5 = temp.newFile(".markedHiddenFile2"); + setAsHiddenOnWindows(file5); + + underTest.shouldVisitFile(file1.toPath()); + underTest.shouldVisitDir(hiddenFolder.toPath()); + underTest.shouldVisitFile(file2.toPath()); + underTest.shouldVisitFile(file3.toPath()); + underTest.exitDirectory(hiddenFolder.toPath()); + underTest.shouldVisitFile(file4.toPath()); + underTest.shouldVisitFile(file5.toPath()); + + verify(hiddenFilesProjectData, never()).markAsHiddenFile(file1.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file2.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file3.toPath(), inputModule); + verify(hiddenFilesProjectData, never()).markAsHiddenFile(file4.toPath(), inputModule); + verify(hiddenFilesProjectData).markAsHiddenFile(file5.toPath(), inputModule); + } + + private static void setAsHiddenOnWindows(File file) throws IOException { + if (SystemUtils.IS_OS_WINDOWS) { + Files.setAttribute(file.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java index a0031f77633..07f7afec036 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStoreTest.java @@ -19,80 +19,151 @@ */ package org.sonar.scanner.scan.filesystem; -import java.io.IOException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.sonar.api.SonarRuntime; import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.fs.internal.SensorStrategy; -import org.sonar.api.batch.fs.internal.DefaultInputProject; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; import org.sonar.scanner.scan.branch.BranchConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class ModuleInputComponentStoreTest { - @Rule - public TemporaryFolder temp = new TemporaryFolder(); +@ExtendWith(MockitoExtension.class) +class ModuleInputComponentStoreTest { + + @TempDir + private File projectBaseDir; + + @Mock + BranchConfiguration branchConfiguration; + + @Mock + SonarRuntime sonarRuntime; + + @Mock + InputComponentStore mockedInputComponentStore; private InputComponentStore componentStore; + private SensorContextTester sensorContextTester; private final String projectKey = "dummy key"; - @Before - public void setUp() throws IOException { - DefaultInputProject root = TestInputFileBuilder.newDefaultInputProject(projectKey, temp.newFolder()); - componentStore = new InputComponentStore(mock(BranchConfiguration.class), mock(SonarRuntime.class)); + @BeforeEach + void setUp() { + TestInputFileBuilder.newDefaultInputProject(projectKey, projectBaseDir); + File moduleBaseDir = new File(projectBaseDir, "module"); + moduleBaseDir.mkdir(); + sensorContextTester = SensorContextTester.create(moduleBaseDir); + componentStore = spy(new InputComponentStore(branchConfiguration, sonarRuntime)); } @Test - public void should_cache_files_by_filename() { + void should_cache_module_files_by_filename() { ModuleInputComponentStore store = newModuleInputComponentStore(); String filename = "some name"; - InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile1); - InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/" + filename).build(); store.doAdd(inputFile2); - InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "some/path/Dummy.java").build(); + InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "module/some/path/Dummy.java").build(); store.doAdd(dummyInputFile); assertThat(store.getFilesByName(filename)).containsExactlyInAnyOrder(inputFile1, inputFile2); } @Test - public void should_cache_files_by_extension() { + void should_cache_filtered_module_files_by_filename() { + ModuleInputComponentStore store = newModuleInputComponentStore(); + + String filename = "some name"; + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/" + filename).build(); + store.doAdd(inputFile2); + + when(componentStore.getFilesByName(filename)).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByName(filename)).containsOnly(inputFile2); + } + + @Test + void should_cache_module_files_by_filename_global_strategy() { + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), componentStore, new SensorStrategy()); + + String filename = "some name"; + // None in the module + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/" + filename).build(); + + when(componentStore.getFilesByName(filename)).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByName(filename)).containsExactlyInAnyOrder(inputFile1, inputFile2); + } + + @Test + void should_cache_module_files_by_extension() { ModuleInputComponentStore store = newModuleInputComponentStore(); - InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/Program.java").build(); + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "module/some/path/Program.java").build(); store.doAdd(inputFile1); - InputFile inputFile2 = new TestInputFileBuilder(projectKey, "other/path/Utils.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/other/path/Utils.java").build(); store.doAdd(inputFile2); - InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "some/path/NotJava.cpp").build(); + InputFile dummyInputFile = new TestInputFileBuilder(projectKey, "module/some/path/NotJava.cpp").build(); store.doAdd(dummyInputFile); assertThat(store.getFilesByExtension("java")).containsExactlyInAnyOrder(inputFile1, inputFile2); } @Test - public void should_not_cache_duplicates() { + void should_cache_filtered_module_files_by_extension() { + ModuleInputComponentStore store = newModuleInputComponentStore(); + + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/NotInModule.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "module/some/path/Other.java").build(); + store.doAdd(inputFile2); + + when(componentStore.getFilesByExtension("java")).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByExtension("java")).containsOnly(inputFile2); + } + + @Test + void should_cache_module_files_by_extension_global_strategy() { + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), componentStore, new SensorStrategy()); + + // None in the module + InputFile inputFile1 = new TestInputFileBuilder(projectKey, "some/path/NotInModule.java").build(); + InputFile inputFile2 = new TestInputFileBuilder(projectKey, "some/path/Other.java").build(); + + when(componentStore.getFilesByExtension("java")).thenReturn(List.of(inputFile1, inputFile2)); + + assertThat(store.getFilesByExtension("java")).containsExactlyInAnyOrder(inputFile1, inputFile2); + } + + @Test + void should_not_cache_duplicates() { ModuleInputComponentStore store = newModuleInputComponentStore(); String ext = "java"; String filename = "Program." + ext; - InputFile inputFile = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile); store.doAdd(inputFile); store.doAdd(inputFile); @@ -102,12 +173,12 @@ public class ModuleInputComponentStoreTest { } @Test - public void should_get_empty_iterable_on_cache_miss() { + void should_get_empty_iterable_on_cache_miss() { ModuleInputComponentStore store = newModuleInputComponentStore(); String ext = "java"; String filename = "Program." + ext; - InputFile inputFile = new TestInputFileBuilder(projectKey, "some/path/" + filename).build(); + InputFile inputFile = new TestInputFileBuilder(projectKey, "module/some/path/" + filename).build(); store.doAdd(inputFile); assertThat(store.getFilesByName("nonexistent")).isEmpty(); @@ -115,48 +186,42 @@ public class ModuleInputComponentStoreTest { } private ModuleInputComponentStore newModuleInputComponentStore() { - InputModule module = mock(InputModule.class); - when(module.key()).thenReturn("moduleKey"); - return new ModuleInputComponentStore(module, componentStore, mock(SensorStrategy.class)); + SensorStrategy strategy = new SensorStrategy(); + strategy.setGlobal(false); + return new ModuleInputComponentStore(sensorContextTester.module(), componentStore, strategy); } @Test - public void should_find_module_components_with_non_global_strategy() { - InputComponentStore inputComponentStore = mock(InputComponentStore.class); + void should_find_module_components_with_non_global_strategy() { SensorStrategy strategy = new SensorStrategy(); - InputModule module = mock(InputModule.class); - when(module.key()).thenReturn("foo"); - ModuleInputComponentStore store = new ModuleInputComponentStore(module, inputComponentStore, strategy); + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), mockedInputComponentStore, strategy); strategy.setGlobal(false); store.inputFiles(); - verify(inputComponentStore).filesByModule("foo"); + verify(mockedInputComponentStore).filesByModule(sensorContextTester.module().key()); String relativePath = "somepath"; store.inputFile(relativePath); - verify(inputComponentStore).getFile(any(String.class), eq(relativePath)); + verify(mockedInputComponentStore).getFile(any(String.class), eq(relativePath)); store.languages(); - verify(inputComponentStore).languages(any(String.class)); + verify(mockedInputComponentStore).languages(any(String.class)); } @Test - public void should_find_all_components_with_global_strategy() { - InputComponentStore inputComponentStore = mock(InputComponentStore.class); + void should_find_all_components_with_global_strategy() { SensorStrategy strategy = new SensorStrategy(); - ModuleInputComponentStore store = new ModuleInputComponentStore(mock(InputModule.class), inputComponentStore, strategy); - - strategy.setGlobal(true); + ModuleInputComponentStore store = new ModuleInputComponentStore(sensorContextTester.module(), mockedInputComponentStore, strategy); store.inputFiles(); - verify(inputComponentStore).inputFiles(); + verify(mockedInputComponentStore).inputFiles(); String relativePath = "somepath"; store.inputFile(relativePath); - verify(inputComponentStore).inputFile(relativePath); + verify(mockedInputComponentStore).inputFile(relativePath); store.languages(); - verify(inputComponentStore).languages(); + verify(mockedInputComponentStore).languages(); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java index 31d3312853b..485708c9936 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/MutableFileSystemTest.java @@ -28,6 +28,9 @@ import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; public class MutableFileSystemTest { @@ -44,9 +47,15 @@ public class MutableFileSystemTest { } @Test - public void return_all_files_when_not_restricted() { + public void restriction_and_hidden_file_should_be_disabled_on_default() { + assertThat(underTest.restrictToChangedFiles).isFalse(); + assertThat(underTest.allowHiddenFileAnalysis).isFalse(); + } + + @Test + public void return_all_non_hidden_files_when_not_restricted_and_disabled() { assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); - addFileWithAllStatus(); + addFilesWithAllStatus(); underTest.setRestrictToChangedFiles(false); assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(3); @@ -58,7 +67,7 @@ public class MutableFileSystemTest { @Test public void return_only_changed_files_when_restricted() { assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); - addFileWithAllStatus(); + addFilesWithAllStatus(); underTest.setRestrictToChangedFiles(true); assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(2); @@ -67,19 +76,95 @@ public class MutableFileSystemTest { assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(InputFile.Status.CHANGED)))).isNotNull(); } - private void addFileWithAllStatus() { + @Test + public void return_all_files_when_allowing_hidden_files_analysis() { + assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); + addFilesWithVisibility(); + underTest.setAllowHiddenFileAnalysis(true); + + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(2); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNotNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + } + + @Test + public void return_only_non_hidden_files_when_not_allowing_hidden_files_analysis() { + assertThat(underTest.inputFiles(underTest.predicates().all())).isEmpty(); + addFilesWithVisibility(); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(1); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + } + + @Test + public void hidden_file_predicate_should_preserve_predicate_optimization() { + addFilesWithVisibility(); + var anotherHiddenFile = spy(new TestInputFileBuilder("foo", String.format("src/%s", ".myHiddenFile.txt")) + .setLanguage(LANGUAGE).setStatus(InputFile.Status.ADDED).setHidden(true).build()); + underTest.add(anotherHiddenFile); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(true)))).isNull(); + assertThat(underTest.inputFile(underTest.predicates().hasFilename(generateFilename(false)))).isNotNull(); + // Verify that predicate optimization is still effective + verify(anotherHiddenFile, never()).isHidden(); + + // This predicate can't be optimized + assertThat(underTest.inputFiles(underTest.predicates().all())).hasSize(1); + verify(anotherHiddenFile).isHidden(); + } + + @Test + public void hidden_file_predicate_should_be_applied_first_for_non_optimized_predicates() { + // Checking the file type is not very costly, but it is not optimized. In real life, something more costly would be reading the file + // content, for example. + addFilesWithVisibility(); + var anotherHiddenFile = spy(new TestInputFileBuilder("foo", String.format("src/%s", ".myHiddenFile." + LANGUAGE)) + .setLanguage(LANGUAGE).setType(InputFile.Type.MAIN).setStatus(InputFile.Status.ADDED).setHidden(true).build()); + underTest.add(anotherHiddenFile); + underTest.setAllowHiddenFileAnalysis(false); + + assertThat(underTest.inputFiles(underTest.predicates().hasType(InputFile.Type.MAIN))).hasSize(1); + // Verify that the file type has not been evaluated + verify(anotherHiddenFile, never()).type(); + } + + private void addFilesWithVisibility() { + addFile(true); + addFile(false); + } + + private void addFilesWithAllStatus() { addFile(InputFile.Status.ADDED); addFile(InputFile.Status.CHANGED); addFile(InputFile.Status.SAME); } private void addFile(InputFile.Status status) { - underTest.add(new TestInputFileBuilder("foo", String.format("src/%s", generateFilename(status))) - .setLanguage(LANGUAGE).setStatus(status).build()); + addFile(status, false); + } + + private void addFile(boolean hidden) { + addFile(InputFile.Status.SAME, hidden); + } + + private void addFile(InputFile.Status status, boolean hidden) { + underTest.add(new TestInputFileBuilder("foo", String.format("src/%s", generateFilename(status, hidden))) + .setLanguage(LANGUAGE).setType(InputFile.Type.MAIN).setStatus(status).setHidden(hidden).build()); + } + + private String generateFilename(boolean hidden) { + return generateFilename(InputFile.Status.SAME, hidden); } private String generateFilename(InputFile.Status status) { - return String.format("%s.%s", status.name().toLowerCase(Locale.ROOT), LANGUAGE); + return generateFilename(status, false); + } + + private String generateFilename(InputFile.Status status, boolean hidden) { + return String.format("%s.%s.%s", status.name().toLowerCase(Locale.ROOT), hidden, LANGUAGE); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java index a6cabe242fb..355fb2cde0e 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/DefaultSensorStorageTest.java @@ -97,7 +97,6 @@ class DefaultSensorStorageTest { public void prepare() { MetricFinder metricFinder = mock(MetricFinder.class); when(metricFinder.<Integer>findByKey(CoreMetrics.NCLOC_KEY)).thenReturn(CoreMetrics.NCLOC); - when(metricFinder.<String>findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION); when(metricFinder.<Integer>findByKey(CoreMetrics.LINES_TO_COVER_KEY)).thenReturn(CoreMetrics.LINES_TO_COVER); settings = new MapSettings(); @@ -112,8 +111,9 @@ class DefaultSensorStorageTest { branchConfiguration = mock(BranchConfiguration.class); - underTest = new DefaultSensorStorage(metricFinder, - moduleIssues, settings.asConfig(), reportPublisher, mock(SonarCpdBlockIndex.class), contextPropertiesCache, telemetryCache, new ScannerMetrics(), branchConfiguration); + underTest = new DefaultSensorStorage( + metricFinder, moduleIssues, settings.asConfig(), reportPublisher, mock(SonarCpdBlockIndex.class), contextPropertiesCache, telemetryCache, new ScannerMetrics(), + branchConfiguration); project = new DefaultInputProject(ProjectDefinition.create() .setKey("foo") diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java index 4ab9f46fb4a..3a0ff7fb4c2 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ModuleSensorContextTest.java @@ -68,7 +68,7 @@ class ModuleSensorContextTest { @BeforeEach void prepare() { fs = new DefaultFileSystem(temp); - underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); } @@ -104,7 +104,7 @@ class ModuleSensorContextTest { @Test void pull_request_can_skip_unchanged_files() { when(branchConfiguration.isPullRequest()).thenReturn(true); - underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, + underTest = new ModuleSensorContext(mock(DefaultInputProject.class), mock(InputModule.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); assertThat(underTest.canSkipUnchangedFiles()).isTrue(); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java index 3c7f3d36793..01c337a5ed0 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/sensor/ProjectSensorContextTest.java @@ -59,8 +59,8 @@ class ProjectSensorContextTest { private ExecutingSensorContext executingSensorContext = mock(ExecutingSensorContext.class); private ScannerPluginRepository pluginRepository = mock(ScannerPluginRepository.class); - private ProjectSensorContext underTest = new ProjectSensorContext(mock(DefaultInputProject.class), settings.asConfig(), settings, fs, activeRules, sensorStorage, runtime, - branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); + private ProjectSensorContext underTest = new ProjectSensorContext(mock(DefaultInputProject.class), settings.asConfig(), fs, activeRules, sensorStorage, runtime, + branchConfiguration, writeCache, readCache, analysisCacheEnabled, unchangedFilesHandler, executingSensorContext, pluginRepository); private static final String PLUGIN_KEY = "org.sonarsource.pluginKey"; @@ -69,7 +69,6 @@ class ProjectSensorContextTest { when(executingSensorContext.getSensorExecuting()).thenReturn(new SensorId(PLUGIN_KEY, "sensorName")); } - @Test void addTelemetryProperty_whenTheOrganizationIsSonarSource_mustStoreTheTelemetry() { @@ -77,16 +76,21 @@ class ProjectSensorContextTest { underTest.addTelemetryProperty("key", "value"); - //then verify that the defaultStorage is called with the telemetry property once + // then verify that the defaultStorage is called with the telemetry property once verify(sensorStorage).storeTelemetry("key", "value"); } @Test - void addTelemetryProperty_whenTheOrganizationIsNotSonarSource_mustThrowExcaption() { + void addTelemetryProperty_whenTheOrganizationIsNotSonarSource_mustThrowException() { when(pluginRepository.getPluginInfo(PLUGIN_KEY)).thenReturn(new PluginInfo(PLUGIN_KEY).setOrganizationName("notSonarsource")); assertThrows(IllegalStateException.class, () -> underTest.addTelemetryProperty("key", "value")); verifyNoInteractions(sensorStorage); } + + @Test + void settings_throwsUnsupportedOperationException() { + assertThrows(UnsupportedOperationException.class, () -> underTest.settings()); + } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java index fe637015ed7..91f59b964b9 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedFileTest.java @@ -88,7 +88,7 @@ public class ChangedFileTest { secure().next(5), Integer.parseInt(secure().nextNumeric(5)), new SensorStrategy(), - oldRelativePath); + oldRelativePath, false); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java index 9f4e0b11e7b..775fe3d05b2 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java @@ -47,6 +47,7 @@ import org.sonar.api.batch.scm.BlameLine; import org.sonar.api.notifications.AnalysisWarnings; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.api.utils.System2; +import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scm.git.strategy.DefaultBlameStrategy.BlameAlgorithmEnum; import static org.assertj.core.api.Assertions.assertThat; diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandTest.java index 87c456dbf69..a1a4cae8b4d 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandTest.java @@ -54,6 +54,7 @@ import org.sonar.api.testfixtures.log.LogTester; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.System2; +import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scm.git.strategy.BlameStrategy; import org.sonar.scm.git.strategy.DefaultBlameStrategy.BlameAlgorithmEnum; diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java index b7c99187971..058c274013d 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java @@ -61,6 +61,7 @@ import org.sonar.api.testfixtures.log.LogTester; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.System2; import org.sonar.core.documentation.DocumentationLinkGenerator; +import org.sonar.core.util.ProcessWrapperFactory; import org.sonar.scm.git.strategy.DefaultBlameStrategy; import static java.lang.String.format; diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java new file mode 100644 index 00000000000..d8264a5745b --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitUtilsTest.java @@ -0,0 +1,101 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.apache.commons.lang.SystemUtils; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.sonar.api.utils.MessageException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class JGitUtilsTest { + + @TempDir + Path rootModuleDir; + + @Test + void getAllIgnoredPaths_ReturnsIgnoredFiles() throws Exception { + setupTestDirectory(); + + List<String> result = JGitUtils.getAllIgnoredPaths(rootModuleDir); + + // in directory1, the entire directory is ignored without listing each file + // in directory2, specific files are ignored, so those files are listed + // in directory3, specific files are ignored via a separate .gitignore file + if (SystemUtils.IS_OS_WINDOWS) { + assertThat(result).isEqualTo(List.of("directory1", "directory2\\file_a.txt", "directory3\\file_b.txt")); + } else { + assertThat(result).isEqualTo(List.of("directory1", "directory2/file_a.txt", "directory3/file_b.txt")); + } + } + + @Test + void getIgnoredPaths_WithNonGitDirectory_ThrowsException() { + assertThatThrownBy(() -> JGitUtils.getAllIgnoredPaths(rootModuleDir)) + .isInstanceOf(MessageException.class) + .hasMessageStartingWith("Not inside a Git work tree: "); + } + + @Test + void getIgnoredPaths_WithDifferentBaseDir_ReturnsIgnoredFilesRelativeToBaseDir() throws Exception { + Path baseDir = rootModuleDir.resolve("directory2"); + setupTestDirectory(); + + List<String> result = JGitUtils.getAllIgnoredPaths(baseDir); + + assertThat(result).isEqualTo(List.of("file_a.txt")); + } + + @Test + void getIgnoredPaths_WithSubDirBaseDirContainingGitIgnore_ReturnsIgnoredFilesRelativeToBaseDir() throws Exception { + Path baseDir = rootModuleDir.resolve("directory3"); + setupTestDirectory(); + + List<String> result = JGitUtils.getAllIgnoredPaths(baseDir); + + assertThat(result).isEqualTo(List.of("file_b.txt")); + } + + private void setupTestDirectory() throws GitAPIException, IOException { + Git.init().setDirectory(rootModuleDir.toFile()).call(); + + var directories = List.of("directory1", "directory2", "directory3"); + var fileNames = List.of("file_a.txt", "file_b.txt"); + + for (String dir : directories) { + Path directoryPath = rootModuleDir.resolve(dir); + Files.createDirectories(directoryPath); + for (String fileName : fileNames) { + Files.write(directoryPath.resolve(fileName), "content".getBytes()); + } + } + + Files.write(rootModuleDir.resolve(".gitignore"), "ignored.txt\ndirectory1\ndirectory2/file_a.txt".getBytes()); + Files.write(rootModuleDir.resolve("directory3/.gitignore"), "file_b.txt".getBytes()); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java index df3d257c287..23d5e6ca5f0 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/NativeGitBlameCommandTest.java @@ -19,7 +19,6 @@ */ package org.sonar.scm.git; -import com.tngtech.java.junit.dataprovider.DataProviderRunner; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -32,21 +31,21 @@ import java.util.function.Consumer; import java.util.stream.Stream; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; import org.slf4j.event.Level; import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.testfixtures.log.LogTesterJUnit5; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; -import org.sonar.api.testfixtures.log.LogTester; -import org.sonar.scm.git.ProcessWrapperFactory.ProcessWrapper; +import org.sonar.core.util.ProcessWrapperFactory; +import org.sonar.core.util.ProcessWrapperFactory.ProcessWrapper; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -54,32 +53,31 @@ import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.sonar.scm.git.GitUtils.createFile; +import static org.sonar.scm.git.GitUtils.createRepository; import static org.sonar.scm.git.NativeGitBlameCommand.BLAME_COMMAND; import static org.sonar.scm.git.NativeGitBlameCommand.GIT_DIR_ARGUMENT; import static org.sonar.scm.git.NativeGitBlameCommand.GIT_DIR_FLAG; import static org.sonar.scm.git.NativeGitBlameCommand.GIT_DIR_FORCE_FLAG; -import static org.sonar.scm.git.GitUtils.createFile; -import static org.sonar.scm.git.GitUtils.createRepository; import static org.sonar.scm.git.Utils.javaUnzip; -@RunWith(DataProviderRunner.class) -public class NativeGitBlameCommandTest { +class NativeGitBlameCommandTest { private static final String DUMMY_JAVA = "src/main/java/org/dummy/Dummy.java"; - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - @Rule - public LogTester logTester = new LogTester(); + @TempDir + private Path tempDir; + @RegisterExtension + private final LogTesterJUnit5 logTester = new LogTesterJUnit5(); private final ProcessWrapperFactory processWrapperFactory = new ProcessWrapperFactory(); private final NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, processWrapperFactory); - @Before - public void skipTestsIfNoGitFound() { + @BeforeEach + void skipTestsIfNoGitFound() { assumeTrue(blameCommand.checkIfEnabled()); } @Test - public void should_read_lines_only_based_on_new_line() throws Exception { + void should_read_lines_only_based_on_new_line() throws Exception { Path baseDir = createNewTempFolder().toPath(); String filePath = "file.txt"; createFile(filePath, "test1\rtest2\r\ttest3", baseDir); @@ -92,7 +90,7 @@ public class NativeGitBlameCommandTest { } @Test - public void blame_collects_all_lines() throws Exception { + void blame_collects_all_lines() throws Exception { File projectDir = createNewTempFolder(); javaUnzip("dummy-git.zip", projectDir); File baseDir = new File(projectDir, "dummy-git"); @@ -123,7 +121,7 @@ public class NativeGitBlameCommandTest { } @Test - public void blame_different_author_and_committer() throws Exception { + void blame_different_author_and_committer() throws Exception { File projectDir = createNewTempFolder(); javaUnzip("dummy-git-different-committer.zip", projectDir); File baseDir = new File(projectDir, "dummy-git"); @@ -154,21 +152,21 @@ public class NativeGitBlameCommandTest { } @Test - public void git_blame_uses_safe_local_repository() throws Exception { + void git_blame_uses_safe_local_repository() throws Exception { File projectDir = createNewTempFolder(); File baseDir = new File(projectDir, "dummy-git"); ProcessWrapperFactory mockFactory = mock(ProcessWrapperFactory.class); ProcessWrapper mockProcess = mock(ProcessWrapper.class); String gitCommand = "git"; - when(mockFactory.create(any(), any(), anyString(), anyString(), anyString(), anyString(), + when(mockFactory.create(any(), any(), any(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), anyString())) - .then(invocation -> mockProcess); + .then(invocation -> mockProcess); NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(gitCommand, System2.INSTANCE, mockFactory); blameCommand.blame(baseDir.toPath(), DUMMY_JAVA); - verify(mockFactory).create(any(), any(), eq(gitCommand), + verify(mockFactory).create(any(), any(), any(), eq(gitCommand), eq(GIT_DIR_FLAG), eq(String.format(GIT_DIR_ARGUMENT, baseDir.toPath())), eq(GIT_DIR_FORCE_FLAG), @@ -178,7 +176,7 @@ public class NativeGitBlameCommandTest { } @Test - public void modified_file_returns_no_blame() throws Exception { + void modified_file_returns_no_blame() throws Exception { File projectDir = createNewTempFolder(); javaUnzip("dummy-git.zip", projectDir); @@ -191,9 +189,8 @@ public class NativeGitBlameCommandTest { } @Test - public void throw_exception_if_symlink_found() throws Exception { + void throw_exception_if_symlink_found(@TempDir File projectDir) throws Exception { assumeTrue(!System2.INSTANCE.isOsWindows()); - File projectDir = temp.newFolder(); javaUnzip("dummy-git.zip", projectDir); Path baseDir = projectDir.toPath().resolve("dummy-git"); @@ -207,69 +204,66 @@ public class NativeGitBlameCommandTest { } @Test - public void git_should_be_detected() { + void git_should_be_detected() { NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, processWrapperFactory); assertThat(blameCommand.checkIfEnabled()).isTrue(); } @Test - public void git_should_not_be_detected() { + void git_should_not_be_detected() { NativeGitBlameCommand blameCommand = new NativeGitBlameCommand("randomcmdthatwillneverbefound", System2.INSTANCE, processWrapperFactory); assertThat(blameCommand.checkIfEnabled()).isFalse(); } @Test - public void git_should_not_be_enabled_if_version_command_is_not_found() { + void git_should_not_be_enabled_if_version_command_is_not_found() { ProcessWrapperFactory mockedCmd = mockGitVersionCommand("error: unknown option `version'"); NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, mockedCmd); assertThat(blameCommand.checkIfEnabled()).isFalse(); } @Test - public void git_should_not_be_enabled_if_version_command_does_not_return_string_output() { + void git_should_not_be_enabled_if_version_command_does_not_return_string_output() { ProcessWrapperFactory mockedCmd = mockGitVersionCommand(null); NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, mockedCmd); assertThat(blameCommand.checkIfEnabled()).isFalse(); } @Test - public void git_should_be_enabled_if_version_is_equal_or_greater_than_required_minimum() { + void git_should_be_enabled_if_version_is_equal_or_greater_than_required_minimum() { Stream.of( "git version 2.24.0", "git version 2.25.2.1", "git version 2.24.1.1.windows.2", - "git version 2.25.1.msysgit.2" - ).forEach(output -> { - ProcessWrapperFactory mockedCmd = mockGitVersionCommand(output); - mockGitWhereOnWindows(mockedCmd); - when(mockedCmd.create(isNull(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { - var argument = (Consumer<String>) invocation.getArgument(1); - argument.accept(output); - return mock(ProcessWrapper.class); + "git version 2.25.1.msysgit.2").forEach(output -> { + ProcessWrapperFactory mockedCmd = mockGitVersionCommand(output); + mockGitWhereOnWindows(mockedCmd); + when(mockedCmd.create(isNull(), any(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { + var argument = (Consumer<String>) invocation.getArgument(1); + argument.accept(output); + return mock(ProcessWrapper.class); + }); + + NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, mockedCmd); + assertThat(blameCommand.checkIfEnabled()).isTrue(); }); - - NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, mockedCmd); - assertThat(blameCommand.checkIfEnabled()).isTrue(); - }); } @Test - public void git_should_not_be_enabled_if_version_is_less_than_required_minimum() { + void git_should_not_be_enabled_if_version_is_less_than_required_minimum() { ProcessWrapperFactory mockFactory = mockGitVersionCommand("git version 1.9.0"); NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, mockFactory); assertThat(blameCommand.checkIfEnabled()).isFalse(); } @Test - public void throw_exception_if_command_fails() throws Exception { - Path baseDir = temp.newFolder().toPath(); + void throw_exception_if_command_fails(@TempDir Path baseDir) { NativeGitBlameCommand blameCommand = new NativeGitBlameCommand("randomcmdthatwillneverbefound", System2.INSTANCE, processWrapperFactory); assertThatThrownBy(() -> blameCommand.blame(baseDir, "file")).isInstanceOf(IOException.class); } @Test - public void blame_without_email_doesnt_fail() throws Exception { - Path baseDir = temp.newFolder().toPath(); + void blame_without_email_doesnt_fail(@TempDir Path baseDir) throws Exception { Git git = createRepository(baseDir); String filePath = "file.txt"; createFile(filePath, "line", baseDir); @@ -286,8 +280,7 @@ public class NativeGitBlameCommandTest { } @Test - public void blame_mail_with_spaces_doesnt_fail() throws Exception { - Path baseDir = temp.newFolder().toPath(); + void blame_mail_with_spaces_doesnt_fail(@TempDir Path baseDir) throws Exception { Git git = createRepository(baseDir); String filePath = "file.txt"; createFile(filePath, "line", baseDir); @@ -301,25 +294,7 @@ public class NativeGitBlameCommandTest { } @Test - public void do_not_execute() throws Exception { - Path baseDir = temp.newFolder().toPath(); - Git git = createRepository(baseDir); - String filePath = "file.txt"; - createFile(filePath, "line", baseDir); - commitWithNoEmail(git, filePath); - - NativeGitBlameCommand blameCommand = new NativeGitBlameCommand(System2.INSTANCE, processWrapperFactory); - assertThat(blameCommand.checkIfEnabled()).isTrue(); - List<BlameLine> blame = blameCommand.blame(baseDir, filePath); - assertThat(blame).hasSize(1); - BlameLine blameLine = blame.get(0); - assertThat(blameLine.author()).isNull(); - assertThat(blameLine.revision()).isNotNull(); - assertThat(blameLine.date()).isNotNull(); - } - - @Test - public void execution_on_windows_should_fallback_to_full_path() { + void execution_on_windows_should_fallback_to_full_path() { logTester.setLevel(Level.DEBUG); System2 system2 = mock(System2.class); when(system2.isOsWindows()).thenReturn(true); @@ -328,7 +303,7 @@ public class NativeGitBlameCommandTest { ProcessWrapper mockProcess = mock(ProcessWrapper.class); mockGitWhereOnWindows(mockFactory); - when(mockFactory.create(isNull(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { + when(mockFactory.create(isNull(), any(), any(), eq("C:\\mockGit.exe"), eq("--version"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept("git version 2.30.1"); return mockProcess; @@ -340,7 +315,7 @@ public class NativeGitBlameCommandTest { } @Test - public void execution_on_windows_is_disabled_if_git_not_on_path() { + void execution_on_windows_is_disabled_if_git_not_on_path() { System2 system2 = mock(System2.class); when(system2.isOsWindows()).thenReturn(true); when(system2.property("PATH")).thenReturn("C:\\some-path;C:\\some-another-path"); @@ -367,11 +342,11 @@ public class NativeGitBlameCommandTest { private File createNewTempFolder() throws IOException { // This is needed for Windows, otherwise the created File point to invalid (shortened by Windows) temp folder path - return temp.newFolder().toPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toFile(); + return tempDir.toRealPath(LinkOption.NOFOLLOW_LINKS).toFile(); } private void mockGitWhereOnWindows(ProcessWrapperFactory processWrapperFactory) { - when(processWrapperFactory.create(isNull(), any(), eq("C:\\Windows\\System32\\where.exe"), eq("$PATH:git.exe"))).then(invocation -> { + when(processWrapperFactory.create(isNull(), any(), any(), eq("C:\\Windows\\System32\\where.exe"), eq("$PATH:git.exe"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept("C:\\mockGit.exe"); return mock(ProcessWrapper.class); @@ -382,7 +357,7 @@ public class NativeGitBlameCommandTest { ProcessWrapperFactory mockFactory = mock(ProcessWrapperFactory.class); ProcessWrapper mockProcess = mock(ProcessWrapper.class); - when(mockFactory.create(isNull(), any(), eq("git"), eq("--version"))).then(invocation -> { + when(mockFactory.create(isNull(), any(), any(), eq("git"), eq("--version"))).then(invocation -> { var argument = (Consumer<String>) invocation.getArgument(1); argument.accept(commandOutput); return mockProcess; diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ProcessWrapperFactoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ProcessWrapperFactoryTest.java deleted file mode 100644 index 92d2be057e6..00000000000 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ProcessWrapperFactoryTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2025 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scm.git; - -import java.io.IOException; -import java.util.Map; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.event.Level; -import org.sonar.api.testfixtures.log.LogTester; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class ProcessWrapperFactoryTest { - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - @Rule - public LogTester logTester = new LogTester(); - private final ProcessWrapperFactory underTest = new ProcessWrapperFactory(); - - @Test - public void should_log_error_output_in_debug_mode() throws IOException { - logTester.setLevel(Level.DEBUG); - var root = temp.newFolder().toPath(); - var processWrapper = underTest.create(root, v -> {}, Map.of("LANG", "en_US"), "git", "blame"); - assertThatThrownBy(processWrapper::execute) - .isInstanceOf(IllegalStateException.class); - - assertThat(logTester.logs(Level.DEBUG).get(0)).startsWith("fatal:"); - } - -} diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat new file mode 100644 index 00000000000..577375b330d --- /dev/null +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.bat @@ -0,0 +1,26 @@ +@echo off +echo Arguments Passed In: %* + +setlocal enabledelayedexpansion +set "POSITIONAL_ARGS=" + +:loop +if "%~1"=="" goto endloop +if "%~1"=="--xz-filename" ( + set "FILENAME=%~2" + shift + shift +) else ( + set "POSITIONAL_ARGS=!POSITIONAL_ARGS! %~1" + shift +) +goto loop +:endloop + +echo TIDELIFT_SKIP_UPDATE_CHECK=%TIDELIFT_SKIP_UPDATE_CHECK% +echo TIDELIFT_ALLOW_MANIFEST_FAILURES=%TIDELIFT_ALLOW_MANIFEST_FAILURES% +echo TIDELIFT_RECURSIVE_MANIFEST_SEARCH=%TIDELIFT_RECURSIVE_MANIFEST_SEARCH% +echo TIDELIFT_CLI_INSIDE_SCANNER_ENGINE=%TIDELIFT_CLI_INSIDE_SCANNER_ENGINE% +echo TIDELIFT_CLI_SQ_SERVER_VERSION=%TIDELIFT_CLI_SQ_SERVER_VERSION% +echo ZIP FILE LOCATION = %FILENAME% +echo. > %FILENAME% diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh new file mode 100755 index 00000000000..f7feed1f501 --- /dev/null +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/sca/echo_args.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +echo "Arguments Passed In:" $@ + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --xz-filename) + FILENAME="$2" + shift + shift + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + +# print specific env variables that should be defined here +echo "TIDELIFT_SKIP_UPDATE_CHECK=${TIDELIFT_SKIP_UPDATE_CHECK}" +echo "TIDELIFT_ALLOW_MANIFEST_FAILURES=${TIDELIFT_ALLOW_MANIFEST_FAILURES}" +echo "TIDELIFT_RECURSIVE_MANIFEST_SEARCH=${TIDELIFT_RECURSIVE_MANIFEST_SEARCH}" +echo "TIDELIFT_CLI_INSIDE_SCANNER_ENGINE=${TIDELIFT_CLI_INSIDE_SCANNER_ENGINE}" +echo "TIDELIFT_CLI_SQ_SERVER_VERSION=${TIDELIFT_CLI_SQ_SERVER_VERSION}" + +# print filename location for debug purposes +echo "ZIP FILE LOCATION = ${FILENAME}" +echo "" > $FILENAME diff --git a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/AnalysisResult.java b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/AnalysisResult.java index f29a377e38f..57843c8abd7 100644 --- a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/AnalysisResult.java +++ b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/AnalysisResult.java @@ -199,8 +199,8 @@ public class AnalysisResult implements AnalysisObserver { return readFromReport(ScannerReportReader::readAdHocRules); } - public List<ScannerReport.Dependency> dependencies() { - return readFromReport(ScannerReportReader::readDependencies); + public List<ScannerReport.AnalysisData> analysisData() { + return readFromReport(ScannerReportReader::readAnalysisData); } @NotNull diff --git a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakeFeatureFlagsLoader.java b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakeFeatureFlagsLoader.java new file mode 100644 index 00000000000..26856c3b98e --- /dev/null +++ b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakeFeatureFlagsLoader.java @@ -0,0 +1,41 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.mediumtest; + +import jakarta.annotation.Priority; +import java.util.HashSet; +import java.util.Set; +import org.sonar.scanner.repository.featureflags.FeatureFlagsLoader; + +@Priority(1) +public class FakeFeatureFlagsLoader implements FeatureFlagsLoader { + + private final Set<String> features = new HashSet<>(); + + @Override + public Set<String> load() { + return features; + } + + public void enableFeature(String featureName) { + features.add(featureName); + } + +} diff --git a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java index 60a8348f778..5934c9da0bc 100644 --- a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java +++ b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/FakePluginInstaller.java @@ -19,13 +19,13 @@ */ package org.sonar.scanner.mediumtest; +import jakarta.annotation.Priority; import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import jakarta.annotation.Priority; import org.sonar.api.Plugin; import org.sonar.core.platform.PluginInfo; import org.sonar.core.plugin.PluginType; @@ -49,6 +49,11 @@ public class FakePluginInstaller implements PluginInstaller { return this; } + public FakePluginInstaller add(PluginInfo pluginInfo, Plugin instance) { + mediumTestPlugins.add(new LocalPlugin(pluginInfo, instance, Set.of())); + return this; + } + public FakePluginInstaller addOptional(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages)); return this; diff --git a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java index 1cdd295a82d..f2edfe609d4 100644 --- a/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java +++ b/sonar-scanner-engine/src/testFixtures/java/org/sonar/scanner/mediumtest/ScannerMediumTester.java @@ -19,6 +19,9 @@ */ package org.sonar.scanner.mediumtest; +import static java.util.Collections.emptySet; + +import jakarta.annotation.Priority; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -38,7 +41,6 @@ import java.util.Properties; import java.util.Set; import javax.annotation.CheckForNull; import javax.annotation.Nullable; -import jakarta.annotation.Priority; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.extension.AfterTestExecutionCallback; import org.junit.jupiter.api.extension.BeforeTestExecutionCallback; @@ -60,6 +62,7 @@ import org.sonar.api.utils.Version; import org.sonar.batch.bootstrapper.Batch; import org.sonar.batch.bootstrapper.EnvironmentInformation; import org.sonar.batch.bootstrapper.LogOutput; +import org.sonar.core.platform.PluginInfo; import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import org.sonar.scanner.cache.AnalysisCacheLoader; import org.sonar.scanner.protocol.internal.SensorCacheData; @@ -86,8 +89,6 @@ import org.sonarqube.ws.NewCodePeriods; import org.sonarqube.ws.Qualityprofiles.SearchWsResponse.QualityProfile; import org.sonarqube.ws.Rules.Rule; -import static java.util.Collections.emptySet; - /** * Main utility class for writing scanner medium tests. */ @@ -111,6 +112,7 @@ public class ScannerMediumTester extends ExternalResource implements BeforeTestE private final CeTaskReportDataHolder reportMetadataHolder = new CeTaskReportDataHolderExt(); private final FakeLanguagesLoader languagesLoader = new FakeLanguagesLoader(); private final FakeLanguagesProvider languagesProvider = new FakeLanguagesProvider(); + private final FakeFeatureFlagsLoader featureFlagsLoader = new FakeFeatureFlagsLoader(); private LogOutput logOutput = null; private static void createWorkingDirs() throws IOException { @@ -145,6 +147,11 @@ public class ScannerMediumTester extends ExternalResource implements BeforeTestE return this; } + public ScannerMediumTester registerPlugin(PluginInfo pluginInfo, Plugin instance) { + pluginInstaller.add(pluginInfo, instance); + return this; + } + public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set<String> requiredForLanguages, Plugin instance) { pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance); return this; @@ -310,6 +317,10 @@ public class ScannerMediumTester extends ExternalResource implements BeforeTestE languagesProvider.addLanguage(key, name, publishAllFiles); } + public void enableFeature(String featureName) { + featureFlagsLoader.enableFeature(featureName); + } + public static class AnalysisBuilder { private final Map<String, String> taskProperties = new HashMap<>(); private final ScannerMediumTester tester; @@ -343,6 +354,7 @@ public class ScannerMediumTester extends ExternalResource implements BeforeTestE tester.reportMetadataHolder, tester.languagesLoader, tester.languagesProvider, + tester.featureFlagsLoader, result); if (tester.logOutput != null) { builder.setLogOutput(tester.logOutput); |