diff options
17 files changed, 295 insertions, 53 deletions
diff --git a/server/sonar-webserver-common/src/it/java/org/sonar/server/common/component/ComponentUpdaterIT.java b/server/sonar-webserver-common/src/it/java/org/sonar/server/common/component/ComponentUpdaterIT.java index 809c680b6e2..82375857e19 100644 --- a/server/sonar-webserver-common/src/it/java/org/sonar/server/common/component/ComponentUpdaterIT.java +++ b/server/sonar-webserver-common/src/it/java/org/sonar/server/common/component/ComponentUpdaterIT.java @@ -40,6 +40,7 @@ import org.sonar.db.component.BranchType; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.ComponentQualifiers; import org.sonar.db.component.ComponentScopes; +import org.sonar.db.portfolio.PortfolioDto; import org.sonar.db.project.CreationMethod; import org.sonar.db.project.ProjectDto; import org.sonar.db.user.UserDto; @@ -569,4 +570,34 @@ public class ComponentUpdaterIT { ProjectDto projectDto = underTest.create(db.getSession(), creationParameters).projectDto(); assertThat(projectDto.getCreationMethod()).isEqualTo(CreationMethod.ALM_IMPORT_BROWSER); } + + @Test + public void fail_to_create_portfolio_using_duplicate_child_portfolio_key() { + var parentPortfolio = db.components().insertPublicPortfolioDto(); + var childPortfolio = new PortfolioDto() + .setKey("ExistingChild") + .setName("name_" + "ExistingChild") + .setSelectionMode(PortfolioDto.SelectionMode.NONE) + .setRootUuid(parentPortfolio.getRootUuid()) + .setParentUuid(parentPortfolio.getUuid()) + .setUuid("uuid_" + "ExistingChild"); + DbSession session = db.getSession(); + db.getDbClient().portfolioDao().insertWithAudit(session, childPortfolio); + session.commit(); + + var newPortfolio = NewComponent.newComponentBuilder() + .setKey(childPortfolio.getKey()) + .setName("New Portfolio") + .setQualifier(VIEW) + .build(); + var creationParameters = ComponentCreationParameters.builder() + .newComponent(newPortfolio) + .creationMethod(CreationMethod.LOCAL_API) + .mainBranchName("main") + .build(); + + assertThatThrownBy(() -> underTest.create(session, creationParameters)) + .isInstanceOf(BadRequestException.class) + .hasMessage("Could not create component with key: \"%s\". Key already in use.", childPortfolio.getKey()); + } } diff --git a/server/sonar-webserver-common/src/main/java/org/sonar/server/common/component/ComponentUpdater.java b/server/sonar-webserver-common/src/main/java/org/sonar/server/common/component/ComponentUpdater.java index 10c39b9a7a4..6e7a0c00fa1 100644 --- a/server/sonar-webserver-common/src/main/java/org/sonar/server/common/component/ComponentUpdater.java +++ b/server/sonar-webserver-common/src/main/java/org/sonar/server/common/component/ComponentUpdater.java @@ -185,8 +185,11 @@ public class ComponentUpdater { } private void checkKeyAlreadyExists(DbSession dbSession, NewComponent newComponent) { + Optional<PortfolioDto> portfolios = dbClient.portfolioDao().selectByKey(dbSession, newComponent.key()); + if (portfolios.isPresent()) { + throwBadRequestException("Could not create component with key: \"%s\". Key already in use.", newComponent.key()); + } List<ComponentDto> componentDtos = dbClient.componentDao().selectByKeyCaseInsensitive(dbSession, newComponent.key()); - if (!componentDtos.isEmpty()) { String alreadyExistingKeys = componentDtos .stream() diff --git a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java index 9d6c9c9ebc5..39f91037d8c 100644 --- a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java +++ b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ServerMonitoringMetrics.java @@ -37,6 +37,7 @@ public class ServerMonitoringMetrics { private final Gauge cePendingTasksTotal; private final Summary ceTasksRunningDuration; + private final Summary ceSystemTasksRunningDuration; private final Gauge elasticsearchDiskSpaceFreeBytesGauge; private final Gauge elasticSearchDiskSpaceTotalBytes; @@ -80,6 +81,12 @@ public class ServerMonitoringMetrics { .labelNames("task_type", "project_key") .register(); + ceSystemTasksRunningDuration = Summary.build() + .name("sonarqube_compute_engine_system_tasks_running_duration_seconds") + .help("Compute engine system task running time in seconds") + .labelNames("task_type") + .register(); + computeEngineGauge = Gauge.build() .name("sonarqube_health_compute_engine_status") .help("Tells whether Compute Engine is up (healthy, ready to take tasks) or down. 1 for up, 0 for down") @@ -165,8 +172,12 @@ public class ServerMonitoringMetrics { cePendingTasksTotal.set(numberOfPendingTasks); } - public void observeComputeEngineTaskDuration(long durationInSeconds, String taskType, String projectKey) { - ceTasksRunningDuration.labels(taskType, projectKey).observe(durationInSeconds); + public void observeComputeEngineTaskDuration(long durationInSeconds, String taskType, String label) { + ceTasksRunningDuration.labels(taskType, label).observe(durationInSeconds); + } + + public void observeComputeEngineSystemTaskDuration(long durationInSeconds, String taskType) { + ceSystemTasksRunningDuration.labels(taskType).observe(durationInSeconds); } public void setComputeEngineStatusToGreen() { diff --git a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ce/RecentTasksDurationTask.java b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ce/RecentTasksDurationTask.java index 74d3dd85387..5a4c22d15d8 100644 --- a/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ce/RecentTasksDurationTask.java +++ b/server/sonar-webserver-monitoring/src/main/java/org/sonar/server/monitoring/ce/RecentTasksDurationTask.java @@ -22,11 +22,12 @@ package org.sonar.server.monitoring.ce; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; -import org.sonar.api.config.Configuration; -import org.sonar.api.utils.System2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.api.config.Configuration; +import org.sonar.api.utils.System2; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.ce.CeActivityDto; @@ -56,6 +57,7 @@ public class RecentTasksDurationTask extends ComputeEngineMetricsTask { Collection<String> entityUuids = recentSuccessfulTasks.stream() .map(CeActivityDto::getEntityUuid) + .filter(Objects::nonNull) .toList(); List<EntityDto> entities = dbClient.entityDao().selectByUuids(dbSession, entityUuids); Map<String, String> entityUuidAndKeys = entities.stream() @@ -75,20 +77,22 @@ public class RecentTasksDurationTask extends ComputeEngineMetricsTask { private void reportObservedDurationForTasks(List<CeActivityDto> tasks, Map<String, String> entityUuidAndKeys) { for (CeActivityDto task : tasks) { - String mainComponentUuid = task.getEntityUuid(); + String entityUuid = task.getEntityUuid(); Long executionTimeMs = task.getExecutionTimeMs(); try { - requireNonNull(mainComponentUuid); requireNonNull(executionTimeMs); - String mainComponentKey = entityUuidAndKeys.get(mainComponentUuid); - requireNonNull(mainComponentKey); - - metrics.observeComputeEngineTaskDuration(executionTimeMs, task.getTaskType(), mainComponentKey); + if (entityUuid != null) { + String label = entityUuidAndKeys.get(entityUuid); + requireNonNull(label); + metrics.observeComputeEngineTaskDuration(executionTimeMs, task.getTaskType(), label); + } else { + metrics.observeComputeEngineSystemTaskDuration(executionTimeMs, task.getTaskType()); + } } catch (RuntimeException e) { - LOGGER.warn("Can't report metric data for a CE task with component uuid " + mainComponentUuid, e); + LOGGER.warn("Can't report metric data for a CE task with entity uuid " + entityUuid, e); } } - } + } diff --git a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ServerMonitoringMetricsTest.java b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ServerMonitoringMetricsTest.java index 8b0cbd624f0..ecacceefccb 100644 --- a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ServerMonitoringMetricsTest.java +++ b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ServerMonitoringMetricsTest.java @@ -158,6 +158,18 @@ public class ServerMonitoringMetricsTest { labelNames, labelValues)).isEqualTo(10); } + @Test + public void observeComputeEngineSystemTaskDurationTest() { + ServerMonitoringMetrics metrics = new ServerMonitoringMetrics(); + String[] labelNames = {"task_type"}; + String[] labelValues = {"AUDIT_PURGE"}; + + metrics.observeComputeEngineSystemTaskDuration(10, labelValues[0]); + + assertThat(CollectorRegistry.defaultRegistry.getSampleValue("sonarqube_compute_engine_system_tasks_running_duration_seconds_sum", + labelNames, labelValues)).isEqualTo(10); + } + private int sizeOfDefaultRegistry() { Enumeration<Collector.MetricFamilySamples> metrics = CollectorRegistry.defaultRegistry.metricFamilySamples(); return Collections.list(metrics).size(); diff --git a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ce/RecentTasksDurationTaskTest.java b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ce/RecentTasksDurationTaskTest.java index 5b5fc0a3570..7fdbfaa9dee 100644 --- a/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ce/RecentTasksDurationTaskTest.java +++ b/server/sonar-webserver-monitoring/src/test/java/org/sonar/server/monitoring/ce/RecentTasksDurationTaskTest.java @@ -86,6 +86,22 @@ public class RecentTasksDurationTaskTest { } @Test + public void run_given1TaskWithEntityUuidAnd1Without_observeDurationFor2Tasks() { + RecentTasksDurationTask task = new RecentTasksDurationTask(dbClient, metrics, config, system); + List<CeActivityDto> recentTasks = createTasks(2, 0); + + recentTasks.get(0).setEntityUuid(null); + + when(entityDao.selectByUuids(any(), any())).thenReturn(createEntityDtos(1)); + when(ceActivityDao.selectNewerThan(any(), anyLong())).thenReturn(recentTasks); + + task.run(); + + verify(metrics, times(1)).observeComputeEngineTaskDuration(anyLong(), any(), any()); + verify(metrics, times(1)).observeComputeEngineSystemTaskDuration(anyLong(), any()); + } + + @Test public void run_givenNullExecutionTime_dontReportMetricData() { RecentTasksDurationTask task = new RecentTasksDurationTask(dbClient, metrics, config, system); List<CeActivityDto> recentTasks = createTasks(1, 0); diff --git a/sonar-scanner-engine-shaded/build.gradle b/sonar-scanner-engine-shaded/build.gradle index 7db17d8d133..172c83e79e7 100644 --- a/sonar-scanner-engine-shaded/build.gradle +++ b/sonar-scanner-engine-shaded/build.gradle @@ -10,6 +10,8 @@ dependencies { api project(':sonar-scanner-engine') } +artifactoryPublish.skip = !deployCommunity + jar { manifest { attributes( @@ -19,3 +21,13 @@ jar { ) } } + +// Used by the scanner integration tester +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + artifact shadowJar + } + } +} diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java index 63317e362b9..6cfd6ea5f94 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/bootstrap/BootstrapMediumIT.java @@ -117,7 +117,7 @@ class BootstrapMediumIT { void should_fail_if_invalid_json_input() { var in = new ByteArrayInputStream("}".getBytes()); - var exitCode = ScannerMain.run(in); + var exitCode = ScannerMain.run(in, System.out); assertThat(exitCode).isEqualTo(1); assertThat(logTester.getLogs(Level.ERROR)).hasSize(1); @@ -128,7 +128,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_key() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes())); + {"scannerProperties": [{"value": "aValueWithoutKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null key. Value='aValueWithoutKey'"); } @@ -136,7 +136,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": null}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -144,7 +144,7 @@ class BootstrapMediumIT { @Test void should_warn_if_not_provided_property_value() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring property with null value. Key='aKey'"); } @@ -152,7 +152,7 @@ class BootstrapMediumIT { @Test void should_warn_if_duplicate_property_keys() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"}, {"key": "aKey", "value": "aValue"}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Duplicated properties. Key='aKey'"); } @@ -160,7 +160,7 @@ class BootstrapMediumIT { @Test void should_warn_if_null_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes())); + {"scannerProperties": [{"key": "aKey", "value": "aValue"},]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -168,7 +168,7 @@ class BootstrapMediumIT { @Test void should_warn_if_empty_property() { ScannerMain.run(new ByteArrayInputStream(""" - {"scannerProperties": [{}]}""".getBytes())); + {"scannerProperties": [{}]}""".getBytes()), System.out); assertThat(logTester.logs(Level.WARN)).contains("Ignoring null or empty property"); } @@ -229,7 +229,7 @@ class BootstrapMediumIT { } private int runScannerEngine(ScannerProperties scannerProperties) { - return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes())); + return ScannerMain.run(new ByteArrayInputStream(scannerProperties.toJson().getBytes()), System.out); } static class ScannerProperties { diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index 04eff59c778..babeadf518e 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -1358,6 +1358,7 @@ class FileSystemMediumIT { .addRules(new XooRulesDefinition()) .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true"); if (setHiddenFileScanningExplicitly) { @@ -1385,6 +1386,7 @@ class FileSystemMediumIT { .addRules(new XooRulesDefinition()) .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "false") .execute(); @@ -1410,6 +1412,7 @@ class FileSystemMediumIT { .addRules(new XooRulesDefinition()) .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.exclusions", "**/*.ignore") .property("sonar.scanner.excludeHiddenFiles", "true") // hidden files are not scanned, so issues can't be raised on them regardless if the sensor wants to process them .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", String.valueOf(sensorHiddenFileProcessingEnabled)) @@ -1490,6 +1493,46 @@ class FileSystemMediumIT { assertHiddenFileScan(result, "moduleB/src/.xoo", true, true); } + @Test + void shouldScanAndAnalyzeAllHiddenFilesWithRespectToExclusions() throws IOException { + prepareHiddenFileProject(); + File projectDir = new File("test-resources/mediumtest/xoo/sample-with-hidden-files"); + + + AnalysisResult result = tester + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssuePerFile", null, "Issue Per File", "MAJOR", null, "xoo") + .newAnalysis(new File(projectDir, "sonar-project.properties")) + .property("sonar.scm.provider", "xoo") + .property("sonar.oneIssuePerFile.enableHiddenFileProcessing", "true") + .property("sonar.exclusions", "**/.nestedHidden/**,**/*.ignore") + .execute(); + + Set<String> excludedFiles = Set.of( + // sonar.exclusions + "xources/.hidden/.nestedHidden/.xoo", + "xources/.hidden/.nestedHidden/Class.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/.xoo.ignore", + "xources/.hidden/.nestedHidden/visibleInHiddenFolder/Class.xoo", + // scm ignore + "xources/nonHidden/.hiddenInVisibleFolder/.xoo"); + + for (Map.Entry<String, Boolean> pathToHiddenStatus : hiddenFileProjectExpectedHiddenStatus().entrySet()) { + String filePath = pathToHiddenStatus.getKey(); + boolean expectedIsHidden = pathToHiddenStatus.getValue(); + + if (excludedFiles.contains(filePath)) { + assertThat(result.inputFile(filePath)).isNull(); + } else { + assertHiddenFileScan(result, filePath, expectedIsHidden, true); + // we expect the sensor to process all non-excluded files, regardless of visibility + assertFileIssue(result, filePath, true); + } + } + assertThat(result.inputFiles()).hasSize(5); + } + private File createModuleWithSubdirectory(String moduleName, String subDirName) { File moduleBaseDir = new File(baseDir, moduleName); File srcDir = moduleBaseDir.toPath().resolve(subDirName).toFile(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 7e27a45e4cf..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,11 +21,14 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -50,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -71,6 +76,8 @@ public class ScannerMain { } catch (Throwable throwable) { handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -156,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index d73924c17e0..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -94,8 +94,8 @@ public class FileIndexer { // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); - // cached from directory file visitation - boolean isHidden = hiddenFilesProjectData.isMarkedAsHiddenFile(sourceFile, module); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java index e2d5f57acae..d779a054455 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -44,12 +44,16 @@ public class HiddenFilesProjectData { hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); } - public boolean isMarkedAsHiddenFile(Path file, DefaultInputModule module) { + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); - if (hiddenFilesPerModule == null) { - return false; + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); } - return hiddenFilesPerModule.contains(file); + return false; } public Path getCachedSonarUserHomePath() throws IOException { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 216f86c4f11..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -160,7 +160,11 @@ public class ProjectFilePreprocessor { processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -173,8 +177,13 @@ public class ProjectFilePreprocessor { InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } diff --git a/sonar-scanner-engine/src/main/resources/logback.xml b/sonar-scanner-engine/src/main/resources/logback.xml index ccd0dfe09b9..ddc2805f08a 100644 --- a/sonar-scanner-engine/src/main/resources/logback.xml +++ b/sonar-scanner-engine/src/main/resources/logback.xml @@ -1,17 +1,8 @@ <?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE configuration> -<!-- This logback configuration is used when the scanner engine is bootstrapped using the SonarScannerCli class. --> +<!-- This logback configuration is used when the scanner engine is bootstrapped using the ScannerMain class. --> <configuration scan="false"> - <import class="ch.qos.logback.core.ConsoleAppender"/> - - <appender name="STDOUT" class="ConsoleAppender"> - <encoder class="org.sonar.scanner.bootstrap.ScannerLogbackEncoder"/> - </appender> - - <root level="info"> - <appender-ref ref="STDOUT"/> - </root> <!-- BeanUtils generate too many DEBUG logs when sonar.verbose is set --> <logger name="org.apache.commons.beanutils.converters" level="WARN"/> @@ -31,4 +22,4 @@ <logger name="nl.altindag.ssl.util.CertificateUtils" level="INFO"/> -</configuration>
\ No newline at end of file +</configuration> diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java index ad3edf95428..d5a6e4ff843 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectDataTest.java @@ -73,10 +73,10 @@ public class HiddenFilesProjectDataTest { underTest.markAsHiddenFile(myFile2, inputModule); assertThat(underTest.hiddenFilesByModule).hasSize(1); - assertThat(underTest.isMarkedAsHiddenFile(myFile, inputModule)).isTrue(); - assertThat(underTest.isMarkedAsHiddenFile(myFile2, inputModule)).isTrue(); - assertThat(underTest.isMarkedAsHiddenFile(myFile, secondInputModule)).isFalse(); - assertThat(underTest.isMarkedAsHiddenFile(myFile2, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isFalse(); } @Test @@ -87,10 +87,10 @@ public class HiddenFilesProjectDataTest { underTest.markAsHiddenFile(myFile2, secondInputModule); assertThat(underTest.hiddenFilesByModule).hasSize(2); - assertThat(underTest.isMarkedAsHiddenFile(myFile, inputModule)).isTrue(); - assertThat(underTest.isMarkedAsHiddenFile(myFile2, inputModule)).isFalse(); - assertThat(underTest.isMarkedAsHiddenFile(myFile, secondInputModule)).isFalse(); - assertThat(underTest.isMarkedAsHiddenFile(myFile2, secondInputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, secondInputModule)).isTrue(); } @Test @@ -100,7 +100,7 @@ public class HiddenFilesProjectDataTest { underTest.markAsHiddenFile(myFile, inputModule); assertThat(underTest.hiddenFilesByModule).isNotEmpty(); - assertThat(underTest.isMarkedAsHiddenFile(notMarkedFile, secondInputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(notMarkedFile, secondInputModule)).isFalse(); } @Test @@ -117,6 +117,36 @@ public class HiddenFilesProjectDataTest { } @Test + public void shouldRemoveVisibilityAfterQuerying() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.hiddenFilesByModule.get(inputModule)).isEmpty(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isFalse(); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile2, inputModule)).isFalse(); + } + + @Test + public void shouldOnlyRemoveModuleIfAllFilesAreRemoved() { + Path myFile = Path.of("myFile"); + Path myFile2 = Path.of("myFile2"); + underTest.markAsHiddenFile(myFile, inputModule); + underTest.markAsHiddenFile(myFile2, inputModule); + + assertThat(underTest.hiddenFilesByModule).hasSize(1); + assertThat(underTest.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(myFile, inputModule)).isTrue(); + + assertThat(underTest.hiddenFilesByModule).isNotEmpty(); + } + + @Test public void shouldNotFailOnUserPathResolving() throws IOException { Path expectedPath = sonarUserHome.getPath().toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize(); assertThat(underTest.getCachedSonarUserHomePath()).isEqualTo(expectedPath); diff --git a/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/sonar-scanner-engine/test-resources/mediumtest/xoo/sample-with-hidden-files/xources/nonHidden/.hiddenInVisibleFolder/.xoo.ignore |