diff options
Diffstat (limited to 'sonar-scanner-engine/src/main/java')
7 files changed, 110 insertions, 21 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java new file mode 100644 index 00000000000..28e052cfa4e --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/JGitCleanupService.java @@ -0,0 +1,47 @@ +/* + * SonarQube + * Copyright (C) 2009-2025 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.lang.reflect.Method; +import org.eclipse.jgit.internal.util.CleanupService; + +/** + * Normally, JGit terminates with a shutdown hook. Since we also want to support running the Scanner Engine in the same JVM, this allows triggering shutdown manually. + */ +class JGitCleanupService implements AutoCloseable { + + private final Method shutDownMethod; + private final CleanupService cleanupService; + + public JGitCleanupService() { + cleanupService = new CleanupService(); + try { + shutDownMethod = CleanupService.class.getDeclaredMethod("shutDown"); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("Unable to find method 'shutDown' on JGit CleanupService", e); + } + shutDownMethod.setAccessible(true); + } + + @Override + public void close() throws Exception { + shutDownMethod.invoke(cleanupService); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java index 7e27a45e4cf..bd8d5b9b99c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerMain.java @@ -21,11 +21,14 @@ package org.sonar.scanner.bootstrap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.OutputStreamAppender; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -50,11 +53,13 @@ public class ScannerMain { private static final String SCANNER_APP_VERSION_KEY = "sonar.scanner.appVersion"; public static void main(String... args) { - System.exit(run(System.in)); + System.exit(run(System.in, System.out)); } - public static int run(InputStream in) { - try { + public static int run(InputStream in, OutputStream out) { + try (var ignored = new JGitCleanupService()) { + configureLogOutput(out); + LOG.info("Starting SonarScanner Engine..."); LOG.atInfo().log(ScannerMain::java); @@ -71,6 +76,8 @@ public class ScannerMain { } catch (Throwable throwable) { handleException(throwable); return 1; + } finally { + stopLogback(); } } @@ -156,6 +163,28 @@ public class ScannerMain { rootLogger.setLevel(Level.toLevel(verbose ? LEVEL_ROOT_VERBOSE : LEVEL_ROOT_DEFAULT)); } + private static void configureLogOutput(OutputStream out) { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + var encoder = new ScannerLogbackEncoder(); + encoder.setContext(loggerContext); + encoder.start(); + + var appender = new OutputStreamAppender<ILoggingEvent>(); + appender.setEncoder(encoder); + appender.setContext(loggerContext); + appender.setOutputStream(out); + appender.start(); + + var rootLogger = (Logger) LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME); + rootLogger.addAppender(appender); + rootLogger.setLevel(Level.toLevel(LEVEL_ROOT_DEFAULT)); + } + + private static void stopLogback() { + var loggerContext = (ch.qos.logback.classic.LoggerContext) LoggerFactory.getILoggerFactory(); + loggerContext.stop(); + } + private static class Input { @SerializedName("scannerProperties") private List<ScannerProperty> scannerProperties; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java index 1b2346f2b62..6b3418a8f6a 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/CliService.java @@ -76,19 +76,19 @@ public class CliService { this.projectExclusionFilters = projectExclusionFilters; } - public File generateManifestsZip(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { + public File generateManifestsArchive(DefaultInputModule module, File cliExecutable, DefaultConfiguration configuration) throws IOException, IllegalStateException { long startTime = system2.now(); boolean success = false; try { - String zipName = "dependency-files.zip"; - Path zipPath = module.getWorkDir().resolve(zipName); + String archiveName = "dependency-files.tar.xz"; + Path archivePath = module.getWorkDir().resolve(archiveName); List<String> args = new ArrayList<>(); args.add(cliExecutable.getAbsolutePath()); args.add("projects"); args.add("save-lockfiles"); - args.add("--zip"); - args.add("--zip-filename"); - args.add(zipPath.toAbsolutePath().toString()); + args.add("--xz"); + args.add("--xz-filename"); + args.add(archivePath.toAbsolutePath().toString()); args.add("--directory"); args.add(module.getBaseDir().toString()); args.add("--recursive"); @@ -117,9 +117,9 @@ public class CliService { Consumer<String> logConsumer = LOG.atLevel(Level.INFO)::log; processWrapperFactory.create(module.getWorkDir(), logConsumer, logConsumer, envProperties, args.toArray(new String[0])).execute(); - LOG.info("Generated manifests zip file: {}", zipName); + LOG.info("Generated manifests archive file: {}", archiveName); success = true; - return zipPath.toFile(); + return archivePath.toFile(); } finally { telemetryCache.put("scanner.sca.execution.cli.duration", String.valueOf(system2.now() - startTime)); telemetryCache.put("scanner.sca.execution.cli.success", String.valueOf(success)); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java index 52a99673e43..143e144c2dc 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/sca/ScaExecutor.java @@ -75,7 +75,7 @@ public class ScaExecutor { LOG.info("Collecting manifests for the dependency analysis..."); if (cliFile.exists()) { try { - File generatedZip = cliService.generateManifestsZip(root, cliFile, configuration); + File generatedZip = cliService.generateManifestsArchive(root, cliFile, configuration); LOG.debug("Zip ready for report: {}", generatedZip); reportPublisher.getWriter().writeScaFile(generatedZip); LOG.debug("Manifest zip written to report"); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index d73924c17e0..0961edbd985 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -94,8 +94,8 @@ public class FileIndexer { // This should be fast; language should be cached from preprocessing step Language language = langDetection.language(sourceFile, projectRelativePath); - // cached from directory file visitation - boolean isHidden = hiddenFilesProjectData.isMarkedAsHiddenFile(sourceFile, module); + // cached from directory file visitation, after querying the data is removed to reduce memory consumption + boolean isHidden = hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(sourceFile, module); DefaultIndexedFile indexedFile = new DefaultIndexedFile( sourceFile, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java index e2d5f57acae..d779a054455 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/HiddenFilesProjectData.java @@ -44,12 +44,16 @@ public class HiddenFilesProjectData { hiddenFilesByModule.computeIfAbsent(module, k -> new HashSet<>()).add(file); } - public boolean isMarkedAsHiddenFile(Path file, DefaultInputModule module) { + /** + * To alleviate additional strain on the memory, we remove the visibility information for <code>hiddenFilesByModule</code> mapdirectly after querying, + * as we don't need it afterward. + */ + public boolean getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(Path file, DefaultInputModule module) { Set<Path> hiddenFilesPerModule = hiddenFilesByModule.get(module); - if (hiddenFilesPerModule == null) { - return false; + if (hiddenFilesPerModule != null) { + return hiddenFilesPerModule.remove(file); } - return hiddenFilesPerModule.contains(file); + return false; } public Path getCachedSonarUserHomePath() throws IOException { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java index 216f86c4f11..3e7b655589c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ProjectFilePreprocessor.java @@ -160,7 +160,11 @@ public class ProjectFilePreprocessor { processedFiles.addAll(processDirectory(module, moduleConfiguration, moduleExclusionFilters, dirOrFile, type, exclusionCounter)); } else { filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand) - .ifPresent(processedFiles::add); + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(dirOrFile, module) + ); } } } catch (IOException e) { @@ -173,8 +177,13 @@ public class ProjectFilePreprocessor { InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException { List<Path> processedFiles = new ArrayList<>(); Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, - new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter, - ignoreCommand).ifPresent(processedFiles::add), module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); + new DirectoryFileVisitor(file -> filePreprocessor + .processFile(module, moduleExclusionFilters, file, type, exclusionCounter, ignoreCommand) + .ifPresentOrElse( + processedFiles::add, + // If the file is not processed, we don't need to save visibility data and can remove it + () -> hiddenFilesProjectData.getIsMarkedAsHiddenFileAndRemoveVisibilityInformation(file, module)), + module, moduleConfiguration, moduleExclusionFilters, inputModuleHierarchy, type, hiddenFilesProjectData)); return processedFiles; } |