diff options
Diffstat (limited to 'sonar-scanner-engine/src/main/java/org/sonar')
| -rw-r--r-- | sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java | 103 | ||||
| -rw-r--r-- | sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java | 4 | ||||
| -rw-r--r-- | sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java (renamed from sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java) | 8 | ||||
| -rw-r--r-- | sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/BlameStrategy.java | 25 | ||||
| -rw-r--r-- | sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/DefaultBlameStrategy.java | 80 |
5 files changed, 206 insertions, 14 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java index 1ea3b4a4f39..69c02674fd3 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java @@ -22,14 +22,19 @@ package org.sonar.scm.git; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; @@ -44,8 +49,13 @@ import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; +import org.sonar.scm.git.blame.BlameResult; +import org.sonar.scm.git.blame.RepositoryBlameCommand; +import org.sonar.scm.git.strategy.BlameStrategy; +import org.sonar.scm.git.strategy.DefaultBlameStrategy.BlameAlgorithmEnum; import static java.util.Optional.ofNullable; +import static org.sonar.scm.git.strategy.DefaultBlameStrategy.BlameAlgorithmEnum.GIT_FILES_BLAME; public class CompositeBlameCommand extends BlameCommand { private static final Logger LOG = Loggers.get(CompositeBlameCommand.class); @@ -53,12 +63,16 @@ public class CompositeBlameCommand extends BlameCommand { private final AnalysisWarnings analysisWarnings; private final PathResolver pathResolver; private final JGitBlameCommand jgitCmd; - private final GitBlameCommand nativeCmd; + private final NativeGitBlameCommand nativeCmd; private boolean nativeGitEnabled = false; - public CompositeBlameCommand(AnalysisWarnings analysisWarnings, PathResolver pathResolver, JGitBlameCommand jgitCmd, GitBlameCommand nativeCmd) { + private final BlameStrategy blameStrategy; + + public CompositeBlameCommand(AnalysisWarnings analysisWarnings, PathResolver pathResolver, JGitBlameCommand jgitCmd, + NativeGitBlameCommand nativeCmd, BlameStrategy blameStrategy) { this.analysisWarnings = analysisWarnings; this.pathResolver = pathResolver; + this.blameStrategy = blameStrategy; this.jgitCmd = jgitCmd; this.nativeCmd = nativeCmd; } @@ -66,23 +80,48 @@ public class CompositeBlameCommand extends BlameCommand { @Override public void blame(BlameInput input, BlameOutput output) { File basedir = input.fileSystem().baseDir(); - try (Repository repo = JGitUtils.buildRepository(basedir.toPath()); Git git = Git.wrap(repo)) { + try (Repository repo = JGitUtils.buildRepository(basedir.toPath())) { + File gitBaseDir = repo.getWorkTree(); if (cloneIsInvalid(gitBaseDir)) { return; } Profiler profiler = Profiler.create(LOG); profiler.startDebug("Collecting committed files"); - Set<String> committedFiles = collectAllCommittedFiles(repo); + Map<String, InputFile> inputFileByGitRelativePath = getCommittedFilesToBlame(repo, gitBaseDir, input); profiler.stopDebug(); + + BlameAlgorithmEnum blameAlgorithmEnum = this.blameStrategy.getBlameAlgorithm(Runtime.getRuntime().availableProcessors(), inputFileByGitRelativePath.size()); + LOG.debug("Using {} strategy to blame files", blameAlgorithmEnum); + if (blameAlgorithmEnum == GIT_FILES_BLAME) { + blameWithFilesGitCommand(output, repo, inputFileByGitRelativePath); + } else { + blameWithNativeGitCommand(output, repo, inputFileByGitRelativePath, gitBaseDir); + } + } + } + + private Map<String, InputFile> getCommittedFilesToBlame(Repository repo, File gitBaseDir, BlameInput input) { + Set<String> committedFiles = collectAllCommittedFiles(repo); + Map<String, InputFile> inputFileByGitRelativePath = new HashMap<>(); + for (InputFile inputFile : input.filesToBlame()) { + String relative = pathResolver.relativePath(gitBaseDir, inputFile.file()); + if (relative == null || !committedFiles.contains(relative)) { + continue; + } + inputFileByGitRelativePath.put(relative, inputFile); + } + return inputFileByGitRelativePath; + } + + private void blameWithNativeGitCommand(BlameOutput output, Repository repo, Map<String, InputFile> inputFileByGitRelativePath, File gitBaseDir) { + try (Git git = Git.wrap(repo)) { nativeGitEnabled = nativeCmd.checkIfEnabled(); ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new GitThreadFactory()); - for (InputFile inputFile : input.filesToBlame()) { - String filename = pathResolver.relativePath(gitBaseDir, inputFile.file()); - if (filename == null || !committedFiles.contains(filename)) { - continue; - } + for (Map.Entry<String, InputFile> e : inputFileByGitRelativePath.entrySet()) { + InputFile inputFile = e.getValue(); + String filename = e.getKey(); // exceptions thrown by the blame method will be ignored executorService.submit(() -> blame(output, git, gitBaseDir, inputFile, filename)); } @@ -151,6 +190,29 @@ public class CompositeBlameCommand extends BlameCommand { } } + private static void blameWithFilesGitCommand(BlameOutput output, Repository repo, Map<String, InputFile> inputFileByGitRelativePath) { + RepositoryBlameCommand blameCommand = new RepositoryBlameCommand(repo) + .setTextComparator(RawTextComparator.WS_IGNORE_ALL) + .setMultithreading(true) + .setFilePaths(inputFileByGitRelativePath.keySet()); + try { + BlameResult blameResult = blameCommand.call(); + + for (Map.Entry<String, InputFile> e : inputFileByGitRelativePath.entrySet()) { + BlameResult.FileBlame fileBlameResult = blameResult.getFileBlameByPath().get(e.getKey()); + + if (fileBlameResult == null) { + LOG.debug("Unable to blame file {}.", e.getValue().filename()); + continue; + } + + saveBlameInformationForFileInTheOutput(fileBlameResult, e.getValue(), output); + } + } catch (GitAPIException e) { + LOG.warn("There was an issue when interacting with git repository: " + e.getMessage(), e); + } + } + private boolean cloneIsInvalid(File gitBaseDir) { if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/objects/info/alternates"))) { LOG.info("This git repository references another local repository which is not well supported. SCM information might be missing for some files. " @@ -166,4 +228,27 @@ public class CompositeBlameCommand extends BlameCommand { return false; } + + private static void saveBlameInformationForFileInTheOutput(BlameResult.FileBlame fileBlame, InputFile file, BlameOutput output) { + List<BlameLine> linesList = new ArrayList<>(); + for (int i = 0; i < fileBlame.lines(); i++) { + if (fileBlame.getAuthorEmails()[i] == null || fileBlame.getCommitHashes()[i] == null || fileBlame.getCommitDates()[i] == null) { + LOG.debug("Unable to blame file {}. No blame info at line {}. Is file committed? [Author: {} Source commit: {}]", file.filename()); + linesList.clear(); + break; + } + linesList.add(new BlameLine() + .date(fileBlame.getCommitDates()[i]) + .revision(fileBlame.getCommitHashes()[i]) + .author(fileBlame.getAuthorEmails()[i])); + } + if (!linesList.isEmpty()) { + if (linesList.size() == file.lines() - 1) { + // SONARPLUGINS-3097 Git does not report blame on last empty line + linesList.add(linesList.get(linesList.size() - 1)); + } + output.blameResult(file, linesList); + } + } + } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java index 70f7e2918d0..e06c95c7c48 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java @@ -22,6 +22,7 @@ package org.sonar.scm.git; import java.util.Arrays; import java.util.List; import org.eclipse.jgit.util.FS; +import org.sonar.scm.git.strategy.DefaultBlameStrategy; public final class GitScmSupport { private GitScmSupport() { @@ -33,8 +34,9 @@ public final class GitScmSupport { return Arrays.asList( JGitBlameCommand.class, CompositeBlameCommand.class, + NativeGitBlameCommand.class, + DefaultBlameStrategy.class, ProcessWrapperFactory.class, - GitBlameCommand.class, GitScmProvider.class, GitIgnoreCommand.class); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java index 9469ec15c38..0dd2e1620ef 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/NativeGitBlameCommand.java @@ -39,13 +39,13 @@ import org.springframework.beans.factory.annotation.Autowired; import static java.util.Collections.emptyList; import static org.sonar.api.utils.Preconditions.checkState; -public class GitBlameCommand { +public class NativeGitBlameCommand { protected static final String BLAME_COMMAND = "blame"; protected static final String GIT_DIR_FLAG = "--git-dir"; protected static final String GIT_DIR_ARGUMENT = "%s/.git"; protected static final String GIT_DIR_FORCE_FLAG = "-C"; - private static final Logger LOG = Loggers.get(GitBlameCommand.class); + private static final Logger LOG = Loggers.get(NativeGitBlameCommand.class); private static final Pattern EMAIL_PATTERN = Pattern.compile("<(.*?)>"); private static final String COMMITTER_TIME = "committer-time "; private static final String AUTHOR_MAIL = "author-mail "; @@ -64,12 +64,12 @@ public class GitBlameCommand { private String gitCommand; @Autowired - public GitBlameCommand(System2 system, ProcessWrapperFactory processWrapperFactory) { + public NativeGitBlameCommand(System2 system, ProcessWrapperFactory processWrapperFactory) { this.system = system; this.processWrapperFactory = processWrapperFactory; } - GitBlameCommand(String gitCommand, System2 system, ProcessWrapperFactory processWrapperFactory) { + NativeGitBlameCommand(String gitCommand, System2 system, ProcessWrapperFactory processWrapperFactory) { this.gitCommand = gitCommand; this.system = system; this.processWrapperFactory = processWrapperFactory; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/BlameStrategy.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/BlameStrategy.java new file mode 100644 index 00000000000..7092d5e5e05 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/BlameStrategy.java @@ -0,0 +1,25 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git.strategy; + +public interface BlameStrategy { + + DefaultBlameStrategy.BlameAlgorithmEnum getBlameAlgorithm(int availableProcessors, int numberOfFiles); +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/DefaultBlameStrategy.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/DefaultBlameStrategy.java new file mode 100644 index 00000000000..d52aed22d0d --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/strategy/DefaultBlameStrategy.java @@ -0,0 +1,80 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git.strategy; + +import org.sonar.api.config.Configuration; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +import static org.sonar.scm.git.strategy.DefaultBlameStrategy.BlameAlgorithmEnum.*; + +/** + * The blame strategy defines when the files-blame algorithm is used over jgit or native git algorithm. + * It has been found that the JGit/Git native algorithm performs better in certain circumstances, such as: + * - When we have many cores available for multi-threading + * - The number of files to be blame by the algorithm + * This two metrics are correlated: + * - The more threads available, the more it is favorable to use the JGit/Git native algorithm + * - The more files available, the more it is favorable to use the git-files-blame algorithm + */ +public class DefaultBlameStrategy implements BlameStrategy { + + private static final Logger LOG = Loggers.get(DefaultBlameStrategy.class); + private static final int FILES_GIT_BLAME_TRIGGER = 10; + public static final String PROP_SONAR_SCM_USE_BLAME_ALGORITHM = "sonar.scm.use.blame.algorithm"; + private final Configuration configuration; + + public DefaultBlameStrategy(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public BlameAlgorithmEnum getBlameAlgorithm(int availableProcessors, int numberOfFiles) { + BlameAlgorithmEnum forcedStrategy = configuration.get(PROP_SONAR_SCM_USE_BLAME_ALGORITHM) + .map(BlameAlgorithmEnum::valueOf) + .orElse(null); + + if (forcedStrategy != null) { + return forcedStrategy; + } + + if (availableProcessors == 0) { + LOG.warn("Available processors are 0. Falling back to native git blame"); + return GIT_NATIVE_BLAME; + } + if (numberOfFiles / availableProcessors > FILES_GIT_BLAME_TRIGGER) { + return GIT_FILES_BLAME; + } + + return GIT_NATIVE_BLAME; + } + + public enum BlameAlgorithmEnum { + /** + * Strategy using native git for the blame, or JGit on single file as a fallback + */ + GIT_NATIVE_BLAME, + /** + * Strategy using git-files-blame algorithm + */ + GIT_FILES_BLAME; + } + +} |