From: Duarte Meneses Date: Fri, 21 Aug 2020 20:23:44 +0000 (-0500) Subject: SONAR-13792 Embed sonar-scm-git X-Git-Tag: 8.5.0.37579~111 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=87bb21e6bb8510620ecea231229964a2163203b3;p=sonarqube.git SONAR-13792 Embed sonar-scm-git --- diff --git a/build.gradle b/build.gradle index d307531b1d5..9d5d86773f4 100644 --- a/build.gradle +++ b/build.gradle @@ -182,7 +182,6 @@ subprojects { dependency 'org.sonarsource.javascript:sonar-javascript-plugin:6.3.0.12464' // bundled_plugin:javascript:SonarJS dependency 'org.sonarsource.php:sonar-php-plugin:3.5.0.5655' // bundled_plugin:php:sonar-php dependency 'org.sonarsource.python:sonar-python-plugin:2.13.0.7236' // bundled_plugin:python:sonar-python - dependency 'org.sonarsource.scm.git:sonar-scm-git-plugin:1.12.0.2034' // bundled_plugin:scmgit:sonar-scm-git dependency 'org.sonarsource.scm.svn:sonar-scm-svn-plugin:1.10.0.1917' // bundled_plugin:scmsvn:sonar-scm-svn dependency 'org.sonarsource.slang:sonar-go-plugin:1.6.0.719' // bundled_plugin:go:slang-enterprise dependency 'org.sonarsource.slang:sonar-kotlin-plugin:1.5.0.315' // bundled_plugin:kotlin:slang-enterprise diff --git a/sonar-application/bundled_plugins.gradle b/sonar-application/bundled_plugins.gradle index 50f146c15e9..934e4c07e6d 100644 --- a/sonar-application/bundled_plugins.gradle +++ b/sonar-application/bundled_plugins.gradle @@ -12,7 +12,6 @@ dependencies { bundledPlugin 'org.sonarsource.slang:sonar-go-plugin@jar' bundledPlugin "org.sonarsource.slang:sonar-kotlin-plugin@jar" bundledPlugin "org.sonarsource.slang:sonar-ruby-plugin@jar" - bundledPlugin 'org.sonarsource.scm.git:sonar-scm-git-plugin@jar' bundledPlugin 'org.sonarsource.scm.svn:sonar-scm-svn-plugin@jar' bundledPlugin "org.sonarsource.slang:sonar-scala-plugin@jar" bundledPlugin 'org.sonarsource.xml:sonar-xml-plugin@jar' diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java index 7e14856626d..ce29d00a358 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java @@ -131,6 +131,7 @@ import org.sonar.scanner.sensor.ProjectSensorContext; import org.sonar.scanner.sensor.ProjectSensorExtensionDictionnary; import org.sonar.scanner.sensor.ProjectSensorOptimizer; import org.sonar.scanner.sensor.ProjectSensorsExecutor; +import org.sonar.scm.git.GitScmSupport; import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH; import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter; @@ -301,6 +302,8 @@ public class ProjectScanContainer extends ComponentContainer { AnalysisObservers.class); + add(GitScmSupport.getClasses()); + addIfMissing(DefaultProjectSettingsLoader.class, ProjectSettingsLoader.class); addIfMissing(DefaultRulesLoader.class, RulesLoader.class); addIfMissing(DefaultActiveRulesLoader.class, ActiveRulesLoader.class); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java new file mode 100644 index 00000000000..1db66e65ba0 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java @@ -0,0 +1,116 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class ChangedLinesComputer { + private final Tracker tracker = new Tracker(); + + private final OutputStream receiver = new OutputStream() { + StringBuilder sb = new StringBuilder(); + + @Override + public void write(int b) { + sb.append((char) b); + if (b == '\n') { + tracker.parseLine(sb.toString()); + sb.setLength(0); + } + } + }; + + /** + * The OutputStream to pass to JGit's diff command. + */ + OutputStream receiver() { + return receiver; + } + + /** + * From a stream of unified diff lines emitted by Git for a single file, + * compute the line numbers that should be considered changed. + * Example input: + * 
+   * diff --git a/lao.txt b/lao.txt
+   * index 635ef2c..7f050f2 100644
+   * --- a/lao.txt
+   * +++ b/lao.txt
+   * @@ -1,7 +1,6 @@
+   * -The Way that can be told of is not the eternal Way;
+   * -The name that can be named is not the eternal name.
+   *  The Nameless is the origin of Heaven and Earth;
+   * -The Named is the mother of all things.
+   * +The named is the mother of all things.
+   * +
+   *  Therefore let there always be non-being,
+   *    so we may see their subtlety,
+   *  And let there always be being,
+   * @@ -9,3 +8,6 @@ And let there always be being,
+   *  The two are the same,
+   *  But after they are produced,
+   *    they have different names.
+   * +They both may be called deep and profound.
+   * +Deeper and more profound,
+   * +The door of all subtleties!names.
+   *
+ * See also: http://www.gnu.org/software/diffutils/manual/html_node/Example-Unified.html#Example-Unified + */ + Set changedLines() { + return tracker.changedLines(); + } + + private static class Tracker { + + private static final Pattern START_LINE_IN_TARGET = Pattern.compile(" \\+(\\d+)"); + + private final Set changedLines = new HashSet<>(); + + private boolean foundStart = false; + private int lineNumInTarget; + + private void parseLine(String line) { + if (line.startsWith("@@ ")) { + Matcher matcher = START_LINE_IN_TARGET.matcher(line); + if (!matcher.find()) { + throw new IllegalStateException("Invalid block header on line " + line); + } + foundStart = true; + lineNumInTarget = Integer.parseInt(matcher.group(1)); + } else if (foundStart) { + char firstChar = line.charAt(0); + if (firstChar == ' ') { + lineNumInTarget++; + } else if (firstChar == '+') { + changedLines.add(lineNumInTarget); + lineNumInTarget++; + } + } + } + + Set changedLines() { + return changedLines; + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java new file mode 100644 index 00000000000..cfe31a1ed67 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.nio.file.Path; +import org.sonar.api.batch.scm.IgnoreCommand; +import org.sonar.api.scanner.ScannerSide; + +import static java.util.Objects.requireNonNull; + +@ScannerSide +public class GitIgnoreCommand implements IgnoreCommand { + + private IncludedFilesRepository includedFilesRepository; + + @Override + public void init(Path baseDir) { + try { + this.includedFilesRepository = new IncludedFilesRepository(baseDir); + } catch (IOException e) { + throw new IllegalStateException("I/O error while indexing ignored files.", e); + } + } + + @Override + public boolean isIgnored(Path absolutePath) { + return !requireNonNull(includedFilesRepository, "Call init first").contains(absolutePath); + } + + @Override + public void clean() { + this.includedFilesRepository = null; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java new file mode 100644 index 00000000000..19b81e1642a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java @@ -0,0 +1,376 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.CheckForNull; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffAlgorithm; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.RawTextComparator; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryBuilder; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.treewalk.AbstractTreeIterator; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.sonar.api.batch.scm.BlameCommand; +import org.sonar.api.batch.scm.ScmProvider; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.api.utils.MessageException; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +public class GitScmProvider extends ScmProvider { + + private static final Logger LOG = Loggers.get(GitScmProvider.class); + + private final JGitBlameCommand jgitBlameCommand; + private final AnalysisWarnings analysisWarnings; + private final GitIgnoreCommand gitIgnoreCommand; + private final System2 system2; + + public GitScmProvider(JGitBlameCommand jgitBlameCommand, AnalysisWarnings analysisWarnings, GitIgnoreCommand gitIgnoreCommand, System2 system2) { + this.jgitBlameCommand = jgitBlameCommand; + this.analysisWarnings = analysisWarnings; + this.gitIgnoreCommand = gitIgnoreCommand; + this.system2 = system2; + } + + @Override + public GitIgnoreCommand ignoreCommand() { + return gitIgnoreCommand; + } + + @Override + public String key() { + return "git"; + } + + @Override + public boolean supports(File baseDir) { + RepositoryBuilder builder = new RepositoryBuilder().findGitDir(baseDir); + return builder.getGitDir() != null; + } + + @Override + public BlameCommand blameCommand() { + return this.jgitBlameCommand; + } + + @CheckForNull + @Override + public Set branchChangedFiles(String targetBranchName, Path rootBaseDir) { + try (Repository repo = buildRepo(rootBaseDir)) { + Ref targetRef = resolveTargetRef(targetBranchName, repo); + if (targetRef == null) { + analysisWarnings.addUnique(String.format("Could not find ref '%s' in refs/heads, refs/remotes/upstream or refs/remotes/origin. " + + "You may see unexpected issues and changes. " + + "Please make sure to fetch this ref before pull request analysis.", targetBranchName)); + return null; + } + + if (isDiffAlgoInvalid(repo.getConfig())) { + LOG.warn("The diff algorithm configured in git is not supported. " + + "No information regarding changes in the branch will be collected, which can lead to unexpected results."); + return null; + } + + Optional mergeBaseCommit = findMergeBase(repo, targetRef); + if (!mergeBaseCommit.isPresent()) { + LOG.warn("No merge base found between HEAD and " + targetRef.getName()); + return null; + } + AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit.get()); + + // we compare a commit with HEAD, so no point ignoring line endings (it will be whatever is committed) + try (Git git = newGit(repo)) { + List diffEntries = git.diff() + .setShowNameAndStatusOnly(true) + .setOldTree(mergeBaseTree) + .setNewTree(prepareNewTree(repo)) + .call(); + + return diffEntries.stream() + .filter(diffEntry -> diffEntry.getChangeType() == DiffEntry.ChangeType.ADD || diffEntry.getChangeType() == DiffEntry.ChangeType.MODIFY) + .map(diffEntry -> repo.getWorkTree().toPath().resolve(diffEntry.getNewPath())) + .collect(Collectors.toSet()); + } + } catch (IOException | GitAPIException e) { + LOG.warn(e.getMessage(), e); + } + return null; + } + + @CheckForNull + @Override + public Map> branchChangedLines(String targetBranchName, Path projectBaseDir, Set changedFiles) { + try (Repository repo = buildRepo(projectBaseDir)) { + Ref targetRef = resolveTargetRef(targetBranchName, repo); + if (targetRef == null) { + analysisWarnings.addUnique(String.format("Could not find ref '%s' in refs/heads, refs/remotes/upstream or refs/remotes/origin. " + + "You may see unexpected issues and changes. " + + "Please make sure to fetch this ref before pull request analysis.", targetBranchName)); + return null; + } + + if (isDiffAlgoInvalid(repo.getConfig())) { + // we already print a warning when branchChangedFiles is called + return null; + } + + // force ignore different line endings when comparing a commit with the workspace + repo.getConfig().setBoolean("core", null, "autocrlf", true); + + Optional mergeBaseCommit = findMergeBase(repo, targetRef); + if (!mergeBaseCommit.isPresent()) { + LOG.warn("No merge base found between HEAD and " + targetRef.getName()); + return null; + } + + Map> changedLines = new HashMap<>(); + Path repoRootDir = repo.getDirectory().toPath().getParent(); + + for (Path path : changedFiles) { + collectChangedLines(repo, mergeBaseCommit.get(), changedLines, repoRootDir, path); + } + return changedLines; + } catch (Exception e) { + LOG.warn("Failed to get changed lines from git", e); + } + return null; + } + + private void collectChangedLines(Repository repo, RevCommit mergeBaseCommit, Map> changedLines, Path repoRootDir, Path changedFile) { + ChangedLinesComputer computer = new ChangedLinesComputer(); + + try (DiffFormatter diffFmt = new DiffFormatter(new BufferedOutputStream(computer.receiver()))) { + // copied from DiffCommand so that we can use a custom DiffFormatter which ignores white spaces. + diffFmt.setRepository(repo); + diffFmt.setProgressMonitor(NullProgressMonitor.INSTANCE); + diffFmt.setDiffComparator(RawTextComparator.WS_IGNORE_ALL); + diffFmt.setPathFilter(PathFilter.create(toGitPath(repoRootDir.relativize(changedFile).toString()))); + + AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit); + List diffEntries = diffFmt.scan(mergeBaseTree, new FileTreeIterator(repo)); + diffFmt.format(diffEntries); + diffFmt.flush(); + diffEntries.stream() + .filter(diffEntry -> diffEntry.getChangeType() == DiffEntry.ChangeType.ADD || diffEntry.getChangeType() == DiffEntry.ChangeType.MODIFY) + .findAny() + .ifPresent(diffEntry -> changedLines.put(changedFile, computer.changedLines())); + } catch (Exception e) { + LOG.warn("Failed to get changed lines from git for file " + changedFile, e); + } + } + + @Override + @CheckForNull + public Instant forkDate(String referenceBranchName, Path projectBaseDir) { + try (Repository repo = buildRepo(projectBaseDir)) { + Ref targetRef = resolveTargetRef(referenceBranchName, repo); + if (targetRef == null) { + LOG.warn("Branch '{}' not found in git", referenceBranchName); + return null; + } + + if (isDiffAlgoInvalid(repo.getConfig())) { + LOG.warn("The diff algorithm configured in git is not supported. " + + "No information regarding changes in the branch will be collected, which can lead to unexpected results."); + return null; + } + + Optional mergeBaseCommit = findMergeBase(repo, targetRef); + if (!mergeBaseCommit.isPresent()) { + LOG.warn("No fork point found between HEAD and " + targetRef.getName()); + return null; + } + + return Instant.ofEpochSecond(mergeBaseCommit.get().getCommitTime()); + } catch (Exception e) { + LOG.warn("Failed to find fork point with git", e); + } + + return null; + } + + private static String toGitPath(String path) { + return path.replaceAll(Pattern.quote(File.separator), "/"); + } + + @CheckForNull + private Ref resolveTargetRef(String targetBranchName, Repository repo) throws IOException { + String localRef = "refs/heads/" + targetBranchName; + String remoteRef = "refs/remotes/origin/" + targetBranchName; + String upstreamRef = "refs/remotes/upstream/" + targetBranchName; + + Ref targetRef; + // Because circle ci destroys the local reference to master, try to load remote ref first. + // https://discuss.circleci.com/t/git-checkout-of-a-branch-destroys-local-reference-to-master/23781 + if (runningOnCircleCI()) { + targetRef = getFirstExistingRef(repo, remoteRef, localRef, upstreamRef); + } else { + targetRef = getFirstExistingRef(repo, localRef, remoteRef, upstreamRef); + } + + if (targetRef == null) { + LOG.warn("Could not find ref: {} in refs/heads, refs/remotes/upstream or refs/remotes/origin", targetBranchName); + } + + return targetRef; + } + + @CheckForNull + private static Ref getFirstExistingRef(Repository repo, String... refs) throws IOException { + Ref targetRef = null; + for (String ref : refs) { + targetRef = repo.exactRef(ref); + if (targetRef != null) { + break; + } + } + return targetRef; + } + + private boolean runningOnCircleCI() { + return "true".equals(system2.envVariable("CIRCLECI")); + } + + @Override + public Path relativePathFromScmRoot(Path path) { + RepositoryBuilder builder = getVerifiedRepositoryBuilder(path); + return builder.getGitDir().toPath().getParent().relativize(path); + } + + @Override + @CheckForNull + public String revisionId(Path path) { + RepositoryBuilder builder = getVerifiedRepositoryBuilder(path); + try { + Ref head = getHead(builder.build()); + if (head == null || head.getObjectId() == null) { + // can happen on fresh, empty repos + return null; + } + return head.getObjectId().getName(); + } catch (IOException e) { + throw new IllegalStateException("I/O error while getting revision ID for path: " + path, e); + } + } + + private static boolean isDiffAlgoInvalid(Config cfg) { + try { + DiffAlgorithm.getAlgorithm(cfg.getEnum( + ConfigConstants.CONFIG_DIFF_SECTION, null, + ConfigConstants.CONFIG_KEY_ALGORITHM, + DiffAlgorithm.SupportedAlgorithm.HISTOGRAM)); + return false; + } catch (IllegalArgumentException e) { + return true; + } + } + + private static AbstractTreeIterator prepareNewTree(Repository repo) throws IOException { + CanonicalTreeParser treeParser = new CanonicalTreeParser(); + try (ObjectReader objectReader = repo.newObjectReader()) { + Ref head = getHead(repo); + if (head == null) { + throw new IOException("HEAD reference not found"); + } + treeParser.reset(objectReader, repo.parseCommit(head.getObjectId()).getTree()); + } + return treeParser; + } + + @CheckForNull + private static Ref getHead(Repository repo) throws IOException { + return repo.exactRef("HEAD"); + } + + private static Optional findMergeBase(Repository repo, Ref targetRef) throws IOException { + try (RevWalk walk = new RevWalk(repo)) { + Ref head = getHead(repo); + if (head == null) { + throw new IOException("HEAD reference not found"); + } + + walk.markStart(walk.parseCommit(targetRef.getObjectId())); + walk.markStart(walk.parseCommit(head.getObjectId())); + walk.setRevFilter(RevFilter.MERGE_BASE); + RevCommit next = walk.next(); + if (next == null) { + return Optional.empty(); + } + RevCommit base = walk.parseCommit(next); + walk.dispose(); + LOG.debug("Merge base sha1: {}", base.getName()); + return Optional.of(base); + } + } + + AbstractTreeIterator prepareTreeParser(Repository repo, RevCommit commit) throws IOException { + CanonicalTreeParser treeParser = new CanonicalTreeParser(); + try (ObjectReader objectReader = repo.newObjectReader()) { + treeParser.reset(objectReader, commit.getTree()); + } + return treeParser; + } + + Git newGit(Repository repo) { + return new Git(repo); + } + + Repository buildRepo(Path basedir) throws IOException { + return getVerifiedRepositoryBuilder(basedir).build(); + } + + static RepositoryBuilder getVerifiedRepositoryBuilder(Path basedir) { + RepositoryBuilder builder = new RepositoryBuilder() + .findGitDir(basedir.toFile()) + .setMustExist(true); + + if (builder.getGitDir() == null) { + throw MessageException.of("Not inside a Git work tree: " + basedir); + } + return builder; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java new file mode 100644 index 00000000000..b5a845edc9a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java @@ -0,0 +1,34 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.util.Arrays; +import java.util.List; +import org.eclipse.jgit.util.FS; + +public final class GitScmSupport { + public static List> getClasses() { + FS.FileStoreAttributes.setBackground(true); + return Arrays.asList( + JGitBlameCommand.class, + GitScmProvider.class, + GitIgnoreCommand.class); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java new file mode 100644 index 00000000000..cbca28cc628 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory; +import java.util.concurrent.ForkJoinWorkerThread; + +public class GitThreadFactory implements ForkJoinWorkerThreadFactory { + private static final String NAME_PREFIX = "git-scm-"; + private int i = 0; + + @Override + public ForkJoinWorkerThread newThread(ForkJoinPool pool) { + ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool); + thread.setName(NAME_PREFIX + i++); + return thread; + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java new file mode 100644 index 00000000000..2d30df40513 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java @@ -0,0 +1,72 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.WorkingTreeIterator; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +public class IncludedFilesRepository { + + private static final Logger LOG = Loggers.get(IncludedFilesRepository.class); + private final Set includedFiles = new HashSet<>(); + + public IncludedFilesRepository(Path baseDir) throws IOException { + indexFiles(baseDir); + LOG.debug("{} non excluded files in this Git repository", includedFiles.size()); + } + + public boolean contains(Path absolutePath) { + return includedFiles.contains(absolutePath); + } + + private void indexFiles(Path baseDir) throws IOException { + try (Repository repo = JGitUtils.buildRepository(baseDir)) { + Path workTreeRoot = repo.getWorkTree().toPath(); + FileTreeIterator workingTreeIt = new FileTreeIterator(repo); + try (TreeWalk treeWalk = new TreeWalk(repo)) { + treeWalk.setRecursive(true); + if (!baseDir.equals(workTreeRoot)) { + Path relativeBaseDir = workTreeRoot.relativize(baseDir); + treeWalk.setFilter(PathFilterGroup.createFromStrings(relativeBaseDir.toString().replace('\\', '/'))); + } + treeWalk.addTree(workingTreeIt); + while (treeWalk.next()) { + + WorkingTreeIterator workingTreeIterator = treeWalk + .getTree(0, WorkingTreeIterator.class); + + if (!workingTreeIterator.isEntryIgnored()) { + includedFiles.add(workTreeRoot.resolve(treeWalk.getPathString())); + } + } + } + } + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java new file mode 100644 index 00000000000..601a65413e5 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java @@ -0,0 +1,129 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.blame.BlameResult; +import org.eclipse.jgit.diff.RawTextComparator; +import org.eclipse.jgit.lib.Repository; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.scm.BlameCommand; +import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +public class JGitBlameCommand extends BlameCommand { + + private static final Logger LOG = Loggers.get(JGitBlameCommand.class); + + private final PathResolver pathResolver; + private final AnalysisWarnings analysisWarnings; + + public JGitBlameCommand(PathResolver pathResolver, AnalysisWarnings analysisWarnings) { + this.pathResolver = pathResolver; + this.analysisWarnings = analysisWarnings; + } + + @Override + public void blame(BlameInput input, BlameOutput output) { + File basedir = input.fileSystem().baseDir(); + try (Repository repo = JGitUtils.buildRepository(basedir.toPath()); Git git = Git.wrap(repo)) { + File gitBaseDir = repo.getWorkTree(); + + if (cloneIsInvalid(gitBaseDir)) { + return; + } + + Stream stream = StreamSupport.stream(input.filesToBlame().spliterator(), true); + ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), new GitThreadFactory(), null, false); + forkJoinPool.submit(() -> stream.forEach(inputFile -> blame(output, git, gitBaseDir, inputFile))); + try { + forkJoinPool.shutdown(); + forkJoinPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.info("Git blame interrupted"); + } + } + } + + private boolean cloneIsInvalid(File gitBaseDir) { + if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/objects/info/alternates"))) { + LOG.info("This git repository references another local repository which is not well supported. SCM information might be missing for some files. " + + "You can avoid borrow objects from another local repository by not using --reference or --shared when cloning it."); + } + + if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/shallow"))) { + LOG.warn("Shallow clone detected, no blame information will be provided. " + + "You can convert to non-shallow with 'git fetch --unshallow'."); + analysisWarnings.addUnique("Shallow clone detected during the analysis. " + + "Some files will miss SCM information. This will affect features like auto-assignment of issues. " + + "Please configure your build to disable shallow clone."); + return true; + } + + return false; + } + + private void blame(BlameOutput output, Git git, File gitBaseDir, InputFile inputFile) { + String filename = pathResolver.relativePath(gitBaseDir, inputFile.file()); + LOG.debug("Blame file {}", filename); + BlameResult blameResult; + try { + blameResult = git.blame() + // Equivalent to -w command line option + .setTextComparator(RawTextComparator.WS_IGNORE_ALL) + .setFilePath(filename).call(); + } catch (Exception e) { + throw new IllegalStateException("Unable to blame file " + inputFile.relativePath(), e); + } + List lines = new ArrayList<>(); + if (blameResult == null) { + LOG.debug("Unable to blame file {}. It is probably a symlink.", inputFile.relativePath()); + return; + } + for (int i = 0; i < blameResult.getResultContents().size(); i++) { + if (blameResult.getSourceAuthor(i) == null || blameResult.getSourceCommit(i) == null) { + LOG.debug("Unable to blame file {}. No blame info at line {}. Is file committed? [Author: {} Source commit: {}]", inputFile.relativePath(), i + 1, + blameResult.getSourceAuthor(i), blameResult.getSourceCommit(i)); + return; + } + lines.add(new BlameLine() + .date(blameResult.getSourceCommitter(i).getWhen()) + .revision(blameResult.getSourceCommit(i).getName()) + .author(blameResult.getSourceAuthor(i).getEmailAddress())); + } + if (lines.size() == inputFile.lines() - 1) { + // SONARPLUGINS-3097 Git do not report blame on last empty line + lines.add(lines.get(lines.size() - 1)); + } + output.blameResult(inputFile, lines); + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java new file mode 100644 index 00000000000..86d8792b53a --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java @@ -0,0 +1,44 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.nio.file.Path; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Repository; + +public class JGitUtils { + + private JGitUtils() { + } + + public static Repository buildRepository(Path basedir) { + try { + Repository repo = GitScmProvider.getVerifiedRepositoryBuilder(basedir).build(); + try (ObjectReader objReader = repo.getObjectDatabase().newReader()) { + // SONARSCGIT-2 Force initialization of shallow commits to avoid later concurrent modification issue + objReader.getShallowCommits(); + return repo; + } + } catch (IOException e) { + throw new IllegalStateException("Unable to open Git repository", e); + } + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java new file mode 100644 index 00000000000..aa91667ae34 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.scm.git; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java index 1f90a946b6c..712687ed105 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java @@ -751,6 +751,7 @@ public class FileSystemMediumTest { .newAnalysis(new File(projectDir, "sonar-project.properties")) .property("sonar.exclusions", "**/*.xoo.measures,**/*.xoo.scm") .property("sonar.test.exclusions", "**/*.xoo.measures,**/*.xoo.scm") + .property("sonar.scm.exclusions.disabled", "true") .execute(); assertThat(result.inputFiles()).hasSize(3); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedLinesComputerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedLinesComputerTest.java new file mode 100644 index 00000000000..141b011a79a --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/ChangedLinesComputerTest.java @@ -0,0 +1,154 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ChangedLinesComputerTest { + @Rule + public ExpectedException exception = ExpectedException.none(); + private final ChangedLinesComputer underTest = new ChangedLinesComputer(); + + @Test + public void do_not_count_deleted_line() throws IOException { + String example = "diff --git a/file-b1.xoo b/file-b1.xoo\n" + + "index 0000000..c2a9048\n" + + "--- a/foo\n" + + "+++ b/bar\n" + + "@@ -1 +0,0 @@\n" + + "-deleted line\n"; + + printDiff(example); + assertThat(underTest.changedLines()).isEmpty(); + } + + @Test + public void count_single_added_line() throws IOException { + String example = "diff --git a/file-b1.xoo b/file-b1.xoo\n" + + "index 0000000..c2a9048\n" + + "--- a/foo\n" + + "+++ b/bar\n" + + "@@ -0,0 +1 @@\n" + + "+added line\n"; + + printDiff(example); + assertThat(underTest.changedLines()).containsExactly(1); + } + + @Test + public void count_multiple_added_lines() throws IOException { + String example = "diff --git a/file-b1.xoo b/file-b1.xoo\n" + + "index 0000000..c2a9048\n" + + "--- a/foo\n" + + "+++ b/bar\n" + + "@@ -1 +1,3 @@\n" + + " unchanged line\n" + + "+added line 1\n" + + "+added line 2\n"; + + printDiff(example); + assertThat(underTest.changedLines()).containsExactly(2, 3); + } + + @Test + public void compute_from_multiple_hunks() throws IOException { + String example = "diff --git a/lao b/lao\n" + + "index 635ef2c..5af88a8 100644\n" + + "--- a/lao\n" + + "+++ b/lao\n" + + "@@ -1,7 +1,6 @@\n" + + "-The Way that can be told of is not the eternal Way;\n" + + "-The name that can be named is not the eternal name.\n" + + " The Nameless is the origin of Heaven and Earth;\n" + + "-The Named is the mother of all things.\n" + + "+The named is the mother of all things.\n" + + "+\n" + + " Therefore let there always be non-being,\n" + + " so we may see their subtlety,\n" + + " And let there always be being,\n" + + "@@ -9,3 +8,6 @@ And let there always be being,\n" + + " The two are the same,\n" + + " But after they are produced,\n" + + " they have different names.\n" + + "+They both may be called deep and profound.\n" + + "+Deeper and more profound,\n" + + "+The door of all subtleties!\n"; + printDiff(example); + assertThat(underTest.changedLines()).containsExactly(2, 3, 11, 12, 13); + } + + @Test + public void compute_from_multiple_hunks_with_extra_header_lines() throws IOException { + String example = "diff --git a/lao b/lao\n" + + "new file mode 100644\n" + + "whatever " + + "other " + + "surprise header lines git might throw at us...\n" + + "index 635ef2c..5af88a8 100644\n" + + "--- a/lao\n" + + "+++ b/lao\n" + + "@@ -1,7 +1,6 @@\n" + + "-The Way that can be told of is not the eternal Way;\n" + + "-The name that can be named is not the eternal name.\n" + + " The Nameless is the origin of Heaven and Earth;\n" + + "-The Named is the mother of all things.\n" + + "+The named is the mother of all things.\n" + + "+\n" + + " Therefore let there always be non-being,\n" + + " so we may see their subtlety,\n" + + " And let there always be being,\n" + + "@@ -9,3 +8,6 @@ And let there always be being,\n" + + " The two are the same,\n" + + " But after they are produced,\n" + + " they have different names.\n" + + "+They both may be called deep and profound.\n" + + "+Deeper and more profound,\n" + + "+The door of all subtleties!\n"; + printDiff(example); + assertThat(underTest.changedLines()).containsExactly(2, 3, 11, 12, 13); + } + + @Test + public void throw_exception_invalid_start_line_format() throws IOException { + String example = "diff --git a/file-b1.xoo b/file-b1.xoo\n" + + "index 0000000..c2a9048\n" + + "--- a/foo\n" + + "+++ b/bar\n" + + "@@ -1 +x1,3 @@\n" + + " unchanged line\n" + + "+added line 1\n" + + "+added line 2\n"; + + exception.expect(IllegalStateException.class); + printDiff(example); + } + + private void printDiff(String unifiedDiff) throws IOException { + try (OutputStreamWriter writer = new OutputStreamWriter(underTest.receiver())) { + writer.write(unifiedDiff); + } + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java new file mode 100644 index 00000000000..c4ea7ae0a5e --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java @@ -0,0 +1,140 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import org.eclipse.jgit.api.Git; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.scm.git.Utils.javaUnzip; + +public class GitIgnoreCommandTest { + + @Rule + public LogTester logTester = new LogTester(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void ignored_files_should_match_files_ignored_by_git() throws IOException { + Path projectDir = temp.newFolder().toPath(); + javaUnzip("ignore-git.zip", projectDir.toFile()); + + Path baseDir = projectDir.resolve("ignore-git"); + GitIgnoreCommand underTest = new GitIgnoreCommand(); + underTest.init(baseDir); + + assertThat(underTest.isIgnored(baseDir.resolve(".gitignore"))).isFalse(); + assertThat(underTest.isIgnored(baseDir.resolve("pom.xml"))).isFalse(); + assertThat(underTest.isIgnored(baseDir.resolve("src/main/java/org/dummy/.gitignore"))).isFalse(); + assertThat(underTest.isIgnored(baseDir.resolve("src/main/java/org/dummy/AnotherDummy.java"))).isFalse(); + assertThat(underTest.isIgnored(baseDir.resolve("src/test/java/org/dummy/AnotherDummyTest.java"))).isFalse(); + + assertThat(underTest.isIgnored(baseDir.resolve("src/main/java/org/dummy/Dummy.java"))).isTrue(); + assertThat(underTest.isIgnored(baseDir.resolve("target"))).isTrue(); + } + + @Test + public void test_pattern_on_deep_repo() throws Exception { + Path projectDir = temp.newFolder().toPath(); + Git.init().setDirectory(projectDir.toFile()).call(); + + Files.write(projectDir.resolve(".gitignore"), Arrays.asList("**/*.java"), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); + int child_folders_per_folder = 2; + int folder_depth = 10; + createDeepFolderStructure(projectDir, child_folders_per_folder, 0, folder_depth); + + logTester.setLevel(LoggerLevel.DEBUG); + + GitIgnoreCommand underTest = new GitIgnoreCommand(); + underTest.init(projectDir); + + assertThat(underTest + .isIgnored(projectDir.resolve("folder_0_0/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.java"))) + .isTrue(); + assertThat(underTest + .isIgnored(projectDir.resolve("folder_0_0/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php"))) + .isFalse(); + + int expectedIncludedFiles = (int) Math.pow(child_folders_per_folder, folder_depth) + 1; // The .gitignore file is indexed + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(expectedIncludedFiles + " non excluded files in this Git repository"); + } + + @Test + public void dont_index_files_outside_basedir() throws Exception { + Path repoRoot = temp.newFolder().toPath(); + Git.init().setDirectory(repoRoot.toFile()).call(); + + Files.write(repoRoot.resolve(".gitignore"), Arrays.asList("**/*.java"), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); + int child_folders_per_folder = 2; + int folder_depth = 10; + createDeepFolderStructure(repoRoot, child_folders_per_folder, 0, folder_depth); + + logTester.setLevel(LoggerLevel.DEBUG); + + GitIgnoreCommand underTest = new GitIgnoreCommand(); + // Define project baseDir as folder_0_0 so that folder_0_1 is excluded + Path projectBasedir = repoRoot.resolve("folder_0_0"); + underTest.init(projectBasedir); + + assertThat(underTest + .isIgnored(projectBasedir.resolve("folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php"))) + .isFalse(); + assertThat(underTest + .isIgnored(repoRoot.resolve("folder_0_1/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php"))) + .isTrue(); + + int expectedIncludedFiles = (int) Math.pow(child_folders_per_folder, folder_depth - 1); + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(expectedIncludedFiles + " non excluded files in this Git repository"); + } + + private void createDeepFolderStructure(Path current, int childCount, int currentDepth, int maxDepth) throws IOException { + if (currentDepth >= maxDepth) { + Path javaFile = current.resolve("Foo.java"); + Path phpFile = current.resolve("Foo.php"); + if (!Files.exists(phpFile)) { + Files.createFile(phpFile); + } + if (!Files.exists(javaFile)) { + Files.createFile(javaFile); + } + return; + } + for (int j = 0; j < childCount; j++) { + Path newPath = current.resolve("folder_" + currentDepth + "_" + j); + if (!Files.exists(newPath)) { + Files.createDirectory(newPath); + } + createDeepFolderStructure(newPath, childCount, currentDepth + 1, maxDepth); + } + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java new file mode 100644 index 00000000000..7f62b8912fa --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmProviderTest.java @@ -0,0 +1,797 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.jgit.api.DiffCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.RefDatabase; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.treewalk.AbstractTreeIterator; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.MessageException; +import org.sonar.api.utils.System2; + +import static java.util.Collections.emptySet; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.data.MapEntry.entry; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.sonar.scm.git.Utils.javaUnzip; + +public class GitScmProviderTest { + + // Sample content for unified diffs + // http://www.gnu.org/software/diffutils/manual/html_node/Example-Unified.html#Example-Unified + private static final String CONTENT_LAO = "The Way that can be told of is not the eternal Way;\n" + + "The name that can be named is not the eternal name.\n" + + "The Nameless is the origin of Heaven and Earth;\n" + + "The Named is the mother of all things.\n" + + "Therefore let there always be non-being,\n" + + " so we may see their subtlety,\n" + + "And let there always be being,\n" + + " so we may see their outcome.\n" + + "The two are the same,\n" + + "But after they are produced,\n" + + " they have different names.\n"; + + private static final String CONTENT_TZU = "The Nameless is the origin of Heaven and Earth;\n" + + "The named is the mother of all things.\n" + + "\n" + + "Therefore let there always be non-being,\n" + + " so we may see their subtlety,\n" + + "And let there always be being,\n" + + " so we may see their outcome.\n" + + "The two are the same,\n" + + "But after they are produced,\n" + + " they have different names.\n" + + "They both may be called deep and profound.\n" + + "Deeper and more profound,\n" + + "The door of all subtleties!"; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private GitIgnoreCommand gitIgnoreCommand = mock(GitIgnoreCommand.class); + private static final Random random = new Random(); + private static final System2 system2 = mock(System2.class); + + private Path worktree; + private Git git; + private final AnalysisWarnings analysisWarnings = mock(AnalysisWarnings.class); + + @Before + public void before() throws IOException, GitAPIException { + worktree = temp.newFolder().toPath(); + Repository repo = FileRepositoryBuilder.create(worktree.resolve(".git").toFile()); + repo.create(); + + git = new Git(repo); + + createAndCommitFile("file-in-first-commit.xoo"); + } + + @Test + public void sanityCheck() { + assertThat(newGitScmProvider().key()).isEqualTo("git"); + } + + @Test + public void returnImplem() { + JGitBlameCommand jblameCommand = new JGitBlameCommand(new PathResolver(), analysisWarnings); + GitScmProvider gitScmProvider = new GitScmProvider(jblameCommand, analysisWarnings, gitIgnoreCommand, system2); + + assertThat(gitScmProvider.blameCommand()).isEqualTo(jblameCommand); + } + + /** + * SONARSCGIT-47 + */ + @Test + public void branchChangedFiles_should_not_crash_if_branches_have_no_common_ancestors() throws GitAPIException, IOException { + String fileName = "file-in-first-commit.xoo"; + String renamedName = "file-renamed.xoo"; + git.checkout().setOrphan(true).setName("b1").call(); + + Path file = worktree.resolve(fileName); + Path renamed = file.resolveSibling(renamedName); + addLineToFile(fileName, 1); + + Files.move(file, renamed); + git.rm().addFilepattern(fileName).call(); + commit(renamedName); + + Set files = newScmProvider().branchChangedFiles("master", worktree); + + // no shared history, so no diff + assertThat(files).isNull(); + } + + @Test + public void testAutodetection() throws IOException { + File baseDirEmpty = temp.newFolder(); + assertThat(newGitScmProvider().supports(baseDirEmpty)).isFalse(); + + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + File baseDir = new File(projectDir, "dummy-git"); + assertThat(newScmProvider().supports(baseDir)).isTrue(); + } + + private static JGitBlameCommand mockCommand() { + return mock(JGitBlameCommand.class); + } + + @Test + public void branchChangedFiles_from_diverged() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo"); + createAndCommitFile("file-m2.xoo"); + createAndCommitFile("file-m3.xoo"); + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + + appendToAndCommitFile("file-m3.xoo"); + createAndCommitFile("file-m4.xoo"); + + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + appendToAndCommitFile("file-m1.xoo"); + deleteAndCommitFile("file-m2.xoo"); + + assertThat(newScmProvider().branchChangedFiles("master", worktree)) + .containsExactlyInAnyOrder( + worktree.resolve("file-b1.xoo"), + worktree.resolve("file-m1.xoo")); + } + + @Test + public void branchChangedFiles_should_not_fail_with_patience_diff_algo() throws IOException { + Path gitConfig = worktree.resolve(".git").resolve("config"); + Files.write(gitConfig, "[diff]\nalgorithm = patience\n".getBytes(StandardCharsets.UTF_8)); + Repository repo = FileRepositoryBuilder.create(worktree.resolve(".git").toFile()); + git = new Git(repo); + + assertThat(newScmProvider().branchChangedFiles("master", worktree)).isNull(); + } + + @Test + public void branchChangedFiles_from_merged_and_diverged() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo"); + createAndCommitFile("file-m2.xoo"); + createAndCommitFile("lao.txt", CONTENT_LAO); + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + + createAndCommitFile("file-m3.xoo"); + ObjectId mergePoint = git.getRepository().exactRef("HEAD").getObjectId(); + + appendToAndCommitFile("file-m3.xoo"); + createAndCommitFile("file-m4.xoo"); + + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + appendToAndCommitFile("file-m1.xoo"); + deleteAndCommitFile("file-m2.xoo"); + + git.merge().include(mergePoint).call(); + createAndCommitFile("file-b2.xoo"); + + createAndCommitFile("file-m5.xoo"); + deleteAndCommitFile("file-m5.xoo"); + + Set changedFiles = newScmProvider().branchChangedFiles("master", worktree); + assertThat(changedFiles) + .containsExactlyInAnyOrder( + worktree.resolve("file-m1.xoo"), + worktree.resolve("file-b1.xoo"), + worktree.resolve("file-b2.xoo")); + + // use a subset of changed files for .branchChangedLines to verify only requested files are returned + assertThat(changedFiles.remove(worktree.resolve("file-b1.xoo"))).isTrue(); + + // generate common sample diff + createAndCommitFile("lao.txt", CONTENT_TZU); + changedFiles.add(worktree.resolve("lao.txt")); + + // a file that should not yield any results + changedFiles.add(worktree.resolve("nonexistent")); + + assertThat(newScmProvider().branchChangedLines("master", worktree, changedFiles)) + .containsOnly( + entry(worktree.resolve("lao.txt"), new HashSet<>(Arrays.asList(2, 3, 11, 12, 13))), + entry(worktree.resolve("file-m1.xoo"), new HashSet<>(Arrays.asList(4))), + entry(worktree.resolve("file-b2.xoo"), new HashSet<>(Arrays.asList(1, 2, 3)))); + + assertThat(newScmProvider().branchChangedLines("master", worktree, Collections.singleton(worktree.resolve("nonexistent")))) + .isEmpty(); + } + + @Test + public void forkDate_from_diverged() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo", Instant.now().minus(8, ChronoUnit.DAYS)); + createAndCommitFile("file-m2.xoo", Instant.now().minus(7, ChronoUnit.DAYS)); + Instant expectedForkDate = Instant.now().minus(6, ChronoUnit.DAYS); + createAndCommitFile("file-m3.xoo", expectedForkDate); + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + + appendToAndCommitFile("file-m3.xoo"); + createAndCommitFile("file-m4.xoo"); + + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + appendToAndCommitFile("file-m1.xoo"); + deleteAndCommitFile("file-m2.xoo"); + + assertThat(newScmProvider().forkDate("master", worktree)) + .isEqualTo(expectedForkDate.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test + public void forkDate_should_not_fail_if_reference_is_the_same_branch() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo", Instant.now().minus(8, ChronoUnit.DAYS)); + createAndCommitFile("file-m2.xoo", Instant.now().minus(7, ChronoUnit.DAYS)); + + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + + Instant expectedForkDate = Instant.now().minus(6, ChronoUnit.DAYS); + createAndCommitFile("file-m3.xoo", expectedForkDate); + + assertThat(newScmProvider().forkDate("b1", worktree)) + .isEqualTo(expectedForkDate.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test + public void forkDate_should_not_fail_with_patience_diff_algo() throws IOException { + Path gitConfig = worktree.resolve(".git").resolve("config"); + Files.write(gitConfig, "[diff]\nalgorithm = patience\n".getBytes(StandardCharsets.UTF_8)); + Repository repo = FileRepositoryBuilder.create(worktree.resolve(".git").toFile()); + git = new Git(repo); + + assertThat(newScmProvider().forkDate("master", worktree)).isNull(); + } + + @Test + public void forkDate_should_not_fail_with_invalid_basedir() throws IOException { + assertThat(newScmProvider().forkDate("master", temp.newFolder().toPath())).isNull(); + } + + @Test + public void forkDate_should_not_fail_when_no_merge_base_is_found() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo", Instant.now().minus(8, ChronoUnit.DAYS)); + + git.checkout().setOrphan(true).setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + + assertThat(newScmProvider().forkDate("master", worktree)).isNull(); + } + + @Test + public void forkDate_without_target_branch() throws IOException, GitAPIException { + createAndCommitFile("file-m1.xoo", Instant.now().minus(8, ChronoUnit.DAYS)); + createAndCommitFile("file-m2.xoo", Instant.now().minus(7, ChronoUnit.DAYS)); + Instant expectedForkDate = Instant.now().minus(6, ChronoUnit.DAYS); + createAndCommitFile("file-m3.xoo", expectedForkDate); + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + + appendToAndCommitFile("file-m3.xoo"); + createAndCommitFile("file-m4.xoo"); + + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + appendToAndCommitFile("file-m1.xoo"); + deleteAndCommitFile("file-m2.xoo"); + + assertThat(newScmProvider().forkDate("unknown", worktree)).isNull(); + } + + @Test + public void branchChangedLines_should_be_correct_when_change_is_not_committed() throws GitAPIException, IOException { + String fileName = "file-in-first-commit.xoo"; + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + + // this line is committed + addLineToFile(fileName, 3); + commit(fileName); + + // this line is not committed + addLineToFile(fileName, 1); + + Path filePath = worktree.resolve(fileName); + Map> changedLines = newScmProvider().branchChangedLines("master", worktree, Collections.singleton(filePath)); + + // both lines appear correctly + assertThat(changedLines).containsExactly(entry(filePath, new HashSet<>(Arrays.asList(1, 4)))); + } + + @Test + public void branchChangedLines_should_not_fail_if_there_is_no_merge_base() throws GitAPIException, IOException { + createAndCommitFile("file-m1.xoo"); + git.checkout().setOrphan(true).setName("b1").call(); + createAndCommitFile("file-b1.xoo"); + + Map> changedLines = newScmProvider().branchChangedLines("master", worktree, Collections.singleton(Paths.get(""))); + assertThat(changedLines).isNull(); + } + + @Test + public void branchChangedLines_returns_empty_set_for_files_with_lines_removed_only() throws GitAPIException, IOException { + String fileName = "file-in-first-commit.xoo"; + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + + removeLineInFile(fileName, 2); + commit(fileName); + + Path filePath = worktree.resolve(fileName); + Map> changedLines = newScmProvider().branchChangedLines("master", worktree, Collections.singleton(filePath)); + + // both lines appear correctly + assertThat(changedLines).containsExactly(entry(filePath, emptySet())); + } + + @Test + public void branchChangedLines_uses_relative_paths_from_project_root() throws GitAPIException, IOException { + String fileName = "project1/file-in-first-commit.xoo"; + createAndCommitFile(fileName); + + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + + // this line is committed + addLineToFile(fileName, 3); + commit(fileName); + + // this line is not committed + addLineToFile(fileName, 1); + + Path filePath = worktree.resolve(fileName); + Map> changedLines = newScmProvider().branchChangedLines("master", + worktree.resolve("project1"), Collections.singleton(filePath)); + + // both lines appear correctly + assertThat(changedLines).containsExactly(entry(filePath, new HashSet<>(Arrays.asList(1, 4)))); + } + + @Test + public void branchChangedFiles_when_git_work_tree_is_above_project_basedir() throws IOException, GitAPIException { + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + + Path projectDir = worktree.resolve("project"); + Files.createDirectory(projectDir); + createAndCommitFile("project/file-b1"); + assertThat(newScmProvider().branchChangedFiles("master", projectDir)) + .containsOnly(projectDir.resolve("file-b1")); + } + + @Test + public void branchChangedLines_should_not_fail_with_patience_diff_algo() throws IOException { + Path gitConfig = worktree.resolve(".git").resolve("config"); + Files.write(gitConfig, "[diff]\nalgorithm = patience\n".getBytes(StandardCharsets.UTF_8)); + Repository repo = FileRepositoryBuilder.create(worktree.resolve(".git").toFile()); + git = new Git(repo); + + assertThat(newScmProvider().branchChangedLines("master", worktree, Collections.singleton(Paths.get("file")))).isNull(); + } + + /** + * Unfortunately it looks like JGit doesn't support this setting using .gitattributes. + */ + @Test + public void branchChangedLines_should_always_ignore_different_line_endings() throws IOException, GitAPIException { + Path filePath = worktree.resolve("file-m1.xoo"); + + createAndCommitFile("file-m1.xoo"); + ObjectId forkPoint = git.getRepository().exactRef("HEAD").getObjectId(); + + git.branchCreate().setName("b1").setStartPoint(forkPoint.getName()).call(); + git.checkout().setName("b1").call(); + + String newFileContent = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8).replaceAll("\n", "\r\n"); + Files.write(filePath, newFileContent.getBytes(StandardCharsets.UTF_8), StandardOpenOption.TRUNCATE_EXISTING); + commit("file-m1.xoo"); + + assertThat(newScmProvider().branchChangedLines("master", worktree, Collections.singleton(filePath))) + .isEmpty(); + } + + @Test + public void branchChangedFiles_falls_back_to_origin_when_local_branch_does_not_exist() throws IOException, GitAPIException { + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1"); + + Path worktree2 = temp.newFolder().toPath(); + Git.cloneRepository() + .setURI(worktree.toString()) + .setDirectory(worktree2.toFile()) + .call(); + + assertThat(newScmProvider().branchChangedFiles("master", worktree2)) + .containsOnly(worktree2.resolve("file-b1")); + verifyZeroInteractions(analysisWarnings); + } + + @Test + public void branchChangedFiles_use_remote_target_ref_when_running_on_circle_ci() throws IOException, GitAPIException { + when(system2.envVariable("CIRCLECI")).thenReturn("true"); + git.checkout().setName("b1").setCreateBranch(true).call(); + createAndCommitFile("file-b1"); + + Path worktree2 = temp.newFolder().toPath(); + Git local = Git.cloneRepository() + .setURI(worktree.toString()) + .setDirectory(worktree2.toFile()) + .call(); + + // Make local master match analyzed branch, so if local ref is used then change files will be empty + local.checkout().setCreateBranch(true).setName("master").setStartPoint("origin/b1").call(); + local.checkout().setName("b1").call(); + + assertThat(newScmProvider().branchChangedFiles("master", worktree2)) + .containsOnly(worktree2.resolve("file-b1")); + verifyZeroInteractions(analysisWarnings); + } + + @Test + public void branchChangedFiles_falls_back_to_local_ref_if_origin_branch_does_not_exist_when_running_on_circle_ci() throws IOException, GitAPIException { + when(system2.envVariable("CIRCLECI")).thenReturn("true"); + git.checkout().setName("b1").setCreateBranch(true).call(); + createAndCommitFile("file-b1"); + + Path worktree2 = temp.newFolder().toPath(); + Git local = Git.cloneRepository() + .setURI(worktree.toString()) + .setDirectory(worktree2.toFile()) + .call(); + + local.checkout().setName("local-only").setCreateBranch(true).setStartPoint("origin/master").call(); + local.checkout().setName("b1").call(); + + assertThat(newScmProvider().branchChangedFiles("local-only", worktree2)) + .containsOnly(worktree2.resolve("file-b1")); + verifyZeroInteractions(analysisWarnings); + } + + @Test + public void branchChangedFiles_falls_back_to_upstream_ref() throws IOException, GitAPIException { + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + createAndCommitFile("file-b1"); + + Path worktree2 = temp.newFolder().toPath(); + Git.cloneRepository() + .setURI(worktree.toString()) + .setRemote("upstream") + .setDirectory(worktree2.toFile()) + .call(); + + assertThat(newScmProvider().branchChangedFiles("master", worktree2)) + .containsOnly(worktree2.resolve("file-b1")); + verifyZeroInteractions(analysisWarnings); + + } + + @Test + public void branchChangedFiles_should_return_null_when_branch_nonexistent() { + assertThat(newScmProvider().branchChangedFiles("nonexistent", worktree)).isNull(); + } + + @Test + public void branchChangedFiles_should_throw_when_repo_nonexistent() throws IOException { + thrown.expect(MessageException.class); + thrown.expectMessage("Not inside a Git work tree: "); + newScmProvider().branchChangedFiles("master", temp.newFolder().toPath()); + } + + @Test + public void branchChangedFiles_should_throw_when_dir_nonexistent() { + thrown.expect(MessageException.class); + thrown.expectMessage("Not inside a Git work tree: "); + newScmProvider().branchChangedFiles("master", temp.getRoot().toPath().resolve("nonexistent")); + } + + @Test + public void branchChangedFiles_should_return_null_on_io_errors_of_repo_builder() { + GitScmProvider provider = new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2) { + @Override + Repository buildRepo(Path basedir) throws IOException { + throw new IOException(); + } + }; + assertThat(provider.branchChangedFiles("branch", worktree)).isNull(); + verifyZeroInteractions(analysisWarnings); + } + + @Test + public void branchChangedFiles_should_return_null_if_repo_exactref_is_null() throws IOException { + Repository repository = mock(Repository.class); + RefDatabase refDatabase = mock(RefDatabase.class); + when(repository.getRefDatabase()).thenReturn(refDatabase); + when(refDatabase.findRef("branch")).thenReturn(null); + + GitScmProvider provider = new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2) { + @Override + Repository buildRepo(Path basedir) { + return repository; + } + }; + assertThat(provider.branchChangedFiles("branch", worktree)).isNull(); + + String warning = "Could not find ref 'branch' in refs/heads, refs/remotes/upstream or refs/remotes/origin." + + " You may see unexpected issues and changes. Please make sure to fetch this ref before pull request analysis."; + verify(analysisWarnings).addUnique(warning); + } + + @Test + public void branchChangedFiles_should_return_null_on_errors() throws GitAPIException { + DiffCommand diffCommand = mock(DiffCommand.class); + when(diffCommand.setShowNameAndStatusOnly(anyBoolean())).thenReturn(diffCommand); + when(diffCommand.setOldTree(any())).thenReturn(diffCommand); + when(diffCommand.setNewTree(any())).thenReturn(diffCommand); + when(diffCommand.call()).thenThrow(mock(GitAPIException.class)); + + Git git = mock(Git.class); + when(git.diff()).thenReturn(diffCommand); + + GitScmProvider provider = new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2) { + @Override + Git newGit(Repository repo) { + return git; + } + }; + assertThat(provider.branchChangedFiles("master", worktree)).isNull(); + verify(diffCommand).call(); + } + + @Test + public void branchChangedLines_returns_null_when_branch_doesnt_exist() { + assertThat(newScmProvider().branchChangedLines("nonexistent", worktree, emptySet())).isNull(); + } + + @Test + public void branchChangedLines_omits_files_with_git_api_errors() throws IOException, GitAPIException { + String f1 = "file-in-first-commit.xoo"; + String f2 = "file2-in-first-commit.xoo"; + + createAndCommitFile(f2); + + git.branchCreate().setName("b1").call(); + git.checkout().setName("b1").call(); + + // both files modified + addLineToFile(f1, 1); + addLineToFile(f2, 2); + + commit(f1); + commit(f2); + + AtomicInteger callCount = new AtomicInteger(0); + GitScmProvider provider = new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2) { + @Override + AbstractTreeIterator prepareTreeParser(Repository repo, RevCommit commit) throws IOException { + if (callCount.getAndIncrement() == 1) { + throw new RuntimeException("error"); + } + return super.prepareTreeParser(repo, commit); + } + }; + Set changedFiles = new LinkedHashSet<>(); + changedFiles.add(worktree.resolve(f1)); + changedFiles.add(worktree.resolve(f2)); + + assertThat(provider.branchChangedLines("master", worktree, changedFiles)) + .isEqualTo(Collections.singletonMap(worktree.resolve(f1), Collections.singleton(1))); + } + + @Test + public void branchChangedLines_returns_null_on_io_errors_of_repo_builder() { + GitScmProvider provider = new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2) { + @Override + Repository buildRepo(Path basedir) throws IOException { + throw new IOException(); + } + }; + assertThat(provider.branchChangedLines("branch", worktree, emptySet())).isNull(); + } + + @Test + public void relativePathFromScmRoot_should_return_dot_project_root() { + assertThat(newGitScmProvider().relativePathFromScmRoot(worktree)).isEqualTo(Paths.get("")); + } + + private GitScmProvider newGitScmProvider() { + return new GitScmProvider(mock(JGitBlameCommand.class), analysisWarnings, gitIgnoreCommand, system2); + } + + @Test + public void relativePathFromScmRoot_should_return_filename_for_file_in_project_root() throws IOException { + Path filename = Paths.get("somefile.xoo"); + Path path = worktree.resolve(filename); + Files.createFile(path); + assertThat(newGitScmProvider().relativePathFromScmRoot(path)).isEqualTo(filename); + } + + @Test + public void relativePathFromScmRoot_should_return_relative_path_for_file_in_project_subdir() throws IOException { + Path relpath = Paths.get("sub/dir/to/somefile.xoo"); + Path path = worktree.resolve(relpath); + Files.createDirectories(path.getParent()); + Files.createFile(path); + assertThat(newGitScmProvider().relativePathFromScmRoot(path)).isEqualTo(relpath); + } + + @Test + public void revisionId_should_return_different_sha1_after_commit() throws IOException, GitAPIException { + Path projectDir = worktree.resolve("project"); + Files.createDirectory(projectDir); + + GitScmProvider provider = newGitScmProvider(); + + String sha1before = provider.revisionId(projectDir); + assertThat(sha1before).hasSize(40); + + createAndCommitFile("project/file1"); + String sha1after = provider.revisionId(projectDir); + assertThat(sha1after).hasSize(40); + + assertThat(sha1after).isNotEqualTo(sha1before); + assertThat(provider.revisionId(projectDir)).isEqualTo(sha1after); + } + + @Test + public void revisionId_should_return_null_in_empty_repo() throws IOException { + worktree = temp.newFolder().toPath(); + Repository repo = FileRepositoryBuilder.create(worktree.resolve(".git").toFile()); + repo.create(); + + git = new Git(repo); + + Path projectDir = worktree.resolve("project"); + Files.createDirectory(projectDir); + + GitScmProvider provider = newGitScmProvider(); + + assertThat(provider.revisionId(projectDir)).isNull(); + } + + private String randomizedContent(String prefix, int numLines) { + StringBuilder sb = new StringBuilder(); + for (int line = 0; line < numLines; line++) { + sb.append(randomizedLine(prefix)); + sb.append("\n"); + } + return sb.toString(); + } + + private String randomizedLine(String prefix) { + StringBuilder sb = new StringBuilder(prefix); + for (int i = 0; i < 4; i++) { + sb.append(' '); + for (int j = 0; j < prefix.length(); j++) { + sb.append((char) ('a' + random.nextInt(26))); + } + } + return sb.toString(); + } + + private void createAndCommitFile(String relativePath) throws IOException, GitAPIException { + createAndCommitFile(relativePath, randomizedContent(relativePath, 3)); + } + + private void createAndCommitFile(String relativePath, Instant commitDate) throws IOException, GitAPIException { + createFile(relativePath, randomizedContent(relativePath, 3)); + commit(relativePath, commitDate); + } + + private void createAndCommitFile(String relativePath, String content) throws IOException, GitAPIException { + createFile(relativePath, content); + commit(relativePath); + } + + private void createFile(String relativePath, String content) throws IOException { + Path newFile = worktree.resolve(relativePath); + Files.createDirectories(newFile.getParent()); + Files.write(newFile, content.getBytes(), StandardOpenOption.CREATE); + } + + private void addLineToFile(String relativePath, int lineNumber) throws IOException { + Path filePath = worktree.resolve(relativePath); + List lines = Files.readAllLines(filePath); + lines.add(lineNumber - 1, randomizedLine(relativePath)); + Files.write(filePath, lines, StandardOpenOption.TRUNCATE_EXISTING); + } + + private void removeLineInFile(String relativePath, int lineNumber) throws IOException { + Path filePath = worktree.resolve(relativePath); + List lines = Files.readAllLines(filePath); + lines.remove(lineNumber - 1); + Files.write(filePath, lines, StandardOpenOption.TRUNCATE_EXISTING); + } + + private void appendToAndCommitFile(String relativePath) throws IOException, GitAPIException { + Files.write(worktree.resolve(relativePath), randomizedContent(relativePath, 1).getBytes(), StandardOpenOption.APPEND); + commit(relativePath); + } + + private void deleteAndCommitFile(String relativePath) throws GitAPIException { + git.rm().addFilepattern(relativePath).call(); + commit(relativePath); + } + + private void commit(String... relativePaths) throws GitAPIException { + for (String path : relativePaths) { + git.add().addFilepattern(path).call(); + } + String msg = String.join(",", relativePaths); + git.commit().setAuthor("joe", "joe@example.com").setMessage(msg).call(); + } + + private void commit(String relativePath, Instant date) throws GitAPIException { + PersonIdent person = new PersonIdent("joe", "joe@example.com", Date.from(date), TimeZone.getDefault()); + git.commit().setAuthor(person).setCommitter(person).setMessage(relativePath).call(); + } + + private GitScmProvider newScmProvider() { + return new GitScmProvider(mockCommand(), analysisWarnings, gitIgnoreCommand, system2); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmSupportTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmSupportTest.java new file mode 100644 index 00000000000..803b02088be --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitScmSupportTest.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GitScmSupportTest { + + @Test + public void getClasses() { + assertThat(GitScmSupport.getClasses()).hasSize(3); + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitThreadFactoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitThreadFactoryTest.java new file mode 100644 index 00000000000..39f46cdd2b8 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitThreadFactoryTest.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.util.concurrent.ForkJoinPool; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GitThreadFactoryTest { + @Test + public void testName() { + GitThreadFactory factory = new GitThreadFactory(); + ForkJoinPool pool = new ForkJoinPool(); + assertThat(factory.newThread(pool).getName()).isEqualTo("git-scm-0"); + assertThat(factory.newThread(pool).getName()).isEqualTo("git-scm-1"); + + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitBlameCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitBlameCommandTest.java new file mode 100644 index 00000000000..936ead4e69b --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/JGitBlameCommandTest.java @@ -0,0 +1,343 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultFileSystem; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.scm.BlameCommand.BlameInput; +import org.sonar.api.batch.scm.BlameCommand.BlameOutput; +import org.sonar.api.batch.scm.BlameLine; +import org.sonar.api.notifications.AnalysisWarnings; +import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.DateUtils; +import org.sonar.api.utils.MessageException; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.LogTester; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assume.assumeTrue; +import static org.mockito.Matchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.sonar.scm.git.Utils.javaUnzip; + +public class JGitBlameCommandTest { + + private static final String DUMMY_JAVA = "src/main/java/org/dummy/Dummy.java"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Rule + public LogTester logTester = new LogTester(); + + private final BlameInput input = mock(BlameInput.class); + + @Test + public void testBlame() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git"); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + DefaultInputFile inputFile = new TestInputFileBuilder("foo", DUMMY_JAVA) + .setModuleBaseDir(baseDir.toPath()) + .build(); + fs.add(inputFile); + + BlameOutput blameResult = mock(BlameOutput.class); + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile)); + jGitBlameCommand.blame(input, blameResult); + + Date revisionDate1 = DateUtils.parseDateTime("2012-07-17T16:12:48+0200"); + String revision1 = "6b3aab35a3ea32c1636fee56f996e677653c48ea"; + String author1 = "david@gageot.net"; + + // second commit, which has a commit date different than the author date + Date revisionDate2 = DateUtils.parseDateTime("2015-05-19T13:31:09+0200"); + String revision2 = "0d269c1acfb8e6d4d33f3c43041eb87e0df0f5e7"; + String author2 = "duarte.meneses@sonarsource.com"; + + List expectedBlame = new LinkedList<>(); + for (int i = 0; i < 25; i++) { + expectedBlame.add(new BlameLine().revision(revision1).date(revisionDate1).author(author1)); + } + for (int i = 0; i < 3; i++) { + expectedBlame.add(new BlameLine().revision(revision2).date(revisionDate2).author(author2)); + } + for (int i = 0; i < 1; i++) { + expectedBlame.add(new BlameLine().revision(revision1).date(revisionDate1).author(author1)); + } + + verify(blameResult).blameResult(inputFile, expectedBlame); + } + + @Test + public void properFailureIfNotAGitProject() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git"); + + // Delete .git + FileUtils.forceDelete(new File(baseDir, ".git")); + + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + DefaultInputFile inputFile = new TestInputFileBuilder("foo", DUMMY_JAVA).build(); + fs.add(inputFile); + + BlameOutput blameResult = mock(BlameOutput.class); + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile)); + + thrown.expect(MessageException.class); + thrown.expectMessage("Not inside a Git work tree: "); + + jGitBlameCommand.blame(input, blameResult); + } + + @Test + public void testBlameOnNestedModule() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git-nested.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git-nested/dummy-project"); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + DefaultInputFile inputFile = new TestInputFileBuilder("foo", DUMMY_JAVA) + .setModuleBaseDir(baseDir.toPath()) + .build(); + fs.add(inputFile); + + BlameOutput blameResult = mock(BlameOutput.class); + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile)); + jGitBlameCommand.blame(input, blameResult); + + Date revisionDate = DateUtils.parseDateTime("2012-07-17T16:12:48+0200"); + String revision = "6b3aab35a3ea32c1636fee56f996e677653c48ea"; + String author = "david@gageot.net"; + verify(blameResult).blameResult(inputFile, + Arrays.asList( + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author), + new BlameLine().revision(revision).date(revisionDate).author(author))); + } + + @Test + public void dontFailOnModifiedFile() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git"); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + String relativePath = DUMMY_JAVA; + DefaultInputFile inputFile = new TestInputFileBuilder("foo", relativePath).build(); + fs.add(inputFile); + + // Emulate a modification + Files.write(baseDir.toPath().resolve(relativePath), "modification and \n some new line".getBytes()); + + BlameOutput blameResult = mock(BlameOutput.class); + + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile)); + jGitBlameCommand.blame(input, blameResult); + } + + @Test + public void dontFailOnNewFile() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git"); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + String relativePath = DUMMY_JAVA; + String relativePath2 = "src/main/java/org/dummy/Dummy2.java"; + DefaultInputFile inputFile = new TestInputFileBuilder("foo", relativePath).build(); + fs.add(inputFile); + DefaultInputFile inputFile2 = new TestInputFileBuilder("foo", relativePath2).build(); + fs.add(inputFile2); + + // Emulate a new file + FileUtils.copyFile(new File(baseDir, relativePath), new File(baseDir, relativePath2)); + + BlameOutput blameResult = mock(BlameOutput.class); + + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile, inputFile2)); + jGitBlameCommand.blame(input, blameResult); + } + + @Test + public void dontFailOnSymlink() throws IOException { + assumeTrue(!System2.INSTANCE.isOsWindows()); + File projectDir = temp.newFolder(); + javaUnzip("dummy-git.zip", projectDir); + + JGitBlameCommand jGitBlameCommand = newJGitBlameCommand(); + + File baseDir = new File(projectDir, "dummy-git"); + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + String relativePath = DUMMY_JAVA; + String relativePath2 = "src/main/java/org/dummy/Dummy2.java"; + DefaultInputFile inputFile = new TestInputFileBuilder("foo", relativePath) + .setModuleBaseDir(baseDir.toPath()) + .build(); + fs.add(inputFile); + DefaultInputFile inputFile2 = new TestInputFileBuilder("foo", relativePath2) + .setModuleBaseDir(baseDir.toPath()) + .build(); + fs.add(inputFile2); + + // Create symlink + Files.createSymbolicLink(inputFile2.file().toPath(), inputFile.file().toPath()); + + BlameOutput blameResult = mock(BlameOutput.class); + + when(input.filesToBlame()).thenReturn(Arrays.asList(inputFile, inputFile2)); + jGitBlameCommand.blame(input, blameResult); + } + + @Test + public void return_early_when_shallow_clone_detected() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("shallow-git.zip", projectDir); + + File baseDir = new File(projectDir, "shallow-git"); + + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + + DefaultInputFile inputFile = new TestInputFileBuilder("foo", DUMMY_JAVA).build(); + when(input.filesToBlame()).thenReturn(Collections.singleton(inputFile)); + + // register warning with default wrapper + AnalysisWarnings analysisWarnings = mock(AnalysisWarnings.class); + JGitBlameCommand jGitBlameCommand = new JGitBlameCommand(new PathResolver(), analysisWarnings); + BlameOutput output = mock(BlameOutput.class); + jGitBlameCommand.blame(input, output); + + assertThat(logTester.logs()).first() + .matches(s -> s.contains("Shallow clone detected, no blame information will be provided.")); + verifyZeroInteractions(output); + + verify(analysisWarnings).addUnique(startsWith("Shallow clone detected")); + } + + @Test + public void return_early_when_clone_with_reference_detected() throws IOException { + File projectDir = temp.newFolder(); + javaUnzip("dummy-git-reference-clone.zip", projectDir); + + Path baseDir = projectDir.toPath().resolve("dummy-git2"); + + DefaultFileSystem fs = new DefaultFileSystem(baseDir); + when(input.fileSystem()).thenReturn(fs); + + DefaultInputFile inputFile = new TestInputFileBuilder("foo", DUMMY_JAVA).setModuleBaseDir(baseDir).build(); + when(input.filesToBlame()).thenReturn(Collections.singleton(inputFile)); + + // register warning + AnalysisWarnings analysisWarnings = mock(AnalysisWarnings.class); + JGitBlameCommand jGitBlameCommand = new JGitBlameCommand(new PathResolver(), analysisWarnings); + TestBlameOutput output = new TestBlameOutput(); + jGitBlameCommand.blame(input, output); + + assertThat(logTester.logs()).first() + .matches(s -> s.contains("This git repository references another local repository which is not well supported")); + + // contains commits referenced from the old clone and commits in the new clone + assertThat(output.blame.keySet()).contains(inputFile); + assertThat(output.blame.get(inputFile).stream().map(BlameLine::revision)) + .containsOnly("6b3aab35a3ea32c1636fee56f996e677653c48ea", "843c7c30d7ebd9a479e8f1daead91036c75cbc4e", "0d269c1acfb8e6d4d33f3c43041eb87e0df0f5e7"); + verifyZeroInteractions(analysisWarnings); + } + + private JGitBlameCommand newJGitBlameCommand() { + return new JGitBlameCommand(new PathResolver(), mock(AnalysisWarnings.class)); + } + + private static class TestBlameOutput implements BlameOutput { + private Map> blame = new LinkedHashMap<>(); + + @Override public void blameResult(InputFile inputFile, List list) { + blame.put(inputFile, list); + } + } + +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/Utils.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/Utils.java new file mode 100644 index 00000000000..cdee9fde16f --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/Utils.java @@ -0,0 +1,67 @@ +/* + * SonarQube + * Copyright (C) 2009-2020 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scm.git; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.Enumeration; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.apache.commons.io.FileUtils; + +import static java.lang.String.format; + +public class Utils { + public static void javaUnzip(String zipFileName, File toDir) throws IOException { + try { + File testRepos = new File(Utils.class.getResource("test-repos").toURI()); + File zipFile = new File(testRepos, zipFileName); + javaUnzip(zipFile, toDir); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + private static void javaUnzip(File zip, File toDir) { + try { + try (ZipFile zipFile = new ZipFile(zip)) { + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + File to = new File(toDir, entry.getName()); + if (entry.isDirectory()) { + FileUtils.forceMkdir(to); + } else { + File parent = to.getParentFile(); + if (parent != null) { + FileUtils.forceMkdir(parent); + } + + Files.copy(zipFile.getInputStream(entry), to.toPath()); + } + } + } + } catch (Exception e) { + throw new IllegalStateException(format("Fail to unzip %s to %s", zip, toDir), e); + } + } +} diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-nested.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-nested.zip new file mode 100644 index 00000000000..b0ee03b6b16 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-nested.zip differ diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-reference-clone.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-reference-clone.zip new file mode 100644 index 00000000000..080812c60d0 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git-reference-clone.zip differ diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git.zip new file mode 100644 index 00000000000..f8900d5be39 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/dummy-git.zip differ diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/ignore-git.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/ignore-git.zip new file mode 100644 index 00000000000..2d5c3289c98 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/ignore-git.zip differ diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/reference-git.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/reference-git.zip new file mode 100644 index 00000000000..d0805aecb50 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/reference-git.zip differ diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/shallow-git.zip b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/shallow-git.zip new file mode 100644 index 00000000000..353633c1d12 Binary files /dev/null and b/sonar-scanner-engine/src/test/resources/org/sonar/scm/git/test-repos/shallow-git.zip differ