SONAR-13792 Embed sonar-scm-git

10 files changed, 885 insertions, 0 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
index 7e14856626d..ce29d00a358 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java
@@ -131,6 +131,7 @@ import org.sonar.scanner.sensor.ProjectSensorContext;
 import org.sonar.scanner.sensor.ProjectSensorExtensionDictionnary;
 import org.sonar.scanner.sensor.ProjectSensorOptimizer;
 import org.sonar.scanner.sensor.ProjectSensorsExecutor;
+import org.sonar.scm.git.GitScmSupport;
 
 import static org.sonar.api.batch.InstantiationStrategy.PER_BATCH;
 import static org.sonar.core.extension.CoreExtensionsInstaller.noExtensionFilter;
@@ -301,6 +302,8 @@ public class ProjectScanContainer extends ComponentContainer {
 
       AnalysisObservers.class);
 
+    add(GitScmSupport.getClasses());
+
     addIfMissing(DefaultProjectSettingsLoader.class, ProjectSettingsLoader.class);
     addIfMissing(DefaultRulesLoader.class, RulesLoader.class);
     addIfMissing(DefaultActiveRulesLoader.class, ActiveRulesLoader.class);
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java
new file mode 100644
index 00000000000..1db66e65ba0
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/ChangedLinesComputer.java
@@ -0,0 +1,116 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+class ChangedLinesComputer {
+  private final Tracker tracker = new Tracker();
+
+  private final OutputStream receiver = new OutputStream() {
+    StringBuilder sb = new StringBuilder();
+
+    @Override
+    public void write(int b) {
+      sb.append((char) b);
+      if (b == '\n') {
+        tracker.parseLine(sb.toString());
+        sb.setLength(0);
+      }
+    }
+  };
+
+  /**
+   * The OutputStream to pass to JGit's diff command.
+   */
+  OutputStream receiver() {
+    return receiver;
+  }
+
+  /**
+   * From a stream of unified diff lines emitted by Git <strong>for a single file</strong>,
+   * compute the line numbers that should be considered changed.
+   * Example input:
+   * <pre>
+   * diff --git a/lao.txt b/lao.txt
+   * index 635ef2c..7f050f2 100644
+   * --- a/lao.txt
+   * +++ b/lao.txt
+   * @@ -1,7 +1,6 @@
+   * -The Way that can be told of is not the eternal Way;
+   * -The name that can be named is not the eternal name.
+   *  The Nameless is the origin of Heaven and Earth;
+   * -The Named is the mother of all things.
+   * +The named is the mother of all things.
+   * +
+   *  Therefore let there always be non-being,
+   *    so we may see their subtlety,
+   *  And let there always be being,
+   * @@ -9,3 +8,6 @@ And let there always be being,
+   *  The two are the same,
+   *  But after they are produced,
+   *    they have different names.
+   * +They both may be called deep and profound.
+   * +Deeper and more profound,
+   * +The door of all subtleties!names.
+   * </pre>
+   * See also: http://www.gnu.org/software/diffutils/manual/html_node/Example-Unified.html#Example-Unified
+   */
+  Set<Integer> changedLines() {
+    return tracker.changedLines();
+  }
+
+  private static class Tracker {
+
+    private static final Pattern START_LINE_IN_TARGET = Pattern.compile(" \\+(\\d+)");
+
+    private final Set<Integer> changedLines = new HashSet<>();
+
+    private boolean foundStart = false;
+    private int lineNumInTarget;
+
+    private void parseLine(String line) {
+      if (line.startsWith("@@ ")) {
+        Matcher matcher = START_LINE_IN_TARGET.matcher(line);
+        if (!matcher.find()) {
+          throw new IllegalStateException("Invalid block header on line " + line);
+        }
+        foundStart = true;
+        lineNumInTarget = Integer.parseInt(matcher.group(1));
+      } else if (foundStart) {
+        char firstChar = line.charAt(0);
+        if (firstChar == ' ') {
+          lineNumInTarget++;
+        } else if (firstChar == '+') {
+          changedLines.add(lineNumInTarget);
+          lineNumInTarget++;
+        }
+      }
+    }
+
+    Set<Integer> changedLines() {
+      return changedLines;
+    }
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java
new file mode 100644
index 00000000000..cfe31a1ed67
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitIgnoreCommand.java
@@ -0,0 +1,52 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import org.sonar.api.batch.scm.IgnoreCommand;
+import org.sonar.api.scanner.ScannerSide;
+
+import static java.util.Objects.requireNonNull;
+
+@ScannerSide
+public class GitIgnoreCommand implements IgnoreCommand {
+
+  private IncludedFilesRepository includedFilesRepository;
+
+  @Override
+  public void init(Path baseDir) {
+    try {
+      this.includedFilesRepository = new IncludedFilesRepository(baseDir);
+    } catch (IOException e) {
+      throw new IllegalStateException("I/O error while indexing ignored files.", e);
+    }
+  }
+
+  @Override
+  public boolean isIgnored(Path absolutePath) {
+    return !requireNonNull(includedFilesRepository, "Call init first").contains(absolutePath);
+  }
+
+  @Override
+  public void clean() {
+    this.includedFilesRepository = null;
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java
new file mode 100644
index 00000000000..19b81e1642a
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java
@@ -0,0 +1,376 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import javax.annotation.CheckForNull;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.diff.DiffAlgorithm;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.diff.DiffFormatter;
+import org.eclipse.jgit.diff.RawTextComparator;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.RepositoryBuilder;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.revwalk.filter.RevFilter;
+import org.eclipse.jgit.treewalk.AbstractTreeIterator;
+import org.eclipse.jgit.treewalk.CanonicalTreeParser;
+import org.eclipse.jgit.treewalk.FileTreeIterator;
+import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.sonar.api.batch.scm.BlameCommand;
+import org.sonar.api.batch.scm.ScmProvider;
+import org.sonar.api.notifications.AnalysisWarnings;
+import org.sonar.api.utils.MessageException;
+import org.sonar.api.utils.System2;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+public class GitScmProvider extends ScmProvider {
+
+  private static final Logger LOG = Loggers.get(GitScmProvider.class);
+
+  private final JGitBlameCommand jgitBlameCommand;
+  private final AnalysisWarnings analysisWarnings;
+  private final GitIgnoreCommand gitIgnoreCommand;
+  private final System2 system2;
+
+  public GitScmProvider(JGitBlameCommand jgitBlameCommand, AnalysisWarnings analysisWarnings, GitIgnoreCommand gitIgnoreCommand, System2 system2) {
+    this.jgitBlameCommand = jgitBlameCommand;
+    this.analysisWarnings = analysisWarnings;
+    this.gitIgnoreCommand = gitIgnoreCommand;
+    this.system2 = system2;
+  }
+
+  @Override
+  public GitIgnoreCommand ignoreCommand() {
+    return gitIgnoreCommand;
+  }
+
+  @Override
+  public String key() {
+    return "git";
+  }
+
+  @Override
+  public boolean supports(File baseDir) {
+    RepositoryBuilder builder = new RepositoryBuilder().findGitDir(baseDir);
+    return builder.getGitDir() != null;
+  }
+
+  @Override
+  public BlameCommand blameCommand() {
+    return this.jgitBlameCommand;
+  }
+
+  @CheckForNull
+  @Override
+  public Set<Path> branchChangedFiles(String targetBranchName, Path rootBaseDir) {
+    try (Repository repo = buildRepo(rootBaseDir)) {
+      Ref targetRef = resolveTargetRef(targetBranchName, repo);
+      if (targetRef == null) {
+        analysisWarnings.addUnique(String.format("Could not find ref '%s' in refs/heads, refs/remotes/upstream or refs/remotes/origin. "
+          + "You may see unexpected issues and changes. "
+          + "Please make sure to fetch this ref before pull request analysis.", targetBranchName));
+        return null;
+      }
+
+      if (isDiffAlgoInvalid(repo.getConfig())) {
+        LOG.warn("The diff algorithm configured in git is not supported. "
+          + "No information regarding changes in the branch will be collected, which can lead to unexpected results.");
+        return null;
+      }
+
+      Optional<RevCommit> mergeBaseCommit = findMergeBase(repo, targetRef);
+      if (!mergeBaseCommit.isPresent()) {
+        LOG.warn("No merge base found between HEAD and " + targetRef.getName());
+        return null;
+      }
+      AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit.get());
+
+      // we compare a commit with HEAD, so no point ignoring line endings (it will be whatever is committed)
+      try (Git git = newGit(repo)) {
+        List<DiffEntry> diffEntries = git.diff()
+          .setShowNameAndStatusOnly(true)
+          .setOldTree(mergeBaseTree)
+          .setNewTree(prepareNewTree(repo))
+          .call();
+
+        return diffEntries.stream()
+          .filter(diffEntry -> diffEntry.getChangeType() == DiffEntry.ChangeType.ADD || diffEntry.getChangeType() == DiffEntry.ChangeType.MODIFY)
+          .map(diffEntry -> repo.getWorkTree().toPath().resolve(diffEntry.getNewPath()))
+          .collect(Collectors.toSet());
+      }
+    } catch (IOException | GitAPIException e) {
+      LOG.warn(e.getMessage(), e);
+    }
+    return null;
+  }
+
+  @CheckForNull
+  @Override
+  public Map<Path, Set<Integer>> branchChangedLines(String targetBranchName, Path projectBaseDir, Set<Path> changedFiles) {
+    try (Repository repo = buildRepo(projectBaseDir)) {
+      Ref targetRef = resolveTargetRef(targetBranchName, repo);
+      if (targetRef == null) {
+        analysisWarnings.addUnique(String.format("Could not find ref '%s' in refs/heads, refs/remotes/upstream or refs/remotes/origin. "
+          + "You may see unexpected issues and changes. "
+          + "Please make sure to fetch this ref before pull request analysis.", targetBranchName));
+        return null;
+      }
+
+      if (isDiffAlgoInvalid(repo.getConfig())) {
+        // we already print a warning when branchChangedFiles is called
+        return null;
+      }
+
+      // force ignore different line endings when comparing a commit with the workspace
+      repo.getConfig().setBoolean("core", null, "autocrlf", true);
+
+      Optional<RevCommit> mergeBaseCommit = findMergeBase(repo, targetRef);
+      if (!mergeBaseCommit.isPresent()) {
+        LOG.warn("No merge base found between HEAD and " + targetRef.getName());
+        return null;
+      }
+
+      Map<Path, Set<Integer>> changedLines = new HashMap<>();
+      Path repoRootDir = repo.getDirectory().toPath().getParent();
+
+      for (Path path : changedFiles) {
+        collectChangedLines(repo, mergeBaseCommit.get(), changedLines, repoRootDir, path);
+      }
+      return changedLines;
+    } catch (Exception e) {
+      LOG.warn("Failed to get changed lines from git", e);
+    }
+    return null;
+  }
+
+  private void collectChangedLines(Repository repo, RevCommit mergeBaseCommit, Map<Path, Set<Integer>> changedLines, Path repoRootDir, Path changedFile) {
+    ChangedLinesComputer computer = new ChangedLinesComputer();
+
+    try (DiffFormatter diffFmt = new DiffFormatter(new BufferedOutputStream(computer.receiver()))) {
+      // copied from DiffCommand so that we can use a custom DiffFormatter which ignores white spaces.
+      diffFmt.setRepository(repo);
+      diffFmt.setProgressMonitor(NullProgressMonitor.INSTANCE);
+      diffFmt.setDiffComparator(RawTextComparator.WS_IGNORE_ALL);
+      diffFmt.setPathFilter(PathFilter.create(toGitPath(repoRootDir.relativize(changedFile).toString())));
+
+      AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit);
+      List<DiffEntry> diffEntries = diffFmt.scan(mergeBaseTree, new FileTreeIterator(repo));
+      diffFmt.format(diffEntries);
+      diffFmt.flush();
+      diffEntries.stream()
+        .filter(diffEntry -> diffEntry.getChangeType() == DiffEntry.ChangeType.ADD || diffEntry.getChangeType() == DiffEntry.ChangeType.MODIFY)
+        .findAny()
+        .ifPresent(diffEntry -> changedLines.put(changedFile, computer.changedLines()));
+    } catch (Exception e) {
+      LOG.warn("Failed to get changed lines from git for file " + changedFile, e);
+    }
+  }
+
+  @Override
+  @CheckForNull
+  public Instant forkDate(String referenceBranchName, Path projectBaseDir) {
+    try (Repository repo = buildRepo(projectBaseDir)) {
+      Ref targetRef = resolveTargetRef(referenceBranchName, repo);
+      if (targetRef == null) {
+        LOG.warn("Branch '{}' not found in git", referenceBranchName);
+        return null;
+      }
+
+      if (isDiffAlgoInvalid(repo.getConfig())) {
+        LOG.warn("The diff algorithm configured in git is not supported. "
+          + "No information regarding changes in the branch will be collected, which can lead to unexpected results.");
+        return null;
+      }
+
+      Optional<RevCommit> mergeBaseCommit = findMergeBase(repo, targetRef);
+      if (!mergeBaseCommit.isPresent()) {
+        LOG.warn("No fork point found between HEAD and " + targetRef.getName());
+        return null;
+      }
+
+      return Instant.ofEpochSecond(mergeBaseCommit.get().getCommitTime());
+    } catch (Exception e) {
+      LOG.warn("Failed to find fork point with git", e);
+    }
+
+    return null;
+  }
+
+  private static String toGitPath(String path) {
+    return path.replaceAll(Pattern.quote(File.separator), "/");
+  }
+
+  @CheckForNull
+  private Ref resolveTargetRef(String targetBranchName, Repository repo) throws IOException {
+    String localRef = "refs/heads/" + targetBranchName;
+    String remoteRef = "refs/remotes/origin/" + targetBranchName;
+    String upstreamRef = "refs/remotes/upstream/" + targetBranchName;
+
+    Ref targetRef;
+    // Because circle ci destroys the local reference to master, try to load remote ref first.
+    // https://discuss.circleci.com/t/git-checkout-of-a-branch-destroys-local-reference-to-master/23781
+    if (runningOnCircleCI()) {
+      targetRef = getFirstExistingRef(repo, remoteRef, localRef, upstreamRef);
+    } else {
+      targetRef = getFirstExistingRef(repo, localRef, remoteRef, upstreamRef);
+    }
+    
+    if (targetRef == null) {
+      LOG.warn("Could not find ref: {} in refs/heads, refs/remotes/upstream or refs/remotes/origin", targetBranchName);
+    }
+
+    return targetRef;
+  }
+
+  @CheckForNull
+  private static Ref getFirstExistingRef(Repository repo, String... refs) throws IOException {
+    Ref targetRef = null;
+    for (String ref : refs) {
+      targetRef = repo.exactRef(ref);
+      if (targetRef != null) {
+        break;
+      }
+    }
+    return targetRef;
+  }
+
+  private boolean runningOnCircleCI() {
+    return "true".equals(system2.envVariable("CIRCLECI"));
+  }
+
+  @Override
+  public Path relativePathFromScmRoot(Path path) {
+    RepositoryBuilder builder = getVerifiedRepositoryBuilder(path);
+    return builder.getGitDir().toPath().getParent().relativize(path);
+  }
+
+  @Override
+  @CheckForNull
+  public String revisionId(Path path) {
+    RepositoryBuilder builder = getVerifiedRepositoryBuilder(path);
+    try {
+      Ref head = getHead(builder.build());
+      if (head == null || head.getObjectId() == null) {
+        // can happen on fresh, empty repos
+        return null;
+      }
+      return head.getObjectId().getName();
+    } catch (IOException e) {
+      throw new IllegalStateException("I/O error while getting revision ID for path: " + path, e);
+    }
+  }
+
+  private static boolean isDiffAlgoInvalid(Config cfg) {
+    try {
+      DiffAlgorithm.getAlgorithm(cfg.getEnum(
+        ConfigConstants.CONFIG_DIFF_SECTION, null,
+        ConfigConstants.CONFIG_KEY_ALGORITHM,
+        DiffAlgorithm.SupportedAlgorithm.HISTOGRAM));
+      return false;
+    } catch (IllegalArgumentException e) {
+      return true;
+    }
+  }
+
+  private static AbstractTreeIterator prepareNewTree(Repository repo) throws IOException {
+    CanonicalTreeParser treeParser = new CanonicalTreeParser();
+    try (ObjectReader objectReader = repo.newObjectReader()) {
+      Ref head = getHead(repo);
+      if (head == null) {
+        throw new IOException("HEAD reference not found");
+      }
+      treeParser.reset(objectReader, repo.parseCommit(head.getObjectId()).getTree());
+    }
+    return treeParser;
+  }
+
+  @CheckForNull
+  private static Ref getHead(Repository repo) throws IOException {
+    return repo.exactRef("HEAD");
+  }
+
+  private static Optional<RevCommit> findMergeBase(Repository repo, Ref targetRef) throws IOException {
+    try (RevWalk walk = new RevWalk(repo)) {
+      Ref head = getHead(repo);
+      if (head == null) {
+        throw new IOException("HEAD reference not found");
+      }
+
+      walk.markStart(walk.parseCommit(targetRef.getObjectId()));
+      walk.markStart(walk.parseCommit(head.getObjectId()));
+      walk.setRevFilter(RevFilter.MERGE_BASE);
+      RevCommit next = walk.next();
+      if (next == null) {
+        return Optional.empty();
+      }
+      RevCommit base = walk.parseCommit(next);
+      walk.dispose();
+      LOG.debug("Merge base sha1: {}", base.getName());
+      return Optional.of(base);
+    }
+  }
+
+  AbstractTreeIterator prepareTreeParser(Repository repo, RevCommit commit) throws IOException {
+    CanonicalTreeParser treeParser = new CanonicalTreeParser();
+    try (ObjectReader objectReader = repo.newObjectReader()) {
+      treeParser.reset(objectReader, commit.getTree());
+    }
+    return treeParser;
+  }
+
+  Git newGit(Repository repo) {
+    return new Git(repo);
+  }
+
+  Repository buildRepo(Path basedir) throws IOException {
+    return getVerifiedRepositoryBuilder(basedir).build();
+  }
+
+  static RepositoryBuilder getVerifiedRepositoryBuilder(Path basedir) {
+    RepositoryBuilder builder = new RepositoryBuilder()
+      .findGitDir(basedir.toFile())
+      .setMustExist(true);
+
+    if (builder.getGitDir() == null) {
+      throw MessageException.of("Not inside a Git work tree: " + basedir);
+    }
+    return builder;
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java
new file mode 100644
index 00000000000..b5a845edc9a
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java
@@ -0,0 +1,34 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.util.Arrays;
+import java.util.List;
+import org.eclipse.jgit.util.FS;
+
+public final class GitScmSupport {
+  public static List<Class<?>> getClasses() {
+    FS.FileStoreAttributes.setBackground(true);
+    return Arrays.asList(
+      JGitBlameCommand.class,
+      GitScmProvider.class,
+      GitIgnoreCommand.class);
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java
new file mode 100644
index 00000000000..cbca28cc628
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitThreadFactory.java
@@ -0,0 +1,36 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory;
+import java.util.concurrent.ForkJoinWorkerThread;
+
+public class GitThreadFactory implements ForkJoinWorkerThreadFactory {
+  private static final String NAME_PREFIX = "git-scm-";
+  private int i = 0;
+
+  @Override
+  public ForkJoinWorkerThread newThread(ForkJoinPool pool) {
+    ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
+    thread.setName(NAME_PREFIX + i++);
+    return thread;
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java
new file mode 100644
index 00000000000..2d30df40513
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java
@@ -0,0 +1,72 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.treewalk.FileTreeIterator;
+import org.eclipse.jgit.treewalk.TreeWalk;
+import org.eclipse.jgit.treewalk.WorkingTreeIterator;
+import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+public class IncludedFilesRepository {
+
+  private static final Logger LOG = Loggers.get(IncludedFilesRepository.class);
+  private final Set<Path> includedFiles = new HashSet<>();
+
+  public IncludedFilesRepository(Path baseDir) throws IOException {
+    indexFiles(baseDir);
+    LOG.debug("{} non excluded files in this Git repository", includedFiles.size());
+  }
+
+  public boolean contains(Path absolutePath) {
+    return includedFiles.contains(absolutePath);
+  }
+
+  private void indexFiles(Path baseDir) throws IOException {
+    try (Repository repo = JGitUtils.buildRepository(baseDir)) {
+      Path workTreeRoot = repo.getWorkTree().toPath();
+      FileTreeIterator workingTreeIt = new FileTreeIterator(repo);
+      try (TreeWalk treeWalk = new TreeWalk(repo)) {
+        treeWalk.setRecursive(true);
+        if (!baseDir.equals(workTreeRoot)) {
+          Path relativeBaseDir = workTreeRoot.relativize(baseDir);
+          treeWalk.setFilter(PathFilterGroup.createFromStrings(relativeBaseDir.toString().replace('\\', '/')));
+        }
+        treeWalk.addTree(workingTreeIt);
+        while (treeWalk.next()) {
+
+          WorkingTreeIterator workingTreeIterator = treeWalk
+            .getTree(0, WorkingTreeIterator.class);
+
+          if (!workingTreeIterator.isEntryIgnored()) {
+            includedFiles.add(workTreeRoot.resolve(treeWalk.getPathString()));
+          }
+        }
+      }
+    }
+  }
+
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java
new file mode 100644
index 00000000000..601a65413e5
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java
@@ -0,0 +1,129 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.blame.BlameResult;
+import org.eclipse.jgit.diff.RawTextComparator;
+import org.eclipse.jgit.lib.Repository;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.scm.BlameCommand;
+import org.sonar.api.batch.scm.BlameLine;
+import org.sonar.api.notifications.AnalysisWarnings;
+import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+public class JGitBlameCommand extends BlameCommand {
+
+  private static final Logger LOG = Loggers.get(JGitBlameCommand.class);
+
+  private final PathResolver pathResolver;
+  private final AnalysisWarnings analysisWarnings;
+
+  public JGitBlameCommand(PathResolver pathResolver, AnalysisWarnings analysisWarnings) {
+    this.pathResolver = pathResolver;
+    this.analysisWarnings = analysisWarnings;
+  }
+
+  @Override
+  public void blame(BlameInput input, BlameOutput output) {
+    File basedir = input.fileSystem().baseDir();
+    try (Repository repo = JGitUtils.buildRepository(basedir.toPath()); Git git = Git.wrap(repo)) {
+      File gitBaseDir = repo.getWorkTree();
+
+      if (cloneIsInvalid(gitBaseDir)) {
+        return;
+      }
+
+      Stream<InputFile> stream = StreamSupport.stream(input.filesToBlame().spliterator(), true);
+      ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), new GitThreadFactory(), null, false);
+      forkJoinPool.submit(() -> stream.forEach(inputFile -> blame(output, git, gitBaseDir, inputFile)));
+      try {
+        forkJoinPool.shutdown();
+        forkJoinPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        LOG.info("Git blame interrupted");
+      }
+    }
+  }
+
+  private boolean cloneIsInvalid(File gitBaseDir) {
+    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/objects/info/alternates"))) {
+      LOG.info("This git repository references another local repository which is not well supported. SCM information might be missing for some files. "
+        + "You can avoid borrow objects from another local repository by not using --reference or --shared when cloning it.");
+    }
+
+    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/shallow"))) {
+      LOG.warn("Shallow clone detected, no blame information will be provided. "
+        + "You can convert to non-shallow with 'git fetch --unshallow'.");
+      analysisWarnings.addUnique("Shallow clone detected during the analysis. "
+        + "Some files will miss SCM information. This will affect features like auto-assignment of issues. "
+        + "Please configure your build to disable shallow clone.");
+      return true;
+    }
+
+    return false;
+  }
+
+  private void blame(BlameOutput output, Git git, File gitBaseDir, InputFile inputFile) {
+    String filename = pathResolver.relativePath(gitBaseDir, inputFile.file());
+    LOG.debug("Blame file {}", filename);
+    BlameResult blameResult;
+    try {
+      blameResult = git.blame()
+        // Equivalent to -w command line option
+        .setTextComparator(RawTextComparator.WS_IGNORE_ALL)
+        .setFilePath(filename).call();
+    } catch (Exception e) {
+      throw new IllegalStateException("Unable to blame file " + inputFile.relativePath(), e);
+    }
+    List<BlameLine> lines = new ArrayList<>();
+    if (blameResult == null) {
+      LOG.debug("Unable to blame file {}. It is probably a symlink.", inputFile.relativePath());
+      return;
+    }
+    for (int i = 0; i < blameResult.getResultContents().size(); i++) {
+      if (blameResult.getSourceAuthor(i) == null || blameResult.getSourceCommit(i) == null) {
+        LOG.debug("Unable to blame file {}. No blame info at line {}. Is file committed? [Author: {} Source commit: {}]", inputFile.relativePath(), i + 1,
+          blameResult.getSourceAuthor(i), blameResult.getSourceCommit(i));
+        return;
+      }
+      lines.add(new BlameLine()
+        .date(blameResult.getSourceCommitter(i).getWhen())
+        .revision(blameResult.getSourceCommit(i).getName())
+        .author(blameResult.getSourceAuthor(i).getEmailAddress()));
+    }
+    if (lines.size() == inputFile.lines() - 1) {
+      // SONARPLUGINS-3097 Git do not report blame on last empty line
+      lines.add(lines.get(lines.size() - 1));
+    }
+    output.blameResult(inputFile, lines);
+  }
+
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java
new file mode 100644
index 00000000000..86d8792b53a
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitUtils.java
@@ -0,0 +1,44 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.Repository;
+
+public class JGitUtils {
+
+  private JGitUtils() {
+  }
+
+  public static Repository buildRepository(Path basedir) {
+    try {
+      Repository repo = GitScmProvider.getVerifiedRepositoryBuilder(basedir).build();
+      try (ObjectReader objReader = repo.getObjectDatabase().newReader()) {
+        // SONARSCGIT-2 Force initialization of shallow commits to avoid later concurrent modification issue
+        objReader.getShallowCommits();
+        return repo;
+      }
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to open Git repository", e);
+    }
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java
new file mode 100644
index 00000000000..aa91667ae34
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2020 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.scm.git;
+
+import javax.annotation.ParametersAreNonnullByDefault;