SONAR-16290 Use native git to collect blame information

5 files changed, 234 insertions, 188 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java
new file mode 100644
index 00000000000..83223959ee8
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/CompositeBlameCommand.java
@@ -0,0 +1,119 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.lib.Repository;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.scm.BlameCommand;
+import org.sonar.api.batch.scm.BlameLine;
+import org.sonar.api.notifications.AnalysisWarnings;
+import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+public class CompositeBlameCommand extends BlameCommand {
+  private static final Logger LOG = Loggers.get(CompositeBlameCommand.class);
+
+  private final AnalysisWarnings analysisWarnings;
+  private final PathResolver pathResolver;
+  private final JGitBlameCommand jgitCmd;
+  private final GitBlameCommand nativeCmd;
+  private boolean nativeGitEnabled = false;
+
+  public CompositeBlameCommand(AnalysisWarnings analysisWarnings, PathResolver pathResolver, JGitBlameCommand jgitCmd, GitBlameCommand nativeCmd) {
+    this.analysisWarnings = analysisWarnings;
+    this.pathResolver = pathResolver;
+    this.jgitCmd = jgitCmd;
+    this.nativeCmd = nativeCmd;
+  }
+
+  @Override
+  public void blame(BlameInput input, BlameOutput output) {
+    File basedir = input.fileSystem().baseDir();
+    try (Repository repo = JGitUtils.buildRepository(basedir.toPath()); Git git = Git.wrap(repo)) {
+      File gitBaseDir = repo.getWorkTree();
+      if (cloneIsInvalid(gitBaseDir)) {
+        return;
+      }
+      nativeGitEnabled = nativeCmd.isEnabled(basedir.toPath());
+      Stream<InputFile> stream = StreamSupport.stream(input.filesToBlame().spliterator(), true);
+      ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), new GitThreadFactory(), null, false);
+      // exceptions thrown by the blame method will be ignored
+      forkJoinPool.submit(() -> stream.forEach(inputFile -> blame(output, git, gitBaseDir, inputFile)));
+      try {
+        forkJoinPool.shutdown();
+        forkJoinPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        LOG.info("Git blame interrupted");
+      }
+    }
+  }
+
+  private void blame(BlameOutput output, Git git, File gitBaseDir, InputFile inputFile) {
+    String filename = pathResolver.relativePath(gitBaseDir, inputFile.file());
+    LOG.debug("Blame file {}", filename);
+    List<BlameLine> blame = null;
+    if (nativeGitEnabled) {
+      try {
+        blame = nativeCmd.blame(gitBaseDir.toPath(), filename);
+      } catch (Exception e) {
+        // fallback to jgit
+      }
+    }
+
+    if (blame == null) {
+      blame = jgitCmd.blame(git, filename);
+    }
+
+    if (!blame.isEmpty()) {
+      if (blame.size() == inputFile.lines() - 1) {
+        // SONARPLUGINS-3097 Git do not report blame on last empty line
+        blame.add(blame.get(blame.size() - 1));
+      }
+      output.blameResult(inputFile, blame);
+    }
+  }
+
+  private boolean cloneIsInvalid(File gitBaseDir) {
+    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/objects/info/alternates"))) {
+      LOG.info("This git repository references another local repository which is not well supported. SCM information might be missing for some files. "
+        + "You can avoid borrow objects from another local repository by not using --reference or --shared when cloning it.");
+    }
+
+    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/shallow"))) {
+      LOG.warn("Shallow clone detected, no blame information will be provided. "
+        + "You can convert to non-shallow with 'git fetch --unshallow'.");
+      analysisWarnings.addUnique("Shallow clone detected during the analysis. "
+        + "Some files will miss SCM information. This will affect features like auto-assignment of issues. "
+        + "Please configure your build to disable shallow clone.");
+      return true;
+    }
+
+    return false;
+  }
+}
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java
index 856a1503ec9..3fffbe0ea83 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitBlameCommand.java
@@ -20,156 +20,143 @@
 package org.sonar.scm.git;
 
 import java.io.BufferedReader;
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import java.util.function.Consumer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.sonar.api.batch.scm.BlameLine;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.springframework.beans.factory.annotation.Autowired;
 
-import static java.util.Objects.requireNonNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+import static org.sonar.api.utils.Preconditions.checkState;
 
 public class GitBlameCommand {
-  private static final String AUTHOR = "author";
-  private static final String COMMIT = "commit";
-  private static final String TIMESTAMP = "timestamp";
-  private static final String TIMEZONE = "timezone";
-  private static final String LINE = "line";
+  private static final Logger LOG = Loggers.get(GitBlameCommand.class);
+  private static final Pattern EMAIL_PATTERN = Pattern.compile("<(\\S*?)>");
+  private static final String COMITTER_TIME = "committer-time ";
+  private static final String COMITTER_MAIL = "committer-mail ";
 
   private static final String GIT_COMMAND = "git";
   private static final String BLAME_COMMAND = "blame";
-  private static final String BLAME_LONG_FLAG = "-l";
-  private static final String BLAME_SHOW_EMAIL_FLAG = "--show-email";
-  private static final String BLAME_TIMESTAMP_FLAG = "-t";
+  private static final String BLAME_LINE_PORCELAIN_FLAG = "--line-porcelain";
+  private static final String IGNORE_WHITESPACES = "-w";
 
-  public static List<BlameLine> executeCommand(Path directory, String... command) throws IOException, InterruptedException {
-    requireNonNull(directory, "directory");
+  private final String gitCommand;
 
-    if (!Files.exists(directory)) {
-      throw new RuntimeException("Directory does not exist, unable to run git operations:'" + directory + "'");
+  @Autowired
+  public GitBlameCommand() {
+    this(GIT_COMMAND);
+  }
+
+  public GitBlameCommand(String gitCommand) {
+    this.gitCommand = gitCommand;
+  }
+
+  public boolean isEnabled(Path baseDir) {
+    try {
+      MutableString stdOut = new MutableString();
+      executeCommand(baseDir, l -> stdOut.string = l, gitCommand, "--version");
+      return stdOut.string != null && stdOut.string.startsWith("git version");
+    } catch (Exception e) {
+      LOG.debug("Failed to find git native client", e);
+      return false;
+    }
+  }
+
+  public List<BlameLine> blame(Path baseDir, String fileName) {
+    BlameOutputProcessor outputProcessor = new BlameOutputProcessor();
+    try {
+      executeCommand(baseDir, outputProcessor::process, gitCommand, BLAME_COMMAND, BLAME_LINE_PORCELAIN_FLAG, IGNORE_WHITESPACES, fileName);
+      return outputProcessor.getBlameLines();
+    } catch (Exception e) {
+      LOG.debug("Blame failed for " + fileName, e);
+      return emptyList();
     }
+  }
 
+  private void executeCommand(Path baseDir, Consumer<String> stdOutLineConsumer, String... command) throws Exception {
     ProcessBuilder pb = new ProcessBuilder()
       .command(command)
-      .directory(directory.toFile());
+      .directory(baseDir.toFile());
 
     Process p = pb.start();
-
-    List<String> commandOutput = new ArrayList<>();
-    InputStream processStdOutput = p.getInputStream();
-
-    try (BufferedReader br = new BufferedReader(new InputStreamReader(processStdOutput))) {
+    try {
+      InputStream processStdOutput = p.getInputStream();
+      try (BufferedReader br = new BufferedReader(new InputStreamReader(processStdOutput, UTF_8))) {
         String outputLine;
-
         while ((outputLine = br.readLine()) != null) {
-          commandOutput.add(outputLine);
+          stdOutLineConsumer.accept(outputLine);
         }
+      }
 
-        int exit = p.waitFor();
-
-        if (exit != 0) {
-          throw new AssertionError(String.format("Command execution exited with code: %d", exit));
-        }
+      int exit = p.waitFor();
+      if (exit != 0) {
+        throw new IllegalStateException(String.format("Command execution exited with code: %d", exit));
+      }
 
-    } catch (Exception e) {
-      e.printStackTrace();
     } finally {
       p.destroy();
     }
-
-    return commandOutput
-      .stream()
-      .map(GitBlameCommand::parseBlameLine)
-      .collect(Collectors.toList());
   }
 
-  private static Map<String, String> getBlameAuthoringData(String blameLine) {
-    String[] blameLineFormatted = blameLine.trim().split("\\s+", 2);
-
-    String commit = blameLineFormatted[0];
+  private static class BlameOutputProcessor {
+    private final List<BlameLine> blameLines = new ArrayList<>();
+    private String sha1;
+    private String committerTime;
+    private String committerMail;
 
-    if (commit.length() != 40) {
-      throw new IllegalStateException(String.format("Failed to fetch correct commit hash, must be of length 40: %s", commit));
+    public List<BlameLine> getBlameLines() {
+      return unmodifiableList(blameLines);
     }
 
-    String authoringData = StringUtils.substringBetween(blameLineFormatted[1], "(", ")");
-    String[] authoringDataFormatted = authoringData.trim().split("\\s+", 4);
-
-    String author = StringUtils.substringBetween(authoringDataFormatted[0], "<", ">");
-    String timestamp = authoringDataFormatted[1];
-    String timezone = authoringDataFormatted[2];
-    String line = authoringDataFormatted[3];
-
-    Map<String, String> blameData = new HashMap<>();
-
-    blameData.put(COMMIT, commit);
-    blameData.put(AUTHOR, author);
-    blameData.put(TIMESTAMP, timestamp);
-    blameData.put(TIMEZONE, timezone);
-    blameData.put(LINE, line);
-
-    return blameData;
-  }
-
-  private static BlameLine parseBlameLine(String blameLine) {
-    Map<String, String> blameData = getBlameAuthoringData(blameLine);
-
-    return new BlameLine()
-      .date(new Date(Long.parseLong(blameData.get(TIMESTAMP)))) // should also take timezone into consideration
-      .revision(blameData.get(COMMIT))
-      .author(blameData.get(AUTHOR));
-  }
-
-  public static void gitInit(Path directory) throws IOException, InterruptedException {
-    executeCommand(directory, "git", "init");
-  }
-
-  public static void gitStage(Path directory) throws IOException, InterruptedException {
-    executeCommand(directory, "git", "add", "-A");
-  }
-
-  public static void gitCommit(Path directory, String message) throws IOException, InterruptedException {
-    executeCommand(directory, GIT_COMMAND, COMMIT, "-m", message);
-  }
-
-  public static void gitClone(Path directory, String originUrl) throws IOException, InterruptedException {
-    executeCommand(directory.getParent(), "git", "clone", originUrl, directory.getFileName().toString());
-  }
-
-  public static List<BlameLine> gitBlame(Path directory, String fileName) throws IOException, InterruptedException {
-    return executeCommand(directory, GIT_COMMAND, BLAME_COMMAND, BLAME_LONG_FLAG, BLAME_SHOW_EMAIL_FLAG, BLAME_TIMESTAMP_FLAG, fileName);
-  }
-
-  private static class StreamGobbler extends Thread {
-    private final InputStream is;
-    private final String type;
-
-    private StreamGobbler(InputStream is, String type) {
-      this.is = is;
-      this.type = type;
+    public void process(String line) {
+      if (sha1 == null) {
+        sha1 = line.split(" ")[0];
+      } else if (line.startsWith("\t")) {
+        saveEntry();
+      } else if (line.startsWith(COMITTER_TIME)) {
+        committerTime = line.substring(COMITTER_TIME.length());
+      } else if (line.startsWith(COMITTER_MAIL)) {
+        Matcher matcher = EMAIL_PATTERN.matcher(line);
+        if (!matcher.find(COMITTER_MAIL.length()) || matcher.groupCount() != 1) {
+          throw new IllegalStateException("Couldn't parse committer email from: " + line);
+        }
+        committerMail = matcher.group(1);
+        if (committerMail.equals("not.committed.yet")) {
+          throw new IllegalStateException("Uncommitted line found");
+        }
+      }
     }
 
-    @Override
-    public void run() {
-      try (BufferedReader br = new BufferedReader(new InputStreamReader(is));) {
-        List<String> commandOutput = new ArrayList<>();
-        String outputLine;
-
-        while ((outputLine = br.readLine()) != null) {
-          commandOutput.add(outputLine);
-          System.out.println(type + "> " + outputLine);
-        }
-      } catch (IOException ioe) {
-        ioe.printStackTrace();
+    private void saveEntry() {
+      checkState(committerMail != null, "Did not find a committer email for an entry");
+      checkState(committerTime != null, "Did not find a committer time for an entry");
+      checkState(sha1 != null, "Did not find a commit sha1 for an entry");
+      try {
+        blameLines.add(new BlameLine()
+          .revision(sha1)
+          .author(committerMail)
+          .date(Date.from(Instant.ofEpochSecond(Long.parseLong(committerTime)))));
+      } catch (NumberFormatException e) {
+        throw new IllegalStateException("Invalid committer time found: " + committerTime);
       }
+      committerMail = null;
+      sha1 = null;
+      committerTime = null;
     }
   }
 
+  private static class MutableString {
+    String string;
+  }
 }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java
index e66e4f79303..bef15b669bc 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmProvider.java
@@ -64,14 +64,14 @@ public class GitScmProvider extends ScmProvider {
 
   private static final Logger LOG = Loggers.get(GitScmProvider.class);
   private static final String COULD_NOT_FIND_REF = "Could not find ref '%s' in refs/heads, refs/remotes, refs/remotes/upstream or refs/remotes/origin";
-  private final JGitBlameCommand jgitBlameCommand;
+  private final BlameCommand blameCommand;
   private final AnalysisWarnings analysisWarnings;
   private final GitIgnoreCommand gitIgnoreCommand;
   private final System2 system2;
   private final String documentationLink;
 
-  public GitScmProvider(JGitBlameCommand jgitBlameCommand, AnalysisWarnings analysisWarnings, GitIgnoreCommand gitIgnoreCommand, System2 system2) {
-    this.jgitBlameCommand = jgitBlameCommand;
+  public GitScmProvider(CompositeBlameCommand blameCommand, AnalysisWarnings analysisWarnings, GitIgnoreCommand gitIgnoreCommand, System2 system2) {
+    this.blameCommand = blameCommand;
     this.analysisWarnings = analysisWarnings;
     this.gitIgnoreCommand = gitIgnoreCommand;
     this.system2 = system2;
@@ -96,7 +96,7 @@ public class GitScmProvider extends ScmProvider {
 
   @Override
   public BlameCommand blameCommand() {
-    return this.jgitBlameCommand;
+    return this.blameCommand;
   }
 
   @CheckForNull
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java
index 2d412fe74b2..a67bd56320d 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/GitScmSupport.java
@@ -32,6 +32,8 @@ public final class GitScmSupport {
     FS.FileStoreAttributes.setBackground(true);
     return Arrays.asList(
       JGitBlameCommand.class,
+      CompositeBlameCommand.class,
+      GitBlameCommand.class,
       GitScmProvider.class,
       GitIgnoreCommand.class);
   }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java
index f939ceb28e4..e9868a03c52 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/JGitBlameCommand.java
@@ -19,80 +19,21 @@
  */
 package org.sonar.scm.git;
 
-import java.io.File;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.blame.BlameResult;
 import org.eclipse.jgit.diff.RawTextComparator;
-import org.eclipse.jgit.lib.Repository;
-import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.scm.BlameCommand;
 import org.sonar.api.batch.scm.BlameLine;
-import org.sonar.api.notifications.AnalysisWarnings;
-import org.sonar.api.scan.filesystem.PathResolver;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 
-public class JGitBlameCommand extends BlameCommand {
+import static java.util.Collections.emptyList;
 
+public class JGitBlameCommand {
   private static final Logger LOG = Loggers.get(JGitBlameCommand.class);
 
-  private final PathResolver pathResolver;
-  private final AnalysisWarnings analysisWarnings;
-
-  public JGitBlameCommand(PathResolver pathResolver, AnalysisWarnings analysisWarnings) {
-    this.pathResolver = pathResolver;
-    this.analysisWarnings = analysisWarnings;
-  }
-
-  @Override
-  public void blame(BlameInput input, BlameOutput output) {
-    File basedir = input.fileSystem().baseDir();
-    try (Repository repo = JGitUtils.buildRepository(basedir.toPath()); Git git = Git.wrap(repo)) {
-      File gitBaseDir = repo.getWorkTree();
-
-      if (cloneIsInvalid(gitBaseDir)) {
-        return;
-      }
-
-      Stream<InputFile> stream = StreamSupport.stream(input.filesToBlame().spliterator(), true);
-      ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), new GitThreadFactory(), null, false);
-      forkJoinPool.submit(() -> stream.forEach(inputFile -> blame(output, git, gitBaseDir, inputFile)));
-      try {
-        forkJoinPool.shutdown();
-        forkJoinPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
-      } catch (InterruptedException e) {
-        LOG.info("Git blame interrupted");
-      }
-    }
-  }
-
-  private boolean cloneIsInvalid(File gitBaseDir) {
-    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/objects/info/alternates"))) {
-      LOG.info("This git repository references another local repository which is not well supported. SCM information might be missing for some files. "
-        + "You can avoid borrow objects from another local repository by not using --reference or --shared when cloning it.");
-    }
-
-    if (Files.isRegularFile(gitBaseDir.toPath().resolve(".git/shallow"))) {
-      LOG.warn("Shallow clone detected, no blame information will be provided. "
-        + "You can convert to non-shallow with 'git fetch --unshallow'.");
-      analysisWarnings.addUnique("Shallow clone detected during the analysis. "
-        + "Some files will miss SCM information. This will affect features like auto-assignment of issues. "
-        + "Please configure your build to disable shallow clone.");
-      return true;
-    }
-
-    return false;
-  }
-
-  private void blame(BlameOutput output, Git git, File gitBaseDir, InputFile inputFile) {
-    String filename = pathResolver.relativePath(gitBaseDir, inputFile.file());
+  public List<BlameLine> blame(Git git, String filename) {
     LOG.debug("Blame file {}", filename);
     BlameResult blameResult;
     try {
@@ -101,29 +42,26 @@ public class JGitBlameCommand extends BlameCommand {
         .setTextComparator(RawTextComparator.WS_IGNORE_ALL)
         .setFilePath(filename).call();
     } catch (Exception e) {
-      throw new IllegalStateException("Unable to blame file " + inputFile.relativePath(), e);
+      throw new IllegalStateException("Unable to blame file " + filename, e);
     }
     List<BlameLine> lines = new ArrayList<>();
     if (blameResult == null) {
-      LOG.debug("Unable to blame file {}. It is probably a symlink.", inputFile.relativePath());
-      return;
+      LOG.debug("Unable to blame file {}. It is probably a symlink.", filename);
+      return emptyList();
     }
     for (int i = 0; i < blameResult.getResultContents().size(); i++) {
       if (blameResult.getSourceAuthor(i) == null || blameResult.getSourceCommit(i) == null) {
-        LOG.debug("Unable to blame file {}. No blame info at line {}. Is file committed? [Author: {} Source commit: {}]", inputFile.relativePath(), i + 1,
+        LOG.debug("Unable to blame file {}. No blame info at line {}. Is file committed? [Author: {} Source commit: {}]", filename, i + 1,
           blameResult.getSourceAuthor(i), blameResult.getSourceCommit(i));
-        return;
+        return emptyList();
       }
       lines.add(new BlameLine()
         .date(blameResult.getSourceCommitter(i).getWhen())
         .revision(blameResult.getSourceCommit(i).getName())
         .author(blameResult.getSourceAuthor(i).getEmailAddress()));
     }
-    if (lines.size() == inputFile.lines() - 1) {
-      // SONARPLUGINS-3097 Git do not report blame on last empty line
-      lines.add(lines.get(lines.size() - 1));
-    }
-    output.blameResult(inputFile, lines);
+
+    return lines;
   }
 
 }