SONAR-5642 Enable multi-threaded blame to speed up processing

diff --git a/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/GitBlameCommand.java b/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/GitBlameCommand.java
index 0e7c05f6b0549a52fe363636cb2ddb9344439785..9e305722fdc00291728b6c7e2cbcfcb716d8ad68 100644 (file)
--- a/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/GitBlameCommand.java
+++ b/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/GitBlameCommand.java
@@ -19,6 +19,7 @@
  */
 package org.sonar.plugins.scm.git;
 
+import org.eclipse.jgit.api.errors.GitAPIException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.batch.fs.FileSystem;
@@ -31,7 +32,13 @@ import org.sonar.api.utils.command.StreamConsumer;
 import org.sonar.api.utils.command.StringStreamConsumer;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 public class GitBlameCommand extends BlameCommand {
 
@@ -50,23 +57,51 @@ public class GitBlameCommand extends BlameCommand {
   public void blame(BlameInput input, BlameOutput output) {
     FileSystem fs = input.fileSystem();
     LOG.debug("Working directory: " + fs.baseDir().getAbsolutePath());
+    ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() + 1);
+    List<Future<Void>> tasks = new ArrayList<Future<Void>>();
     for (InputFile inputFile : input.filesToBlame()) {
-      String filename = inputFile.relativePath();
-      Command cl = createCommandLine(fs.baseDir(), filename);
-      GitBlameConsumer consumer = new GitBlameConsumer(filename);
-      StringStreamConsumer stderr = new StringStreamConsumer();
-
-      int exitCode = execute(cl, consumer, stderr);
-      if (exitCode != 0) {
-        throw new IllegalStateException("The git blame command [" + cl.toString() + "] failed: " + stderr.getOutput());
+      tasks.add(submitTask(fs.baseDir(), output, inputFile, executorService));
+    }
+    for (Future<Void> task : tasks) {
+      try {
+        task.get();
+      } catch (ExecutionException e) {
+        // Unwrap ExecutionException
+        throw e.getCause() instanceof RuntimeException ? (RuntimeException) e.getCause() : new IllegalStateException(e.getCause());
+      } catch (InterruptedException e) {
+        throw new IllegalStateException(e);
       }
-      List<BlameLine> lines = consumer.getLines();
-      if (lines.size() == inputFile.lines() - 1) {
-        // SONARPLUGINS-3097 Git do not report blame on last empty line
-        lines.add(lines.get(lines.size() - 1));
+    }
+  }
+
+  private Future<Void> submitTask(final File baseDir, final BlameOutput output, final InputFile inputFile, ExecutorService executorService) {
+    return executorService.submit(new Callable<Void>() {
+      @Override
+      public Void call() throws GitAPIException {
+        blame(baseDir, output, inputFile);
+        return null;
       }
-      output.blameResult(inputFile, lines);
+
+    });
+  }
+
+  private void blame(File baseDir, BlameOutput output, InputFile inputFile) {
+    String filename = inputFile.relativePath();
+    Command cl = createCommandLine(baseDir, filename);
+    GitBlameConsumer consumer = new GitBlameConsumer(filename);
+    StringStreamConsumer stderr = new StringStreamConsumer();
+
+    int exitCode = execute(cl, consumer, stderr);
+    if (exitCode != 0) {
+      throw new IllegalStateException("The git blame command [" + cl.toString() + "] failed: " + stderr.getOutput());
     }
+    List<BlameLine> lines = consumer.getLines();
+    if (lines.size() == inputFile.lines() - 1) {
+      // SONARPLUGINS-3097 Git do not report blame on last empty line
+      lines.add(lines.get(lines.size() - 1));
+    }
+    output.blameResult(inputFile, lines);
+
   }
 
   public int execute(Command cl, StreamConsumer consumer, StreamConsumer stderr) {

diff --git a/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/JGitBlameCommand.java b/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/JGitBlameCommand.java
index 9f69f924fdb947a5ee4f0d32ebfd195ad8a65857..4e7f15e3e0708484ba719734c85b84eaf38a0050 100644 (file)
--- a/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/JGitBlameCommand.java
+++ b/plugins/sonar-git-plugin/src/main/java/org/sonar/plugins/scm/git/JGitBlameCommand.java
@@ -35,6 +35,11 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 public class JGitBlameCommand extends BlameCommand {
 
@@ -57,13 +62,23 @@ public class JGitBlameCommand extends BlameCommand {
         .build();
       git = Git.wrap(repo);
       File gitBaseDir = repo.getWorkTree();
+      ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() + 1);
+      List<Future<Void>> tasks = new ArrayList<Future<Void>>();
       for (InputFile inputFile : input.filesToBlame()) {
-        blame(output, git, gitBaseDir, inputFile);
+        tasks.add(submitTask(output, git, gitBaseDir, inputFile, executorService));
+      }
+      for (Future<Void> task : tasks) {
+        try {
+          task.get();
+        } catch (ExecutionException e) {
+          // Unwrap ExecutionException
+          throw e.getCause() instanceof RuntimeException ? (RuntimeException) e.getCause() : new IllegalStateException(e.getCause());
+        } catch (InterruptedException e) {
+          throw new IllegalStateException(e);
+        }
       }
     } catch (IOException e) {
       throw new IllegalStateException("Unable to open Git repository", e);
-    } catch (GitAPIException e) {
-      throw new IllegalStateException("Unable to blame", e);
     } finally {
       if (git != null && git.getRepository() != null) {
         git.getRepository().close();
@@ -71,6 +86,16 @@ public class JGitBlameCommand extends BlameCommand {
     }
   }
 
+  private Future<Void> submitTask(final BlameOutput output, final Git git, final File gitBaseDir, final InputFile inputFile, ExecutorService executorService) {
+    return executorService.submit(new Callable<Void>() {
+      @Override
+      public Void call() throws GitAPIException {
+        blame(output, git, gitBaseDir, inputFile);
+        return null;
+      }
+    });
+  }
+
   private void blame(BlameOutput output, Git git, File gitBaseDir, InputFile inputFile) throws GitAPIException {
     String filename = pathResolver.relativePath(gitBaseDir, inputFile.file());
     org.eclipse.jgit.blame.BlameResult blameResult = git.blame()

diff --git a/plugins/sonar-svn-plugin/src/main/java/org/sonar/plugins/scm/svn/SvnBlameCommand.java b/plugins/sonar-svn-plugin/src/main/java/org/sonar/plugins/scm/svn/SvnBlameCommand.java
index 27647c8cad2426092d6ee400ad6b9292c658b42f..d34f1c26740b6eba014cada022e167783b0520f9 100644 (file)
--- a/plugins/sonar-svn-plugin/src/main/java/org/sonar/plugins/scm/svn/SvnBlameCommand.java
+++ b/plugins/sonar-svn-plugin/src/main/java/org/sonar/plugins/scm/svn/SvnBlameCommand.java
@@ -74,7 +74,7 @@ public class SvnBlameCommand extends BlameCommand {
         // Unwrap ExecutionException
         throw e.getCause() instanceof RuntimeException ? (RuntimeException) e.getCause() : new IllegalStateException(e.getCause());
       } catch (InterruptedException e) {
-        // Ignore
+        throw new IllegalStateException(e);
       }
     }
   }