SONAR-6785 Optimize project sync with multithreading

diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java
index 1b03acd1cea484d633986e09e7271b40ea5b18f1..9e2e7da5244979611ee0913bcbbaf811ba523b3e 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java
@@ -50,10 +50,10 @@ public class WSLoader {
     SERVER_FIRST, CACHE_FIRST, SERVER_ONLY, CACHE_ONLY;
   }
 
-  private LoadStrategy loadStrategy;
+  private final LoadStrategy loadStrategy;
   private ServerStatus serverStatus;
-  private ServerClient client;
-  private PersistentCache cache;
+  private final ServerClient client;
+  private final PersistentCache cache;
 
   public WSLoader(LoadStrategy strategy, PersistentCache cache, ServerClient client) {
     this.loadStrategy = strategy;

diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java
index 47b0d2dfd32379ae607269053ee90b4792ecf7da..58c98c52f685e307762827cfe4f8792fb0ef7f0a 100644 (file)
--- a/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/cache/ProjectCacheSynchronizer.java
@@ -20,7 +20,6 @@
 package org.sonar.batch.cache;
 
 import org.apache.commons.lang.mutable.MutableBoolean;
-
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.utils.log.Loggers;
 import org.slf4j.Logger;
@@ -37,6 +36,10 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import org.sonar.batch.protocol.input.ProjectRepositories;
 import org.sonar.api.batch.bootstrap.ProjectDefinition;
@@ -47,6 +50,8 @@ import org.sonar.batch.repository.ProjectRepositoriesLoader;
 
 public class ProjectCacheSynchronizer {
   private static final Logger LOG = LoggerFactory.getLogger(ProjectCacheSynchronizer.class);
+  private static final int NUM_THREAD = 2;
+  
   private ProjectDefinition project;
   private AnalysisProperties properties;
   private ProjectRepositoriesLoader projectRepositoryLoader;
@@ -121,12 +126,12 @@ public class ProjectCacheSynchronizer {
   }
 
   private void loadLineHashes(Map<String, Map<String, FileData>> fileDataByModuleAndPath, Profiler profiler) {
+    ExecutorService executor = Executors.newFixedThreadPool(NUM_THREAD);
     int numFiles = 0;
 
     for (Map<String, FileData> fileDataByPath : fileDataByModuleAndPath.values()) {
       numFiles += fileDataByPath.size();
     }
-
     profiler.startInfo("Load line file hashes (" + numFiles + " files)");
 
     for (Entry<String, Map<String, FileData>> e1 : fileDataByModuleAndPath.entrySet()) {
@@ -134,13 +139,40 @@ public class ProjectCacheSynchronizer {
 
       for (Entry<String, FileData> e2 : e1.getValue().entrySet()) {
         String filePath = e2.getKey();
-        lineHashesLoader.getLineHashes(getComponentKey(moduleKey, filePath), null);
+        executor.submit(new LineHashLoadWorker(getComponentKey(moduleKey, filePath)));
       }
     }
 
+    executor.shutdown();
+
+    try {
+      boolean done = executor.awaitTermination(30, TimeUnit.MINUTES);
+      if (!done) {
+        executor.shutdownNow();
+        throw new IllegalStateException("Timeout while fetching line hashes");
+      }
+    } catch (InterruptedException e) {
+      executor.shutdownNow();
+      throw new IllegalStateException("Interrupted while fetching line hashes", e);
+    }
+
     profiler.stopInfo("Load line file hashes (done)");
   }
 
+  private class LineHashLoadWorker implements Callable<Void> {
+    private String fileKey;
+
+    LineHashLoadWorker(String fileKey) {
+      this.fileKey = fileKey;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      lineHashesLoader.getLineHashes(fileKey, null);
+      return null;
+    }
+  }
+
   private static class UserLoginAccumulator implements Function<ServerIssue, Void> {
     Set<String> loginSet = new HashSet<>();
 

diff --git a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java
index 918f2b1a147f1e4df250f6b8d96375b7e37be2b9..2193f5a1ba454893e4fefe51e5fd4724c856fe2f 100644 (file)
--- a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java
+++ b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java
@@ -46,12 +46,11 @@ public class PersistentCache {
   private static final String DIGEST_ALGO = "MD5";
   private static final String LOCK_FNAME = ".lock";
 
-  private Path baseDir;
-
   // eviction strategy is to expire entries after modification once a time duration has elapsed
   private final long defaultDurationToExpireMs;
   private final Logger logger;
   private final String version;
+  private final Path baseDir;
 
   public PersistentCache(Path baseDir, long defaultDurationToExpireMs, Logger logger, String version) {
     this.baseDir = baseDir;
@@ -63,7 +62,7 @@ public class PersistentCache {
     logger.debug("cache: " + baseDir + ", default expiration time (ms): " + defaultDurationToExpireMs);
   }
 
-  public void reconfigure() {
+  public synchronized void reconfigure() {
     try {
       Files.createDirectories(baseDir);
     } catch (IOException e) {