SONAR-2507 Batch must load plugins without connecting to database

3 files changed, 124 insertions, 102 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ArtifactDownloader.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ArtifactDownloader.java
index 6aa16b1c1e1..b8ae7330848 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ArtifactDownloader.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ArtifactDownloader.java
@@ -19,18 +19,26 @@
  */
 package org.sonar.batch.bootstrap;
 
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.CharUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.utils.HttpDownloader;
 import org.sonar.api.utils.SonarException;
 import org.sonar.batch.ServerMetadata;
-import org.sonar.core.plugin.JpaPluginFile;
 
 import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.List;
 
 public class ArtifactDownloader implements BatchComponent {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ArtifactDownloader.class);
+
   private HttpDownloader httpDownloader;
   private TempDirectories workingDirectories;
   private String baseUrl;
@@ -45,6 +53,7 @@ public class ArtifactDownloader implements BatchComponent {
     String url = baseUrl + "/deploy/jdbc-driver.jar";
     try {
       File jdbcDriver = new File(workingDirectories.getRoot(), "jdbc-driver.jar");
+      LOG.info("Download JDBC driver to " + jdbcDriver);
       httpDownloader.download(new URI(url), jdbcDriver);
       return jdbcDriver;
 
@@ -53,15 +62,87 @@ public class ArtifactDownloader implements BatchComponent {
     }
   }
 
-  public File downloadExtension(JpaPluginFile extension) {
-    File targetFile = new File(workingDirectories.getDir(extension.getPluginKey()), extension.getFilename());
-    String url = baseUrl + "/deploy/plugins/" + extension.getPluginKey() + "/" + extension.getFilename();
+  public File downloadPlugin(RemotePluginLocation remote) {
+    File targetFile = new File(workingDirectories.getDir("plugins/" + remote.getPluginKey()), remote.getFilename());
+    String url = baseUrl + "/deploy/plugins/" + remote.getRemotePath();
     try {
+      FileUtils.forceMkdir(targetFile.getParentFile());
+      LOG.info("Download plugin to " + targetFile);
       httpDownloader.download(new URI(url), targetFile);
       return targetFile;
 
-    } catch (URISyntaxException e) {
+    } catch (Exception e) {
       throw new SonarException("Fail to download extension: " + url, e);
     }
   }
+
+  public List<RemotePluginLocation> downloadPluginIndex() {
+    String url = baseUrl + "/deploy/plugins/index.txt";
+    try {
+      String indexContent = httpDownloader.downloadPlainText(new URI(url), "UTF-8");
+      String[] rows = StringUtils.split(indexContent, CharUtils.LF);
+      List<RemotePluginLocation> remoteLocations = Lists.newArrayList();
+      for (String row : rows) {
+        remoteLocations.add(RemotePluginLocation.createFromRow(row));
+      }
+      return remoteLocations;
+
+    } catch (Exception e) {
+      throw new SonarException("Fail to download plugins index: " + url, e);
+    }
+  }
+
+  public static final class RemotePluginLocation {
+    private String pluginKey;
+    private String remotePath;
+    private boolean core;
+
+    private RemotePluginLocation(String pluginKey, String remotePath, boolean core) {
+      this.pluginKey = pluginKey;
+      this.remotePath = remotePath;
+      this.core = core;
+    }
+
+    static RemotePluginLocation create(String key) {
+      return new RemotePluginLocation(key, null, false);
+    }
+
+    static RemotePluginLocation createFromRow(String row) {
+      String[] fields = StringUtils.split(row, ",");
+      return new RemotePluginLocation(fields[0], fields[1], Boolean.parseBoolean(fields[2]));
+    }
+
+    public String getPluginKey() {
+      return pluginKey;
+    }
+
+    public String getRemotePath() {
+      return remotePath;
+    }
+
+    public String getFilename() {
+      return StringUtils.substringAfterLast(remotePath, "/");
+    }
+
+    public boolean isCore() {
+      return core;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      RemotePluginLocation that = (RemotePluginLocation) o;
+      return pluginKey.equals(that.pluginKey);
+    }
+
+    @Override
+    public int hashCode() {
+      return pluginKey.hashCode();
+    }
+  }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
index 99a7fc88aee..2452983bd11 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginRepository.java
@@ -20,10 +20,9 @@
 package org.sonar.batch.bootstrap;
 
 import com.google.common.base.Joiner;
-import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import org.apache.commons.configuration.Configuration;
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,125 +30,64 @@ import org.sonar.api.CoreProperties;
 import org.sonar.api.Plugin;
 import org.sonar.api.Properties;
 import org.sonar.api.Property;
+import org.sonar.api.platform.PluginMetadata;
 import org.sonar.api.platform.PluginRepository;
-import org.sonar.api.utils.SonarException;
-import org.sonar.core.classloaders.ClassLoadersCollection;
-import org.sonar.core.plugin.JpaPlugin;
-import org.sonar.core.plugin.JpaPluginDao;
-import org.sonar.core.plugin.JpaPluginFile;
+import org.sonar.core.plugins.PluginClassloaders;
+import org.sonar.core.plugins.PluginFileExtractor;
 
 import java.io.File;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.*;
 
 public class BatchPluginRepository implements PluginRepository {
 
   private static final Logger LOG = LoggerFactory.getLogger(BatchPluginRepository.class);
 
-  private JpaPluginDao dao;
   private ArtifactDownloader artifactDownloader;
   private Map<String, Plugin> pluginsByKey;
+  private Map<String, PluginMetadata> metadataByKey;
   private Set<String> whiteList = null;
   private Set<String> blackList = null;
+  private PluginClassloaders classLoaders;
 
-  public BatchPluginRepository(JpaPluginDao dao, ArtifactDownloader artifactDownloader, Configuration configuration) {
-    this.dao = dao;
+  public BatchPluginRepository(ArtifactDownloader artifactDownloader, Configuration configuration) {
     this.artifactDownloader = artifactDownloader;
-    if (configuration.getString(CoreProperties.INCLUDE_PLUGINS)!=null) {
+    if (configuration.getString(CoreProperties.INCLUDE_PLUGINS) != null) {
       whiteList = Sets.newTreeSet(Arrays.asList(configuration.getStringArray(CoreProperties.INCLUDE_PLUGINS)));
       LOG.info("Include plugins: " + Joiner.on(", ").join(whiteList));
     }
-    if (configuration.getString(CoreProperties.EXCLUDE_PLUGINS)!=null) {
+    if (configuration.getString(CoreProperties.EXCLUDE_PLUGINS) != null) {
       blackList = Sets.newTreeSet(Arrays.asList(configuration.getStringArray(CoreProperties.EXCLUDE_PLUGINS)));
       LOG.info("Exclude plugins: " + Joiner.on(", ").join(blackList));
     }
-//  TODO reactivate somewhere else:  LOG.info("Execution environment: {} {}", environment.getKey(), environment.getVersion());
+    // TODO reactivate somewhere else:  LOG.info("Execution environment: {} {}", environment.getKey(), environment.getVersion());
   }
 
   public void start() {
-    List<JpaPlugin> plugins = filter(dao.getPlugins());
-    LOG.debug("Starting plugins: " + Joiner.on(", ").join(plugins));
-    doStart(plugins);
+    doStart(artifactDownloader.downloadPluginIndex());
   }
 
-  List<JpaPlugin> filter(List<JpaPlugin> plugins) {
-    return ImmutableList.copyOf(Iterables.filter(plugins, new Predicate<JpaPlugin>() {
-      public boolean apply(JpaPlugin p) {
-        return isAccepted(p.getKey()) && (StringUtils.isBlank(p.getBasePlugin()) || isAccepted(p.getBasePlugin()));
-      }
-    }));
-  }
-
-  public void doStart(List<JpaPlugin> basePlugins) {
-    pluginsByKey = Maps.newHashMap();
-    ClassLoadersCollection classLoaders = new ClassLoadersCollection(Thread.currentThread().getContextClassLoader());
-
-    List<JpaPlugin> pluginsMetadata = Lists.newArrayList(basePlugins);
-    createClassloaders(classLoaders, basePlugins);
-    pluginsMetadata.addAll(extendClassloaders(classLoaders, pluginsMetadata));
-    instantiatePluginEntryPoints(classLoaders, pluginsMetadata);
-
-    classLoaders.done();
-  }
-
-  private void instantiatePluginEntryPoints(ClassLoadersCollection classLoaders, List<JpaPlugin> pluginsMetadata) {
-    for (JpaPlugin pluginMetadata : pluginsMetadata) {
-      try {
-        Class claz = classLoaders.get(pluginMetadata.getKey()).loadClass(pluginMetadata.getPluginClass());
-        Plugin plugin = (Plugin) claz.newInstance();
-        pluginsByKey.put(pluginMetadata.getKey(), plugin);
-
-      } catch (Exception e) {
-        throw new SonarException("Fail to load plugin " + pluginMetadata.getKey(), e);
-      }
-    }
-  }
-
-  private List<JpaPlugin> extendClassloaders(ClassLoadersCollection classLoaders, List<JpaPlugin> pluginsMetadata) {
-    List<JpaPlugin> extensions = Lists.newArrayList();
-    // Extend plugins by other plugins
-    for (JpaPlugin pluginMetadata : pluginsMetadata) {
-      String pluginKey = pluginMetadata.getKey();
-      String basePluginKey = pluginMetadata.getBasePlugin();
-      if (StringUtils.isNotEmpty(basePluginKey)) {
-        if (classLoaders.get(basePluginKey) != null) {
-          LOG.debug("Plugin {} extends {}", pluginKey, basePluginKey);
-          List<URL> urls = download(pluginMetadata);
-          classLoaders.extend(basePluginKey, pluginKey, urls);
-          extensions.add(pluginMetadata);
-
-        } else {
-          // Ignored, because base plugin doesn't exists
-          LOG.warn("Plugin {} extends nonexistent plugin {}", pluginKey, basePluginKey);
+  void doStart(List<ArtifactDownloader.RemotePluginLocation> remoteLocations) {
+    PluginFileExtractor extractor = new PluginFileExtractor();
+    metadataByKey = Maps.newHashMap();
+    for (ArtifactDownloader.RemotePluginLocation remoteLocation : remoteLocations) {
+      if (isAccepted(remoteLocation.getPluginKey())) {
+        File pluginFile = artifactDownloader.downloadPlugin(remoteLocation);
+        PluginMetadata metadata = extractor.installInSameLocation(pluginFile, remoteLocation.isCore());
+        if (StringUtils.isBlank(metadata.getBasePlugin()) || isAccepted(metadata.getBasePlugin())) {
+          // TODO log when excluding plugin
+          metadataByKey.put(metadata.getKey(), metadata);
         }
       }
     }
-    return extensions;
+    classLoaders = new PluginClassloaders(Thread.currentThread().getContextClassLoader());
+    pluginsByKey = classLoaders.init(metadataByKey.values());
   }
 
-  private void createClassloaders(ClassLoadersCollection classLoaders, List<JpaPlugin> basePlugins) {
-    for (JpaPlugin pluginMetadata : basePlugins) {
-      if (StringUtils.isEmpty(pluginMetadata.getBasePlugin())) {
-        String key = pluginMetadata.getKey();
-        List<URL> urls = download(pluginMetadata);
-        classLoaders.createClassLoader(key, urls, pluginMetadata.isUseChildFirstClassLoader() == Boolean.TRUE);
-      }
-    }
-  }
-
-  private List<URL> download(JpaPlugin pluginMetadata) {
-    List<URL> urls = Lists.newArrayList();
-    for (JpaPluginFile pluginFile : pluginMetadata.getFiles()) {
-      File file = artifactDownloader.downloadExtension(pluginFile);
-      try {
-        urls.add(file.toURI().toURL());
-
-      } catch (MalformedURLException e) {
-        throw new SonarException("Can not get the URL of: " + file, e);
-      }
+  public void stop() {
+    if (classLoaders != null) {
+      classLoaders.clean();
+      classLoaders = null;
     }
-    return urls;
   }
 
   public Collection<Plugin> getPlugins() {
@@ -175,13 +113,18 @@ public class BatchPluginRepository implements PluginRepository {
     return new Property[0];
   }
 
+  public Collection<PluginMetadata> getMetadata() {
+    return metadataByKey.values();
+  }
+
+  public PluginMetadata getMetadata(String pluginKey) {
+    return metadataByKey.get(pluginKey);
+  }
+
   boolean isAccepted(String pluginKey) {
-    if (whiteList!=null) {
+    if (whiteList != null) {
       return whiteList.contains(pluginKey);
     }
-    if (blackList!=null) {
-      return !blackList.contains(pluginKey);
-    }
-    return true;
+    return blackList == null || !blackList.contains(pluginKey);
   }
 }
diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java
index fda540156a8..0eaf338f8fa 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java
@@ -26,7 +26,6 @@ import org.sonar.api.utils.HttpDownloader;
 import org.sonar.batch.FakeMavenPluginExecutor;
 import org.sonar.batch.MavenPluginExecutor;
 import org.sonar.batch.ServerMetadata;
-import org.sonar.core.plugin.JpaPluginDao;
 import org.sonar.jpa.session.DatabaseSessionProvider;
 import org.sonar.jpa.session.DriverDatabaseConnector;
 import org.sonar.jpa.session.ThreadLocalDatabaseSessionFactory;
@@ -74,7 +73,6 @@ public class BootstrapModule extends Module {
 
     // LIMITATION : list of plugins to download is currently loaded from database. It should be loaded from
     // remote HTTP index.
-    addComponent(JpaPluginDao.class);
     addComponent(BatchPluginRepository.class);
     addComponent(BatchExtensionInstaller.class);
     addComponent(ProjectExtensionInstaller.class);