From: Julien HENRY Date: Wed, 30 Jan 2013 09:24:51 +0000 (+0100) Subject: SONAR-2291 Generate a new bootstrap index with md5 X-Git-Tag: 3.5~321 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=9963f42534c2ed037a0c38bb72a3815313e4fb35;p=sonarqube.git SONAR-2291 Generate a new bootstrap index with md5 to allow caching by Sonar Runner. --- diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java b/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java index b503dc463df..da4caca7ee3 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java @@ -25,7 +25,6 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonar.api.utils.SonarException; import java.io.File; import java.io.FileInputStream; @@ -57,11 +56,11 @@ public class SonarCache { this.cacheLocation = cacheLocation; tmpDir = new File(cacheLocation, ".tmp"); if (!cacheLocation.exists()) { - LOG.debug("Creating cache directory: " + cacheLocation.getAbsolutePath()); + LOG.debug("Creating cache directory: {}", cacheLocation.getAbsolutePath()); try { FileUtils.forceMkdir(cacheLocation); } catch (IOException e) { - throw new SonarException("Unable to create cache directory " + cacheLocation.getAbsolutePath(), e); + throw new RuntimeException("Unable to create cache directory " + cacheLocation.getAbsolutePath(), e); } } } @@ -125,7 +124,7 @@ public class SonarCache { if (!rename) { // Check if the file was already in cache if (!finalFileName.exists()) { - LOG.warn("Unable to rename " + tmpFileName.getAbsolutePath() + " to " + finalFileName.getAbsolutePath()); + LOG.warn("Unable to rename {} to {}", tmpFileName.getAbsolutePath(), finalFileName.getAbsolutePath()); LOG.warn("A copy/delete will be tempted but with no garantee of atomicity"); FileUtils.moveFile(tmpFileName, finalFileName); } @@ -183,11 +182,11 @@ public class SonarCache { public File getTmpDir() { if (!tmpDir.exists()) { - LOG.debug("Creating temporary cache directory: " + tmpDir.getAbsolutePath()); + LOG.debug("Creating temporary cache directory: {}", tmpDir.getAbsolutePath()); try { FileUtils.forceMkdir(tmpDir); } catch (IOException e) { - throw new SonarException("Unable to create temporary cache directory " + tmpDir.getAbsolutePath(), e); + throw new RuntimeException("Unable to create temporary cache directory " + tmpDir.getAbsolutePath(), e); } } return tmpDir; diff --git a/sonar-server/src/main/java/org/sonar/server/platform/DefaultServerFileSystem.java b/sonar-server/src/main/java/org/sonar/server/platform/DefaultServerFileSystem.java index 0560513eff2..faf605a190f 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/DefaultServerFileSystem.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/DefaultServerFileSystem.java @@ -158,11 +158,14 @@ public class DefaultServerFileSystem implements ServerFileSystem { return new File(getHomeDir(), "extensions/deprecated"); } - public File getPluginIndex() { return new File(getDeployDir(), "plugins/index.txt"); } + public File getBootstrapIndex() { + return new File(getDeployDir(), "bootstrap/index.txt"); + } + public List getExtensions(String dirName, String... suffixes) { File dir = new File(getHomeDir(), "extensions/rules/" + dirName); if (dir.exists() && dir.isDirectory()) { diff --git a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java index 7d94eaf2e81..964382f6682 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java @@ -86,6 +86,7 @@ import org.sonar.server.qualitymodel.DefaultModelManager; import org.sonar.server.rules.ProfilesConsole; import org.sonar.server.rules.RulesConsole; import org.sonar.server.startup.DeleteDeprecatedMeasures; +import org.sonar.server.startup.GenerateBootstrapIndex; import org.sonar.server.startup.GeneratePluginIndex; import org.sonar.server.startup.GwtPublisher; import org.sonar.server.startup.JdbcDriverDeployer; @@ -267,7 +268,6 @@ public final class Platform { ServerExtensionInstaller extensionRegistrar = servicesContainer.getComponentByType(ServerExtensionInstaller.class); extensionRegistrar.registerExtensions(servicesContainer); - servicesContainer.startComponents(); } @@ -282,6 +282,7 @@ public final class Platform { startupContainer.addSingleton(RegisterQualityModels.class); startupContainer.addSingleton(DeleteDeprecatedMeasures.class); startupContainer.addSingleton(GeneratePluginIndex.class); + startupContainer.addSingleton(GenerateBootstrapIndex.class); startupContainer.addSingleton(RegisterNewMeasureFilters.class); startupContainer.addSingleton(RegisterNewDashboards.class); startupContainer.addSingleton(RenameDeprecatedPropertyKeys.class); diff --git a/sonar-server/src/main/java/org/sonar/server/plugins/BatchResourcesServlet.java b/sonar-server/src/main/java/org/sonar/server/plugins/BatchResourcesServlet.java index 7e40cadeb19..4e399c56ff4 100644 --- a/sonar-server/src/main/java/org/sonar/server/plugins/BatchResourcesServlet.java +++ b/sonar-server/src/main/java/org/sonar/server/plugins/BatchResourcesServlet.java @@ -19,24 +19,22 @@ */ package org.sonar.server.plugins; -import com.google.common.collect.Lists; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.util.List; -import java.util.Set; +import org.sonar.server.startup.GenerateBootstrapIndex; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; + /** * This servlet allows to load libraries from directory "WEB-INF/lib" in order to provide them for batch-bootstrapper. * Most probably this is not a best solution. @@ -54,7 +52,7 @@ public class BatchResourcesServlet extends HttpServlet { try { response.setContentType("text/plain"); writer = response.getWriter(); - writer.print(StringUtils.join(getLibs(), ',')); + writer.print(StringUtils.join(GenerateBootstrapIndex.getLibs(getServletContext()), ',')); } catch (IOException e) { LOG.error("Unable to provide list of batch resources", e); response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); @@ -83,35 +81,6 @@ public class BatchResourcesServlet extends HttpServlet { } } - List getLibs() { - List libs = Lists.newArrayList(); - Set paths = getServletContext().getResourcePaths("/WEB-INF/lib"); - for (Object obj : paths) { - String path = (String) obj; - if (StringUtils.endsWith(path, ".jar")) { - String filename = StringUtils.removeStart(path, "/WEB-INF/lib/"); - if (!isIgnored(filename)) { - libs.add(filename); - } - } - } - return libs; - } - - private static final String[] IGNORE = { "jtds", "mysql", "postgresql", "jruby", "jfreechart", "eastwood", "jetty"}; - - /** - * Dirty hack to disable downloading for certain files. - */ - static boolean isIgnored(String filename) { - for (String prefix : IGNORE) { - if (StringUtils.startsWith(filename, prefix)) { - return true; - } - } - return false; - } - /** * @return part of request URL after servlet path */ diff --git a/sonar-server/src/main/java/org/sonar/server/startup/GenerateBootstrapIndex.java b/sonar-server/src/main/java/org/sonar/server/startup/GenerateBootstrapIndex.java new file mode 100644 index 00000000000..e449ad76500 --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/startup/GenerateBootstrapIndex.java @@ -0,0 +1,113 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.server.startup; + +import com.google.common.collect.Lists; +import com.google.common.io.Closeables; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.CharUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.server.platform.DefaultServerFileSystem; + +import javax.servlet.ServletContext; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Set; + +/** + * @since 3.5 + */ +public final class GenerateBootstrapIndex { + + private static final Logger LOG = LoggerFactory.getLogger(GenerateBootstrapIndex.class); + + private final ServletContext servletContext; + private final DefaultServerFileSystem fileSystem; + + public GenerateBootstrapIndex(DefaultServerFileSystem fileSystem, ServletContext servletContext) { + this.servletContext = servletContext; + this.fileSystem = fileSystem; + } + + public void start() throws IOException { + writeIndex(fileSystem.getBootstrapIndex()); + } + + void writeIndex(File indexFile) throws IOException { + FileUtils.forceMkdir(indexFile.getParentFile()); + FileWriter writer = new FileWriter(indexFile, false); + try { + for (String path : getLibs(servletContext)) { + writer.append(path); + // Compute MD5 + InputStream is = servletContext.getResourceAsStream("/WEB-INF/lib/" + path); + try { + String md5 = DigestUtils.md5Hex(is); + writer.append("|").append(md5); + } catch (IOException e) { + LOG.warn("Unable to compute checksum of {}", path, e); + } finally { + Closeables.closeQuietly(is); + } + writer.append(CharUtils.LF); + } + writer.flush(); + + } finally { + IOUtils.closeQuietly(writer); + } + } + + public static List getLibs(ServletContext servletContext) { + List libs = Lists.newArrayList(); + Set paths = servletContext.getResourcePaths("/WEB-INF/lib"); + for (String path : paths) { + if (StringUtils.endsWith(path, ".jar")) { + String filename = StringUtils.removeStart(path, "/WEB-INF/lib/"); + if (!isIgnored(filename)) { + libs.add(filename); + } + } + } + return libs; + } + + private static final String[] IGNORE = {"jtds", "mysql", "postgresql", "jruby", "jfreechart", "eastwood", "jetty"}; + + /** + * Dirty hack to disable downloading for certain files. + */ + static boolean isIgnored(String filename) { + for (String prefix : IGNORE) { + if (StringUtils.startsWith(filename, prefix)) { + return true; + } + } + return false; + } +} diff --git a/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb b/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb index ebc5a53524f..3dbbe318bea 100644 --- a/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb +++ b/sonar-server/src/main/webapp/WEB-INF/app/controllers/batch_bootstrap_controller.rb @@ -47,6 +47,11 @@ class BatchBootstrapController < Api::ApiController render :json => JSON(json_properties) end + + # GET /batch_bootstrap/index + def index + redirect_to "/deploy/bootstrap/index.txt" + end private diff --git a/sonar-server/src/test/java/org/sonar/server/plugins/BatchResourcesServletTest.java b/sonar-server/src/test/java/org/sonar/server/plugins/BatchResourcesServletTest.java index d1462ebdd16..b1c6a6b39e7 100644 --- a/sonar-server/src/test/java/org/sonar/server/plugins/BatchResourcesServletTest.java +++ b/sonar-server/src/test/java/org/sonar/server/plugins/BatchResourcesServletTest.java @@ -19,17 +19,14 @@ */ package org.sonar.server.plugins; -import com.google.common.collect.Sets; import org.junit.Before; import org.junit.Test; -import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; -import java.util.Set; import static org.fest.assertions.Assertions.assertThat; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class BatchResourcesServletTest { private BatchResourcesServlet servlet; @@ -62,31 +59,4 @@ public class BatchResourcesServletTest { assertThat(servlet.filename(request)).isNull(); } - @Test - public void shouldDetermineListOfResources() { - ServletContext servletContext = mock(ServletContext.class); - servlet = spy(servlet); - doReturn(servletContext).when(servlet).getServletContext(); - Set libs = Sets.newHashSet(); - libs.add("/WEB-INF/lib/sonar-core-2.6.jar"); - libs.add("/WEB-INF/lib/treemap.rb"); - libs.add("/WEB-INF/lib/directory/"); - when(servletContext.getResourcePaths(anyString())).thenReturn(libs); - - assertThat(servlet.getLibs()).hasSize(1); - assertThat(servlet.getLibs().get(0)).isEqualTo("sonar-core-2.6.jar"); - } - - @Test - public void shouldIgnore() { - assertThat(BatchResourcesServlet.isIgnored("sonar-batch-2.6-SNAPSHOT.jar")).isFalse(); - assertThat(BatchResourcesServlet.isIgnored("mysql-connector-java-5.1.13.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("postgresql-9.0-801.jdbc3.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("jtds-1.2.4.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("jfreechart-1.0.9.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("eastwood-1.1.0.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("jetty-util-6.1.24.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("jruby-complete-1.5.6.jar")).isTrue(); - assertThat(BatchResourcesServlet.isIgnored("jruby-rack-1.0.5.jar")).isTrue(); - } } diff --git a/sonar-server/src/test/java/org/sonar/server/startup/GenerateBootstrapIndexTest.java b/sonar-server/src/test/java/org/sonar/server/startup/GenerateBootstrapIndexTest.java new file mode 100644 index 00000000000..b8806bba8b5 --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/startup/GenerateBootstrapIndexTest.java @@ -0,0 +1,67 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.server.startup; + +import com.google.common.collect.Sets; +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.ServletContext; + +import java.util.Set; + +import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GenerateBootstrapIndexTest { + + @Before + public void setUp() throws Exception { + } + + @Test + public void shouldDetermineListOfResources() { + ServletContext servletContext = mock(ServletContext.class); + Set libs = Sets.newHashSet(); + libs.add("/WEB-INF/lib/sonar-core-2.6.jar"); + libs.add("/WEB-INF/lib/treemap.rb"); + libs.add("/WEB-INF/lib/directory/"); + when(servletContext.getResourcePaths(anyString())).thenReturn(libs); + + assertThat(GenerateBootstrapIndex.getLibs(servletContext)).hasSize(1); + assertThat(GenerateBootstrapIndex.getLibs(servletContext).get(0)).isEqualTo("sonar-core-2.6.jar"); + } + + @Test + public void shouldIgnore() { + assertThat(GenerateBootstrapIndex.isIgnored("sonar-batch-2.6-SNAPSHOT.jar")).isFalse(); + assertThat(GenerateBootstrapIndex.isIgnored("mysql-connector-java-5.1.13.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("postgresql-9.0-801.jdbc3.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("jtds-1.2.4.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("jfreechart-1.0.9.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("eastwood-1.1.0.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("jetty-util-6.1.24.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("jruby-complete-1.5.6.jar")).isTrue(); + assertThat(GenerateBootstrapIndex.isIgnored("jruby-rack-1.0.5.jar")).isTrue(); + } + +}