From ee10ce224ea3f6eaf389ad0c4572270de80b1cd9 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Mon, 28 Jan 2013 14:56:39 +0100 Subject: [PATCH] SONAR-2291 Implement caching of downloaded Sonar plugins. * By default cache location is ~/.sonar/.cache * Cache location can be changed by property sonar.cacheLocation * To know if a plugin file have to be downloaded there is a checksum (MD5) comparison --- .../batch/bootstrap/BatchSonarCache.java | 51 +++++ .../batch/bootstrap/BootstrapModule.java | 1 + .../batch/bootstrap/PluginDownloader.java | 49 +++-- .../org/sonar/batch/cache/SonarCache.java | 192 ++++++++++++++++++ .../bootstrap/BatchPluginRepositoryTest.java | 31 +-- .../batch/bootstrap/PluginDownloaderTest.java | 61 ++++-- .../org/sonar/batch/cache/SonarCacheTest.java | 94 +++++++++ .../org/sonar/core/plugins/RemotePlugin.java | 40 ++-- .../sonar/core/plugins/RemotePluginFile.java | 40 ++++ .../sonar/core/plugins/RemotePluginTest.java | 28 +-- .../java/org/sonar/api/CoreProperties.java | 5 + 11 files changed, 524 insertions(+), 68 deletions(-) create mode 100644 sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchSonarCache.java create mode 100644 sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java create mode 100644 sonar-batch/src/test/java/org/sonar/batch/cache/SonarCacheTest.java create mode 100644 sonar-core/src/main/java/org/sonar/core/plugins/RemotePluginFile.java diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchSonarCache.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchSonarCache.java new file mode 100644 index 00000000000..ea2ba927304 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchSonarCache.java @@ -0,0 +1,51 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.batch.bootstrap; + +import org.sonar.api.CoreProperties; +import org.sonar.api.config.Settings; +import org.sonar.api.task.TaskExtension; +import org.sonar.batch.cache.SonarCache; + +import java.io.File; + +public class BatchSonarCache implements TaskExtension { + + private Settings settings; + private SonarCache cache; + + public BatchSonarCache(Settings settings) { + this.settings = settings; + } + + public void start() { + String cacheLocation = settings.getString(CoreProperties.CACHE_LOCATION); + SonarCache.Builder builder = SonarCache.create(); + if (cacheLocation != null) { + File cacheLocationFolder = new File(cacheLocation); + builder.setCacheLocation(cacheLocationFolder); + } + this.cache = builder.build(); + } + + public SonarCache getCache() { + return cache; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java index c65099b3f22..39614208ea1 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BootstrapModule.java @@ -74,6 +74,7 @@ public class BootstrapModule extends Module { container.addSingleton(HttpDownloader.class); container.addSingleton(UriReader.class); container.addSingleton(PluginDownloader.class); + container.addSingleton(BatchSonarCache.class); for (Object component : boostrapperComponents) { if (component != null) { container.addSingleton(component); diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginDownloader.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginDownloader.java index e462afbc10b..d0852e487a2 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginDownloader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PluginDownloader.java @@ -20,14 +20,15 @@ package org.sonar.batch.bootstrap; import com.google.common.collect.Lists; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.BatchComponent; import org.sonar.api.utils.SonarException; +import org.sonar.batch.cache.SonarCache; import org.sonar.core.plugins.RemotePlugin; +import org.sonar.core.plugins.RemotePluginFile; import java.io.File; import java.util.List; @@ -36,26 +37,48 @@ public class PluginDownloader implements BatchComponent { private static final Logger LOG = LoggerFactory.getLogger(PluginDownloader.class); - private TempDirectories workingDirectories; private ServerClient server; + private BatchSonarCache batchCache; - public PluginDownloader(TempDirectories workingDirectories, ServerClient server) { - this.workingDirectories = workingDirectories; + public PluginDownloader(BatchSonarCache batchCache, ServerClient server) { this.server = server; + this.batchCache = batchCache; + } + + private SonarCache getSonarCache() { + return batchCache.getCache(); } public List downloadPlugin(RemotePlugin remote) { try { - File targetDir = workingDirectories.getDir("plugins/" + remote.getKey()); - FileUtils.forceMkdir(targetDir); - LOG.debug("Downloading plugin " + remote.getKey() + " into " + targetDir); - List files = Lists.newArrayList(); - for (String filename : remote.getFilenames()) { - String url = "/deploy/plugins/" + remote.getKey() + "/" + filename; - File toFile = new File(targetDir, filename); - server.download(url, toFile); - files.add(toFile); + for (RemotePluginFile file : remote.getFiles()) { + LOG.debug("Looking if plugin file {} with md5 {} is already in cache", file.getFilename(), file.getMd5()); + File fileInCache = getSonarCache().getFileFromCache(file.getFilename(), file.getMd5()); + if (fileInCache != null) { + LOG.debug("File is already cached at location {}", fileInCache.getAbsolutePath()); + } + else { + LOG.debug("File is not cached"); + File tmpDownloadFile = getSonarCache().getTemporaryFile(); + String url = "/deploy/plugins/" + remote.getKey() + "/" + file.getFilename(); + if (LOG.isDebugEnabled()) { + LOG.debug("Downloading {} to {}", url, tmpDownloadFile.getAbsolutePath()); + } + else { + LOG.info("Downloading {}", file.getFilename()); + } + server.download(url, tmpDownloadFile); + LOG.debug("Trying to cache file"); + String md5 = getSonarCache().cacheFile(tmpDownloadFile, file.getFilename()); + fileInCache = getSonarCache().getFileFromCache(file.getFilename(), md5); + if (!md5.equals(file.getMd5())) { + LOG.warn("INVALID CHECKSUM: File {} was expected to have checksum {} but was cached with checksum {}", + new String[] {fileInCache.getAbsolutePath(), file.getMd5(), md5}); + } + LOG.debug("File cached at location {}", fileInCache.getAbsolutePath()); + } + files.add(fileInCache); } return files; diff --git a/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java b/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java new file mode 100644 index 00000000000..bd203b2d17b --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/cache/SonarCache.java @@ -0,0 +1,192 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.batch.cache; + +import com.google.common.io.Files; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.utils.SonarException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +/** + * This class is responsible for managing Sonar batch file cache. You can put file into cache and + * later try to retrieve them. MD5 is used to differentiate files (name is not secure as files may come + * from different Sonar servers and have same name but be actually different, and same for SNAPSHOTs). + * Default location of cache is + * @author Julien HENRY + * + */ +public class SonarCache { + + private static final Logger LOG = LoggerFactory.getLogger(SonarCache.class); + + private static final int TEMP_FILE_ATTEMPTS = 10000; + + private File cacheLocation; + /** + * Temporary directory where files should be stored before be inserted in the cache. + * Having a temporary close to the final location (read on same FS) will assure + * the move will be atomic. + */ + private File tmpDir; + + private SonarCache(File cacheLocation) { + this.cacheLocation = cacheLocation; + tmpDir = new File(cacheLocation, ".tmp"); + if (!cacheLocation.exists()) { + LOG.debug("Creating cache directory: " + cacheLocation.getAbsolutePath()); + try { + FileUtils.forceMkdir(cacheLocation); + } catch (IOException e) { + throw new SonarException("Unable to create cache directory " + cacheLocation.getAbsolutePath(), e); + } + } + } + + public static class Builder { + + private File cacheLocation; + + public Builder setCacheLocation(File cacheLocation) { + this.cacheLocation = cacheLocation; + return this; + } + + public SonarCache build() { + if (cacheLocation == null) { + File sonarHome = new File(System.getProperty("user.home"), ".sonar"); + return new SonarCache(new File(sonarHome, ".cache")); + } + else { + return new SonarCache(cacheLocation); + } + } + + } + + public static Builder create() { + return new Builder(); + } + + /** + * Move the given file inside the cache. Return the MD5 of the cached file. + * @param sourceFile + * @throws IOException + */ + public String cacheFile(File sourceFile, String filename) throws IOException { + File tmpFileName = null; + try { + if (!sourceFile.getParentFile().equals(getTmpDir())) { + // Provided file is not close to the cache so we will move it first in a temporary file (could be non atomic) + tmpFileName = getTemporaryFile(); + Files.move(sourceFile, tmpFileName); + } + else { + tmpFileName = sourceFile; + } + // Now compute the md5 to find the final destination + FileInputStream fis = new FileInputStream(tmpFileName); + String md5 = DigestUtils.md5Hex(fis); + File finalDir = new File(cacheLocation, md5); + File finalFileName = new File(finalDir, filename); + // Try to create final destination folder + FileUtils.forceMkdir(finalDir); + // Now try to move the file from temporary folder to final location + boolean rename = tmpFileName.renameTo(finalFileName); + if (!rename) { + // Check if the file was already in cache + if (!finalFileName.exists()) { + LOG.warn("Unable to rename " + tmpFileName.getAbsolutePath() + " to " + finalFileName.getAbsolutePath()); + LOG.warn("A copy/delete will be tempted but with no garantee of atomicity"); + FileUtils.moveFile(tmpFileName, finalFileName); + } + } + return md5; + } finally { + FileUtils.deleteQuietly(tmpFileName); + } + + } + + /** + * Look for a file in the cache by its filename and md5 checksum. If the file is not + * present then return null. + */ + public File getFileFromCache(String filename, String md5) { + File location = new File(new File(cacheLocation, md5), filename); + if (location.exists()) { + return location; + } + LOG.debug("No file found in the cache with name {} and checksum {}", filename, md5); + return null; + } + + /** + * Return a temporary file that caller can use to store file content before + * asking for caching it with {@link #cacheFile(File)}. + * This is to avoid extra copy. + * @return + * @throws IOException + */ + public File getTemporaryFile() throws IOException { + return createTempFile(getTmpDir()); + } + + /** + * Create a temporary file in the given directory. + * @param baseDir + * @return + * @throws IOException + */ + private static File createTempFile(File baseDir) throws IOException { + String baseName = System.currentTimeMillis() + "-"; + + for (int counter = 0; counter < TEMP_FILE_ATTEMPTS; counter++) { + File tempFile = new File(baseDir, baseName + counter); + if (tempFile.createNewFile()) { + return tempFile; + } + } + throw new IOException("Failed to create temporary file in " + baseDir.getAbsolutePath() + " within " + + TEMP_FILE_ATTEMPTS + " attempts (tried " + + baseName + "0 to " + baseName + (TEMP_FILE_ATTEMPTS - 1) + ')'); + } + + public File getTmpDir() { + if (!tmpDir.exists()) { + LOG.debug("Creating temporary cache directory: " + tmpDir.getAbsolutePath()); + try { + FileUtils.forceMkdir(tmpDir); + } catch (IOException e) { + throw new SonarException("Unable to create temporary cache directory " + tmpDir.getAbsolutePath(), e); + } + } + return tmpDir; + } + + public File getCacheLocation() { + return cacheLocation; + } +} diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginRepositoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginRepositoryTest.java index 6da2f0192bf..f0cf8a22478 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginRepositoryTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginRepositoryTest.java @@ -26,6 +26,7 @@ import org.junit.Test; import org.sonar.api.CoreProperties; import org.sonar.api.config.Settings; import org.sonar.core.plugins.RemotePlugin; +import org.sonar.core.plugins.RemotePluginFile; import org.sonar.test.TestUtils; import java.io.File; @@ -87,8 +88,8 @@ public class BatchPluginRepositoryTest { @Test public void shouldLoadPluginDeprecatedExtensions() throws IOException { - RemotePlugin checkstyle = new RemotePlugin("checkstyle", true) - .addFilename("checkstyle-ext.xml"); + RemotePlugin checkstyle = new RemotePlugin("checkstyle", true); + checkstyle.getFiles().add(new RemotePluginFile("checkstyle-ext.xml", "fakemd5")); PluginDownloader downloader = mock(PluginDownloader.class); when(downloader.downloadPlugin(checkstyle)).thenReturn(copyFiles("sonar-checkstyle-plugin-2.8.jar", "checkstyle-ext.xml")); @@ -143,8 +144,8 @@ public class BatchPluginRepositoryTest { @Test public void whiteListShouldTakePrecedenceOverBlackList() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs") - .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "cobertura,pmd"); + .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs") + .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "cobertura,pmd"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("pmd")).isTrue(); } @@ -152,7 +153,7 @@ public class BatchPluginRepositoryTest { @Test public void corePluginShouldAlwaysBeInWhiteList() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); + .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("core")).isTrue(); } @@ -160,7 +161,7 @@ public class BatchPluginRepositoryTest { @Test public void corePluginShouldNeverBeInBlackList() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "core,findbugs"); + .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "core,findbugs"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("core")).isTrue(); } @@ -169,7 +170,7 @@ public class BatchPluginRepositoryTest { @Test public void englishPackPluginShouldNeverBeInBlackList() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "l10nen,findbugs"); + .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "l10nen,findbugs"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("l10nen")).isTrue(); } @@ -177,7 +178,7 @@ public class BatchPluginRepositoryTest { @Test public void shouldCheckWhitelist() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); + .setProperty(CoreProperties.BATCH_INCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("checkstyle")).isTrue(); assertThat(filter.accepts("pmd")).isTrue(); @@ -187,7 +188,7 @@ public class BatchPluginRepositoryTest { @Test public void shouldCheckBlackListIfNoWhiteList() { Settings settings = new Settings() - .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); + .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "checkstyle,pmd,findbugs"); BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); assertThat(filter.accepts("checkstyle")).isFalse(); assertThat(filter.accepts("pmd")).isFalse(); @@ -195,14 +196,14 @@ public class BatchPluginRepositoryTest { } @Test - public void should_concatenate_dry_run_filters() { - Settings settings = new Settings() + public void should_concatenate_dry_run_filters() { + Settings settings = new Settings() .setProperty(CoreProperties.DRY_RUN, true) .setProperty(CoreProperties.DRY_RUN_INCLUDE_PLUGINS, "cockpit") .setProperty(CoreProperties.DRY_RUN_EXCLUDE_PLUGINS, "views") .setProperty(CoreProperties.BATCH_EXCLUDE_PLUGINS, "checkstyle,pmd"); - BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); - assertThat(filter.whites).containsOnly("cockpit"); - assertThat(filter.blacks).containsOnly("views", "checkstyle", "pmd"); - } + BatchPluginRepository.PluginFilter filter = new BatchPluginRepository.PluginFilter(settings); + assertThat(filter.whites).containsOnly("cockpit"); + assertThat(filter.blacks).containsOnly("views", "checkstyle", "pmd"); + } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PluginDownloaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PluginDownloaderTest.java index ad32777a15b..65c9fdbe149 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PluginDownloaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PluginDownloaderTest.java @@ -23,7 +23,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; +import org.sonar.api.config.Settings; import org.sonar.api.utils.SonarException; +import org.sonar.batch.cache.SonarCache; import org.sonar.core.plugins.RemotePlugin; import java.io.File; @@ -32,6 +35,7 @@ import java.util.List; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -45,10 +49,9 @@ public class PluginDownloaderTest { @Test public void should_request_list_of_plugins() { - TempDirectories tempDirs = mock(TempDirectories.class); ServerClient server = mock(ServerClient.class); when(server.request("/deploy/plugins/index.txt")).thenReturn("checkstyle,true\nsqale,false"); - PluginDownloader downloader = new PluginDownloader(tempDirs, server); + PluginDownloader downloader = new PluginDownloader(new BatchSonarCache(new Settings()), server); List plugins = downloader.downloadPluginIndex(); assertThat(plugins).hasSize(2); @@ -59,24 +62,52 @@ public class PluginDownloaderTest { } @Test - public void should_download_plugin() throws Exception { - TempDirectories tempDirs = mock(TempDirectories.class); - File toDir = temp.newFolder(); - when(tempDirs.getDir("plugins/checkstyle")).thenReturn(toDir); + public void should_download_plugin_if_not_cached() throws Exception { + SonarCache cache = mock(SonarCache.class); + BatchSonarCache batchCache = mock(BatchSonarCache.class); + when(batchCache.getCache()).thenReturn(cache); + + File fileInCache = temp.newFile(); + when(cache.cacheFile(Mockito.any(File.class), Mockito.anyString())).thenReturn("fakemd51").thenReturn("fakemd52"); + when(cache.getFileFromCache(Mockito.anyString(), Mockito.anyString())) + .thenReturn(null) + .thenReturn(fileInCache) + .thenReturn(null) + .thenReturn(fileInCache); + ServerClient server = mock(ServerClient.class); + PluginDownloader downloader = new PluginDownloader(batchCache, server); + + RemotePlugin plugin = new RemotePlugin("checkstyle", true) + .addFile("checkstyle-plugin.jar", "fakemd51") + .addFile("checkstyle-extensions.jar", "fakemd52"); + List files = downloader.downloadPlugin(plugin); + + assertThat(files).hasSize(2); + verify(server).download(Mockito.eq("/deploy/plugins/checkstyle/checkstyle-plugin.jar"), Mockito.any(File.class)); + verify(server).download(Mockito.eq("/deploy/plugins/checkstyle/checkstyle-extensions.jar"), Mockito.any(File.class)); + } + + @Test + public void should_not_download_plugin_if_cached() throws Exception { + SonarCache cache = mock(SonarCache.class); + BatchSonarCache batchCache = mock(BatchSonarCache.class); + when(batchCache.getCache()).thenReturn(cache); + + File fileInCache = temp.newFile(); + when(cache.getFileFromCache(Mockito.anyString(), Mockito.anyString())) + .thenReturn(fileInCache) + .thenReturn(fileInCache); ServerClient server = mock(ServerClient.class); - PluginDownloader downloader = new PluginDownloader(tempDirs, server); + PluginDownloader downloader = new PluginDownloader(batchCache, server); RemotePlugin plugin = new RemotePlugin("checkstyle", true) - .addFilename("checkstyle-plugin.jar") - .addFilename("checkstyle-extensions.jar"); + .addFile("checkstyle-plugin.jar", "fakemd51") + .addFile("checkstyle-extensions.jar", "fakemd52"); List files = downloader.downloadPlugin(plugin); - File pluginFile = new File(toDir, "checkstyle-plugin.jar"); - File extFile = new File(toDir, "checkstyle-extensions.jar"); assertThat(files).hasSize(2); - assertThat(files).containsOnly(pluginFile, extFile); - verify(server).download("/deploy/plugins/checkstyle/checkstyle-plugin.jar", pluginFile); - verify(server).download("/deploy/plugins/checkstyle/checkstyle-extensions.jar", extFile); + verify(server, never()).download(Mockito.anyString(), Mockito.any(File.class)); + verify(cache, never()).cacheFile(Mockito.any(File.class), Mockito.anyString()); } @Test @@ -86,6 +117,6 @@ public class PluginDownloaderTest { ServerClient server = mock(ServerClient.class); doThrow(new SonarException()).when(server).request("/deploy/plugins/index.txt"); - new PluginDownloader(mock(TempDirectories.class), server).downloadPluginIndex(); + new PluginDownloader(new BatchSonarCache(new Settings()), server).downloadPluginIndex(); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/cache/SonarCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/cache/SonarCacheTest.java new file mode 100644 index 00000000000..2566de8e6bf --- /dev/null +++ b/sonar-batch/src/test/java/org/sonar/batch/cache/SonarCacheTest.java @@ -0,0 +1,94 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.batch.cache; + +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static org.fest.assertions.Assertions.assertThat; + +public class SonarCacheTest { + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private SonarCache cache; + + @Before + public void prepare() throws IOException { + cache = SonarCache.create().setCacheLocation(tempFolder.newFolder()).build(); + } + + @Test + public void testCacheExternalFile() throws IOException { + // Create a file outside the cache + File fileToCache = tempFolder.newFile(); + FileUtils.write(fileToCache, "Sample data"); + // Put it in the cache + String md5 = cache.cacheFile(fileToCache, "foo.txt"); + // Verify the temporary location was created to do the copy in the cache in 2 stages + File tmpCache = new File(cache.getCacheLocation(), ".tmp"); + assertThat(tmpCache).exists(); + // The tmp location should be empty as the file was moved inside the cache + assertThat(tmpCache.list()).isEmpty(); + // Verify it is present in the cache folder + File fileInCache = new File(new File(cache.getCacheLocation(), md5), "foo.txt"); + assertThat(fileInCache).exists(); + String content = FileUtils.readFileToString(fileInCache); + assertThat(content).isEqualTo("Sample data"); + // Now retrieve from cache API + File fileFromCache = cache.getFileFromCache("foo.txt", md5); + assertThat(fileFromCache.getCanonicalPath()).isEqualTo(fileInCache.getCanonicalPath()); + } + + @Test + public void testCacheInternalFile() throws IOException { + // Create a file in the cache temp location + File fileToCache = cache.getTemporaryFile(); + // Verify the temporary location was created + File tmpCache = new File(cache.getCacheLocation(), ".tmp"); + assertThat(tmpCache).exists(); + assertThat(tmpCache.list().length).isEqualTo(1); + + FileUtils.write(fileToCache, "Sample data"); + String md5 = cache.cacheFile(fileToCache, "foo.txt"); + // Verify it is present in the cache folder + File fileInCache = new File(new File(cache.getCacheLocation(), md5), "foo.txt"); + assertThat(fileInCache).exists(); + String content = FileUtils.readFileToString(fileInCache); + assertThat(content).isEqualTo("Sample data"); + // Now retrieve from cache API + File fileFromCache = cache.getFileFromCache("foo.txt", md5); + assertThat(fileFromCache.getCanonicalPath()).isEqualTo(fileInCache.getCanonicalPath()); + } + + @Test + public void testGetFileNotInCache() throws IOException { + File fileFromCache = cache.getFileFromCache("foo.txt", "mockmd5"); + assertThat(fileFromCache).isNull(); + } + +} diff --git a/sonar-core/src/main/java/org/sonar/core/plugins/RemotePlugin.java b/sonar-core/src/main/java/org/sonar/core/plugins/RemotePlugin.java index b00c3230a05..7ae665f1c8c 100644 --- a/sonar-core/src/main/java/org/sonar/core/plugins/RemotePlugin.java +++ b/sonar-core/src/main/java/org/sonar/core/plugins/RemotePlugin.java @@ -20,14 +20,16 @@ package org.sonar.core.plugins; import com.google.common.collect.Lists; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import java.io.File; +import java.io.FileInputStream; import java.util.List; public class RemotePlugin { private String pluginKey; - private List filenames = Lists.newArrayList(); + private List files = Lists.newArrayList(); private boolean core; public RemotePlugin(String pluginKey, boolean core) { @@ -37,9 +39,9 @@ public class RemotePlugin { public static RemotePlugin create(DefaultPluginMetadata metadata) { RemotePlugin result = new RemotePlugin(metadata.getKey(), metadata.isCore()); - result.addFilename(metadata.getFile().getName()); + result.addFile(metadata.getFile()); for (File file : metadata.getDeprecatedExtensions()) { - result.addFilename(file.getName()); + result.addFile(file); } return result; } @@ -49,7 +51,8 @@ public class RemotePlugin { RemotePlugin result = new RemotePlugin(fields[0], Boolean.parseBoolean(fields[1])); if (fields.length > 2) { for (int index = 2; index < fields.length; index++) { - result.addFilename(fields[index]); + String[] nameAndMd5 = StringUtils.split(fields[index], "|"); + result.addFile(nameAndMd5[0], nameAndMd5.length > 1 ? nameAndMd5[1] : null); } } return result; @@ -59,8 +62,11 @@ public class RemotePlugin { StringBuilder sb = new StringBuilder(); sb.append(pluginKey).append(","); sb.append(String.valueOf(core)); - for (String filename : filenames) { - sb.append(",").append(filename); + for (RemotePluginFile file : files) { + sb.append(",").append(file.getFilename()); + if (StringUtils.isNotBlank(file.getMd5())) { + sb.append("|").append(file.getMd5()); + } } return sb.toString(); } @@ -69,22 +75,32 @@ public class RemotePlugin { return pluginKey; } - public boolean isCore() { return core; } - public RemotePlugin addFilename(String s) { - filenames.add(s); + public RemotePlugin addFile(String filename, String md5) { + files.add(new RemotePluginFile(filename, md5)); return this; } - public List getFilenames() { - return filenames; + public RemotePlugin addFile(File f) { + String md5; + try { + FileInputStream fis = new FileInputStream(f); + md5 = DigestUtils.md5Hex(fis); + } catch (Exception e) { + md5 = null; + } + return this.addFile(f.getName(), md5); + } + + public List getFiles() { + return files; } public String getPluginFilename() { - return (!filenames.isEmpty() ? filenames.get(0) : null); + return (!files.isEmpty() ? files.get(0).getFilename() : null); } @Override diff --git a/sonar-core/src/main/java/org/sonar/core/plugins/RemotePluginFile.java b/sonar-core/src/main/java/org/sonar/core/plugins/RemotePluginFile.java new file mode 100644 index 00000000000..88d85c3485b --- /dev/null +++ b/sonar-core/src/main/java/org/sonar/core/plugins/RemotePluginFile.java @@ -0,0 +1,40 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.core.plugins; + + +public class RemotePluginFile { + + private String filename; + private String md5; + + public RemotePluginFile(String filename, String md5) { + this.filename = filename; + this.md5 = md5; + } + + public String getFilename() { + return filename; + } + + public String getMd5() { + return md5; + } +} diff --git a/sonar-core/src/test/java/org/sonar/core/plugins/RemotePluginTest.java b/sonar-core/src/test/java/org/sonar/core/plugins/RemotePluginTest.java index f882da5683d..b62dc1d3ad4 100644 --- a/sonar-core/src/test/java/org/sonar/core/plugins/RemotePluginTest.java +++ b/sonar-core/src/test/java/org/sonar/core/plugins/RemotePluginTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.junit.matchers.JUnitMatchers.hasItems; public class RemotePluginTest { @Test @@ -38,29 +37,30 @@ public class RemotePluginTest { @Test public void shouldMarshal() { - RemotePlugin clirr = new RemotePlugin("clirr", false).addFilename("clirr-1.1.jar"); + RemotePlugin clirr = new RemotePlugin("clirr", false).addFile("clirr-1.1.jar", "fakemd5"); String text = clirr.marshal(); - assertThat(text, is("clirr,false,clirr-1.1.jar")); + assertThat(text, is("clirr,false,clirr-1.1.jar|fakemd5")); } @Test public void shouldMarshalDeprecatedExtensions() { - RemotePlugin checkstyle = new RemotePlugin("checkstyle", true); - checkstyle.addFilename("checkstyle-2.8.jar"); - checkstyle.addFilename("ext.xml"); - checkstyle.addFilename("ext.jar"); + RemotePlugin checkstyle = new RemotePlugin("checkstyle", true) + .addFile("checkstyle-2.8.jar", "fakemd51") + .addFile("ext.xml", "fakemd52") + .addFile("ext.jar", "fakemd53"); String text = checkstyle.marshal(); - assertThat(text, is("checkstyle,true,checkstyle-2.8.jar,ext.xml,ext.jar")); + assertThat(text, is("checkstyle,true,checkstyle-2.8.jar|fakemd51,ext.xml|fakemd52,ext.jar|fakemd53")); } @Test public void shouldUnmarshal() { - RemotePlugin clirr = RemotePlugin.unmarshal("clirr,false,clirr-1.1.jar"); + RemotePlugin clirr = RemotePlugin.unmarshal("clirr,false,clirr-1.1.jar|fakemd5"); assertThat(clirr.getKey(), is("clirr")); assertThat(clirr.isCore(), is(false)); - assertThat(clirr.getFilenames().size(), is(1)); - assertThat(clirr.getFilenames().get(0), is("clirr-1.1.jar")); + assertThat(clirr.getFiles().size(), is(1)); + assertThat(clirr.getFiles().get(0).getFilename(), is("clirr-1.1.jar")); + assertThat(clirr.getFiles().get(0).getMd5(), is("fakemd5")); } @@ -69,7 +69,9 @@ public class RemotePluginTest { RemotePlugin checkstyle = RemotePlugin.unmarshal("checkstyle,true,checkstyle-2.8.jar,ext.xml,ext.jar"); assertThat(checkstyle.getKey(), is("checkstyle")); assertThat(checkstyle.isCore(), is(true)); - assertThat(checkstyle.getFilenames().size(), is(3)); - assertThat(checkstyle.getFilenames(), hasItems("checkstyle-2.8.jar", "ext.xml", "ext.jar")); + assertThat(checkstyle.getFiles().size(), is(3)); + assertThat(checkstyle.getFiles().get(0).getFilename(), is("checkstyle-2.8.jar")); + assertThat(checkstyle.getFiles().get(1).getFilename(), is("ext.xml")); + assertThat(checkstyle.getFiles().get(2).getFilename(), is("ext.jar")); } } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java index 3186a6ca6d7..e4c1a85868c 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java @@ -376,6 +376,11 @@ public interface CoreProperties { */ String TASK = "sonar.task"; + /** + * @since 3.5 + */ + String CACHE_LOCATION = "sonar.cacheLocation"; + /** * @deprecated replaced in v3.4 by properties specific to languages, for example sonar.java.coveragePlugin * See http://jira.codehaus.org/browse/SONARJAVA-39 for more details. -- 2.39.5