From: Duarte Meneses Date: Thu, 7 Dec 2017 10:38:28 +0000 (+0100) Subject: SONAR-10142 Download compressed plugins X-Git-Tag: 7.0-RC1~173 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=0001182a007e70261d447023f1fa5ad5e1a0b69f;p=sonarqube.git SONAR-10142 Download compressed plugins --- diff --git a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java index a1054b47650..1aa95c7245d 100644 --- a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java +++ b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import javax.annotation.CheckForNull; import org.apache.commons.lang.SystemUtils; @@ -238,6 +239,12 @@ public class FileUtilsTest { assertThat(childDir2).doesNotExist(); } + @Test + public void getPack200FilePath_transforms_jar_name() { + Path jarPath = Paths.get("plugin.jar"); + assertThat(FileUtils.getPack200FilePath(jarPath)).isEqualTo(Paths.get("plugin.pack.gz")); + } + private void expectDirectoryCanNotBeNullNPE() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("Directory can not be null"); diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java index 4a03ec24794..e3cbafd22de 100644 --- a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java +++ b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java @@ -24,10 +24,8 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.jar.JarOutputStream; import java.util.jar.Pack200; import java.util.zip.GZIPInputStream; @@ -43,7 +41,7 @@ public class FileCache { /** Maximum loop count when creating temp directories. */ private static final int TEMP_DIR_ATTEMPTS = 10_000; - private final File dir; + private final File cacheDir; private final File tmpDir; private final FileHashes hashes; private final Logger logger; @@ -51,7 +49,7 @@ public class FileCache { FileCache(File dir, FileHashes fileHashes, Logger logger) { this.hashes = fileHashes; this.logger = logger; - this.dir = createDir(dir, "user cache: "); + this.cacheDir = createDir(dir, "user cache: "); logger.info(String.format("User cache: %s", dir.getAbsolutePath())); this.tmpDir = createDir(new File(dir, "_tmp"), "temp dir"); } @@ -61,7 +59,7 @@ public class FileCache { } public File getDir() { - return dir; + return cacheDir; } /** @@ -70,7 +68,7 @@ public class FileCache { */ @CheckForNull public File get(String filename, String hash) { - File cachedFile = new File(new File(dir, hash), filename); + File cachedFile = new File(new File(cacheDir, hash), filename); if (cachedFile.exists()) { return cachedFile; } @@ -82,41 +80,54 @@ public class FileCache { void download(String filename, File toFile) throws IOException; } - public File get(String jarFilename, String hash, Downloader downloader) { + public File get(String filename, String hash, Downloader downloader) { // Does not fail if another process tries to create the directory at the same time. File hashDir = hashDir(hash); - File targetFile = new File(hashDir, jarFilename); + File targetFile = new File(hashDir, filename); if (!targetFile.exists()) { - File tempPackedFile = newTempFile(); - File tempJarFile = newTempFile(); - String packedFileName = getPackedFileName(jarFilename); - download(downloader, packedFileName, tempPackedFile); - - logger.debug("Unpacking plugin " + jarFilename); - - unpack200(tempPackedFile.toPath(), tempJarFile.toPath()); - logger.debug("Done"); - String downloadedHash = hashes.of(tempJarFile); - // if (!hash.equals(downloadedHash)) { - // throw new IllegalStateException("INVALID HASH: File " + tempJarFile.getAbsolutePath() + " was expected to have hash " + hash - // + " but was downloaded with hash " + downloadedHash); - // } - mkdirQuietly(hashDir); - renameQuietly(tempJarFile, targetFile); - + cacheMiss(targetFile, hash, downloader); } return targetFile; } - private static String getPackedFileName(String jarName) { - return jarName.substring(0, jarName.length() - 3) + "pack.gz"; + private void cacheMiss(File targetFile, String expectedHash, Downloader downloader) { + File tempFile = newTempFile(); + download(downloader, targetFile.getName(), tempFile); + String downloadedHash = hashes.of(tempFile); + if (!expectedHash.equals(downloadedHash)) { + throw new IllegalStateException("INVALID HASH: File " + tempFile.getAbsolutePath() + " was expected to have hash " + expectedHash + + " but was downloaded with hash " + downloadedHash); + } + mkdirQuietly(targetFile.getParentFile()); + renameQuietly(tempFile, targetFile); + } + + public File getCompressed(String filename, String hash, Downloader downloader) { + File hashDir = hashDir(hash); + File compressedFile = new File(hashDir, filename); + File jarFile = new File(compressedFile.getParentFile(), getUnpackedFileName(compressedFile.getName())); + + if (!jarFile.exists()) { + if (!compressedFile.exists()) { + cacheMiss(compressedFile, hash, downloader); + } + File tempFile = newTempFile(); + unpack200(compressedFile.toPath(), tempFile.toPath()); + renameQuietly(tempFile, jarFile); + } + return jarFile; + } + + private static String getUnpackedFileName(String packedName) { + return packedName.substring(0, packedName.length() - 7) + "jar"; } - private static void unpack200(Path tempFile, Path targetFile) { + private void unpack200(Path compressedFile, Path jarFile) { + logger.debug("Unpacking plugin " + compressedFile); Pack200.Unpacker unpacker = Pack200.newUnpacker(); try { - try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(targetFile))); - InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(tempFile)))) { + try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(jarFile))); + InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(compressedFile)))) { unpacker.unpack(in, jarStream); } } catch (IOException e) { @@ -147,7 +158,7 @@ public class FileCache { } private File hashDir(String hash) { - return new File(dir, hash); + return new File(cacheDir, hash); } private static void mkdirQuietly(File hashDir) { diff --git a/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java b/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java index 2a7381c866a..67635b5ab70 100644 --- a/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java +++ b/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java @@ -22,7 +22,6 @@ package org.sonar.home.cache; import java.io.File; import java.io.IOException; import org.apache.commons.io.FileUtils; -import org.assertj.core.util.Files; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -32,6 +31,7 @@ import org.junit.rules.TemporaryFolder; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class FileCacheTest { @@ -77,7 +77,7 @@ public class FileCacheTest { thrown.expectMessage("Fail to download"); cache.get("sonar-foo-plugin-1.5.jar", "ABCDE", downloader); } - + @Test public void fail_create_hash_dir() throws IOException { File file = tempFolder.newFile(); @@ -85,11 +85,11 @@ public class FileCacheTest { thrown.expectMessage("Unable to create user cache"); cache = new FileCache(file, fileHashes, mock(Logger.class)); } - + @Test public void fail_to_create_hash_dir() throws IOException { when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); - + File hashDir = new File(cache.getDir(), "ABCDE"); hashDir.createNewFile(); thrown.expect(IllegalStateException.class); @@ -113,6 +113,60 @@ public class FileCacheTest { assertThat(FileUtils.readFileToString(cachedFile)).isEqualTo("body"); } + @Test + public void download_and_add_to_cache_compressed_file() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + + FileCache.Downloader downloader = new FileCache.Downloader() { + public void download(String filename, File toFile) throws IOException { + FileUtils.copyFile(compressedFile(), toFile); + } + }; + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + } + + @Test + public void dont_download_compressed_file_if_jar_exists() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + FileCache.Downloader downloader = mock(FileCache.Downloader.class); + + File hashDir = new File(cache.getDir(), "ABCDE"); + hashDir.mkdirs(); + File jar = new File(new File(cache.getDir(), "ABCDE"), "sonar-foo-plugin-1.5.jar"); + jar.createNewFile(); + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + + verifyZeroInteractions(downloader); + } + + @Test + public void dont_download_compressed_file_if_it_exists() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + FileCache.Downloader downloader = mock(FileCache.Downloader.class); + + File hashDir = new File(cache.getDir(), "ABCDE"); + hashDir.mkdirs(); + FileUtils.copyFile(compressedFile(), new File(hashDir, "sonar-foo-plugin-1.5.pack.gz")); + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + + verifyZeroInteractions(downloader); + } + @Test public void download_corrupted_file() throws IOException { thrown.expect(IllegalStateException.class); @@ -149,4 +203,8 @@ public class FileCacheTest { assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); assertThat(FileUtils.readFileToString(cachedFile)).contains("downloaded by"); } + + private File compressedFile() { + return new File("src/test/resources/test.pack.gz"); + } } diff --git a/sonar-home/src/test/resources/test.pack.gz b/sonar-home/src/test/resources/test.pack.gz new file mode 100644 index 00000000000..8e3a43c8e3d Binary files /dev/null and b/sonar-home/src/test/resources/test.pack.gz differ diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index 046acc54cd0..690cd70af97 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -92,7 +93,11 @@ public class ScannerPluginInstaller implements PluginInstaller { @VisibleForTesting File download(final InstalledPlugin remote) { try { - return fileCache.get(remote.filename, remote.hash, new FileDownloader(remote.key)); + if (remote.compressedFilename != null) { + return fileCache.getCompressed(remote.compressedFilename, remote.compressedHash, new FileDownloader(remote.key)); + } else { + return fileCache.get(remote.filename, remote.hash, new FileDownloader(remote.key)); + } } catch (Exception e) { throw new IllegalStateException("Fail to download plugin: " + remote.key, e); } @@ -125,6 +130,10 @@ public class ScannerPluginInstaller implements PluginInstaller { String hash; String filename; long updatedAt; + @Nullable + String compressedHash; + @Nullable + String compressedFilename; } private class FileDownloader implements FileCache.Downloader { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java index 580ccd31c21..276fae19122 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java @@ -80,6 +80,24 @@ public class ScannerPluginInstallerTest { assertThat(file).isEqualTo(pluginJar); } + @Test + public void should_download_compressed_plugin() throws Exception { + File pluginJar = temp.newFile(); + when(fileCache.getCompressed(eq("checkstyle-plugin.pack.gz"), eq("hash"), any(FileCache.Downloader.class))).thenReturn(pluginJar); + + ScannerPluginInstaller underTest = new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate); + + InstalledPlugin remote = new InstalledPlugin(); + remote.key = "checkstyle"; + remote.filename = "checkstyle-plugin.jar"; + remote.hash = "fakemd5_1"; + remote.compressedFilename = "checkstyle-plugin.pack.gz"; + remote.compressedHash = "hash"; + File file = underTest.download(remote); + + assertThat(file).isEqualTo(pluginJar); + } + @Test public void should_fail_to_get_plugin_index() { WsTestUtil.mockException(wsClient, "/api/plugins/installed", new IllegalStateException()); diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java b/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java new file mode 100644 index 00000000000..b421d7e02de --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java @@ -0,0 +1,76 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.tests.plugins; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.sonar.orchestrator.Orchestrator; +import com.sonar.orchestrator.build.SonarScanner; +import org.json.JSONException; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.sonarqube.qa.util.Tester; +import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.WsResponse; + +import static org.assertj.core.api.Assertions.assertThat; +import static util.ItUtils.projectDir; +import static util.ItUtils.xooPlugin; + +/** + * Checks the feature of compressing plugins with pack200 and making them available for the scanners. + */ +public class CompressPluginsTest { + @ClassRule + public static Orchestrator orchestrator = Orchestrator.builderEnv() + .addPlugin(xooPlugin()) + .setServerProperty("sonar.pluginsCompression.enable", "true") + .build(); + + @Rule + public Tester tester = new Tester(orchestrator); + + @Before + public void setUp() { + orchestrator.resetData(); + } + + @Test + public void dont_fail_analysis() { + SonarScanner scanner = SonarScanner.create(projectDir("shared/xoo-sample")); + orchestrator.executeBuild(scanner); + } + + @Test + public void plugins_installed_ws_should_expose_compressed_plugin() throws JSONException { + WsResponse response = tester.wsClient().wsConnector().call(new GetRequest("api/plugins/installed")); + String content = response.content(); + JsonParser parser = new JsonParser(); + JsonObject root = parser.parse(content).getAsJsonObject(); + JsonArray plugins = root.getAsJsonArray("plugins"); + plugins.forEach(p -> { + assertThat(p.getAsJsonObject().has("compressedHash")).isTrue(); + assertThat(p.getAsJsonObject().has("compressedFilename")).isTrue(); + }); + } +} diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java b/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java new file mode 100644 index 00000000000..6d896f3a911 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.tests.plugins; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + CompressPluginsTest.class, +}) +public class CompressedPluginsSuite { + +}