From 0001182a007e70261d447023f1fa5ad5e1a0b69f Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Thu, 7 Dec 2017 11:38:28 +0100 Subject: [PATCH] SONAR-10142 Download compressed plugins --- .../org/sonar/core/util/FileUtilsTest.java | 7 ++ .../java/org/sonar/home/cache/FileCache.java | 73 ++++++++++------- .../org/sonar/home/cache/FileCacheTest.java | 66 ++++++++++++++- sonar-home/src/test/resources/test.pack.gz | Bin 0 -> 2689 bytes .../bootstrap/ScannerPluginInstaller.java | 11 ++- .../bootstrap/ScannerPluginInstallerTest.java | 18 +++++ .../tests/plugins/CompressPluginsTest.java | 76 ++++++++++++++++++ .../tests/plugins/CompressedPluginsSuite.java | 31 +++++++ 8 files changed, 246 insertions(+), 36 deletions(-) create mode 100644 sonar-home/src/test/resources/test.pack.gz create mode 100644 tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java create mode 100644 tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java diff --git a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java index a1054b47650..1aa95c7245d 100644 --- a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java +++ b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import javax.annotation.CheckForNull; import org.apache.commons.lang.SystemUtils; @@ -238,6 +239,12 @@ public class FileUtilsTest { assertThat(childDir2).doesNotExist(); } + @Test + public void getPack200FilePath_transforms_jar_name() { + Path jarPath = Paths.get("plugin.jar"); + assertThat(FileUtils.getPack200FilePath(jarPath)).isEqualTo(Paths.get("plugin.pack.gz")); + } + private void expectDirectoryCanNotBeNullNPE() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("Directory can not be null"); diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java index 4a03ec24794..e3cbafd22de 100644 --- a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java +++ b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java @@ -24,10 +24,8 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.jar.JarOutputStream; import java.util.jar.Pack200; import java.util.zip.GZIPInputStream; @@ -43,7 +41,7 @@ public class FileCache { /** Maximum loop count when creating temp directories. */ private static final int TEMP_DIR_ATTEMPTS = 10_000; - private final File dir; + private final File cacheDir; private final File tmpDir; private final FileHashes hashes; private final Logger logger; @@ -51,7 +49,7 @@ public class FileCache { FileCache(File dir, FileHashes fileHashes, Logger logger) { this.hashes = fileHashes; this.logger = logger; - this.dir = createDir(dir, "user cache: "); + this.cacheDir = createDir(dir, "user cache: "); logger.info(String.format("User cache: %s", dir.getAbsolutePath())); this.tmpDir = createDir(new File(dir, "_tmp"), "temp dir"); } @@ -61,7 +59,7 @@ public class FileCache { } public File getDir() { - return dir; + return cacheDir; } /** @@ -70,7 +68,7 @@ public class FileCache { */ @CheckForNull public File get(String filename, String hash) { - File cachedFile = new File(new File(dir, hash), filename); + File cachedFile = new File(new File(cacheDir, hash), filename); if (cachedFile.exists()) { return cachedFile; } @@ -82,41 +80,54 @@ public class FileCache { void download(String filename, File toFile) throws IOException; } - public File get(String jarFilename, String hash, Downloader downloader) { + public File get(String filename, String hash, Downloader downloader) { // Does not fail if another process tries to create the directory at the same time. File hashDir = hashDir(hash); - File targetFile = new File(hashDir, jarFilename); + File targetFile = new File(hashDir, filename); if (!targetFile.exists()) { - File tempPackedFile = newTempFile(); - File tempJarFile = newTempFile(); - String packedFileName = getPackedFileName(jarFilename); - download(downloader, packedFileName, tempPackedFile); - - logger.debug("Unpacking plugin " + jarFilename); - - unpack200(tempPackedFile.toPath(), tempJarFile.toPath()); - logger.debug("Done"); - String downloadedHash = hashes.of(tempJarFile); - // if (!hash.equals(downloadedHash)) { - // throw new IllegalStateException("INVALID HASH: File " + tempJarFile.getAbsolutePath() + " was expected to have hash " + hash - // + " but was downloaded with hash " + downloadedHash); - // } - mkdirQuietly(hashDir); - renameQuietly(tempJarFile, targetFile); - + cacheMiss(targetFile, hash, downloader); } return targetFile; } - private static String getPackedFileName(String jarName) { - return jarName.substring(0, jarName.length() - 3) + "pack.gz"; + private void cacheMiss(File targetFile, String expectedHash, Downloader downloader) { + File tempFile = newTempFile(); + download(downloader, targetFile.getName(), tempFile); + String downloadedHash = hashes.of(tempFile); + if (!expectedHash.equals(downloadedHash)) { + throw new IllegalStateException("INVALID HASH: File " + tempFile.getAbsolutePath() + " was expected to have hash " + expectedHash + + " but was downloaded with hash " + downloadedHash); + } + mkdirQuietly(targetFile.getParentFile()); + renameQuietly(tempFile, targetFile); + } + + public File getCompressed(String filename, String hash, Downloader downloader) { + File hashDir = hashDir(hash); + File compressedFile = new File(hashDir, filename); + File jarFile = new File(compressedFile.getParentFile(), getUnpackedFileName(compressedFile.getName())); + + if (!jarFile.exists()) { + if (!compressedFile.exists()) { + cacheMiss(compressedFile, hash, downloader); + } + File tempFile = newTempFile(); + unpack200(compressedFile.toPath(), tempFile.toPath()); + renameQuietly(tempFile, jarFile); + } + return jarFile; + } + + private static String getUnpackedFileName(String packedName) { + return packedName.substring(0, packedName.length() - 7) + "jar"; } - private static void unpack200(Path tempFile, Path targetFile) { + private void unpack200(Path compressedFile, Path jarFile) { + logger.debug("Unpacking plugin " + compressedFile); Pack200.Unpacker unpacker = Pack200.newUnpacker(); try { - try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(targetFile))); - InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(tempFile)))) { + try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(jarFile))); + InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(compressedFile)))) { unpacker.unpack(in, jarStream); } } catch (IOException e) { @@ -147,7 +158,7 @@ public class FileCache { } private File hashDir(String hash) { - return new File(dir, hash); + return new File(cacheDir, hash); } private static void mkdirQuietly(File hashDir) { diff --git a/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java b/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java index 2a7381c866a..67635b5ab70 100644 --- a/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java +++ b/sonar-home/src/test/java/org/sonar/home/cache/FileCacheTest.java @@ -22,7 +22,6 @@ package org.sonar.home.cache; import java.io.File; import java.io.IOException; import org.apache.commons.io.FileUtils; -import org.assertj.core.util.Files; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -32,6 +31,7 @@ import org.junit.rules.TemporaryFolder; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class FileCacheTest { @@ -77,7 +77,7 @@ public class FileCacheTest { thrown.expectMessage("Fail to download"); cache.get("sonar-foo-plugin-1.5.jar", "ABCDE", downloader); } - + @Test public void fail_create_hash_dir() throws IOException { File file = tempFolder.newFile(); @@ -85,11 +85,11 @@ public class FileCacheTest { thrown.expectMessage("Unable to create user cache"); cache = new FileCache(file, fileHashes, mock(Logger.class)); } - + @Test public void fail_to_create_hash_dir() throws IOException { when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); - + File hashDir = new File(cache.getDir(), "ABCDE"); hashDir.createNewFile(); thrown.expect(IllegalStateException.class); @@ -113,6 +113,60 @@ public class FileCacheTest { assertThat(FileUtils.readFileToString(cachedFile)).isEqualTo("body"); } + @Test + public void download_and_add_to_cache_compressed_file() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + + FileCache.Downloader downloader = new FileCache.Downloader() { + public void download(String filename, File toFile) throws IOException { + FileUtils.copyFile(compressedFile(), toFile); + } + }; + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + } + + @Test + public void dont_download_compressed_file_if_jar_exists() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + FileCache.Downloader downloader = mock(FileCache.Downloader.class); + + File hashDir = new File(cache.getDir(), "ABCDE"); + hashDir.mkdirs(); + File jar = new File(new File(cache.getDir(), "ABCDE"), "sonar-foo-plugin-1.5.jar"); + jar.createNewFile(); + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + + verifyZeroInteractions(downloader); + } + + @Test + public void dont_download_compressed_file_if_it_exists() throws IOException { + when(fileHashes.of(any(File.class))).thenReturn("ABCDE"); + FileCache.Downloader downloader = mock(FileCache.Downloader.class); + + File hashDir = new File(cache.getDir(), "ABCDE"); + hashDir.mkdirs(); + FileUtils.copyFile(compressedFile(), new File(hashDir, "sonar-foo-plugin-1.5.pack.gz")); + File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader); + assertThat(cachedFile).isNotNull().exists().isFile(); + + assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar"); + assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz"); + assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); + + verifyZeroInteractions(downloader); + } + @Test public void download_corrupted_file() throws IOException { thrown.expect(IllegalStateException.class); @@ -149,4 +203,8 @@ public class FileCacheTest { assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir()); assertThat(FileUtils.readFileToString(cachedFile)).contains("downloaded by"); } + + private File compressedFile() { + return new File("src/test/resources/test.pack.gz"); + } } diff --git a/sonar-home/src/test/resources/test.pack.gz b/sonar-home/src/test/resources/test.pack.gz new file mode 100644 index 0000000000000000000000000000000000000000..8e3a43c8e3d2b6c270cf0f85f60418d795fa7a80 GIT binary patch literal 2689 zcmV-{3V!t;iwFP!000000L@y>ZyU!IUrI79(Uz^Kh!Hq$I#%q$AKhI_jva%hH=y-l z%ONE}A9jEs2u8a@a;)VpJF_c^HgfE#_x=;Tv=`rdOWU+5&;b1dj5?PBMS>m*G>Q9W zc6PZGDOn2Kbm>6ZdGqGYZ{EE5y_uc;!*_ofKmMyPH0}4l*1rDqA79;{ypkImJDwZS zG;KOJlFR3gY1*VVstt{uyipwer8aT;tugIK+UUu%+H1!r&Oqh+nl_RUb#lj9>3mo8UtJ*eJ_{>uNCsYe5=daE|S zUeOnq7R*{@X>nnGZQZCXxOCIRUf?o)#(3AbVLZajZW4Qo2OXE_*Ntf*@mnq5XHTtW?jsA)yEEDkMK4&UFoMmY1&v$%jZV& zx!k|MfgeQ2ci+Unp%MAKI)85H$7e5T!#VMrfEXGX8Ga#Z2F?uY*?s`y?O5Z)P;*$;l#=DQ>P~;U(-Gh|4>yeB$20GY7gpr7>M|BfNW^|m$Q#`VFlLj-s8P~*nKfVxk7a~yFt>!7LC`Ll z=FZMeWG$L}SXHg0%_8R^S!dYZA}rnJ5oNf${KOjAWd06^_*`D9tPbq5gw{kSYqFC> zV~(_m=Mc|^P%WW^B?L9t{Y!t3LKEzuJ$%utvWH%hZkKS2lu*}INkuY^GsewCD5qo& zCK1HN>1n-qU7x-2R1>%U1)~}O;pCH z$w>Y7l=5b3_FzEKG}OVV#PCR9PFY=?WfNOSj5O)*eFEWQ_Yb5>ZQ^l2e0OOBRqw7; z(Y2)Rp5G#EAeenaKuqZVYzi2ABWst4i%BC#K8o|gRAh!fRm(8RoE195J`KSu&S(HfI=QQwH2mEE(DS!6X zBn`HNn#dPnyWO!F-f@k<_uZbA7Cnk(0QtZGngo~uVHM74!)pP#%6TiBTZv)lt_$l2 zgy;|f$+$-yvH`h<1|7*HFl9Vop$(=p166r*jkek@t#|H#8o}QDZks_;glNeOPzXv1 zGyrz9A7DoDrb`5`WCmdiTv?}t88!8=%wQ9f8FS#R*jASXp<{--lbqBjGpOMs&E;*3y3jq|#tLu7= z1V3D7l}#8Xwu9*Ou)miN-)l<7|IY!b=n(d%*p-KnOdc=QDd_>_bW^@l$m%w$h8xsg z3PncF47BtVXL5Fvl&sc7roaIrMs05Nza8=+(U9Ft9Y++_*^ce{E%CDG0Fd7&&$RIF zzVtU;d==yuJ&KBQe8Xj-g7{Z{Z@1;5_(ks2v{9Tw#EaJ5?Wv9R1^wpKdsacb-W6A8 z2(&$(y!Vzjc?OnuBJ!Q{%5u$o$0!ObQnR$nX`(G&Qtw=odNal1wE01;x&{qfSf_GJ zYZGY2X^BgrSXqGt_0rDJej2;niG;e;GoUuLJt6yRvQ2(_SS_b{Vx1uPvnC-*|ptJNVwG{ zyF_TE*)hSX?igLaAp)@vY#-pg`u(Vds|IxXav(;-_ZuQ__^u0FDaP?qJW&(9r^zdX z6pa-h#WDC5dnLL8K7vaIwmBrAl!y-s`$%+N*U|d&-16Zr!*y|GEE_%~$Ror-bwK9t z2zu$-{u%NEpr0Me3EC0Y#IB1Nu?f(&Y%A_N4dGm4$(L6WbELlDhn_Rfn9t-V0cMoB zqKsv#jBCnI4OXIpwB&^17Jr;A#?y>+SkH386q!PGY(+)#}STXc6>w$=5+@si4+ z63KxC5M@_|^#Do=2oDaRqyG(AicY~*dJ{7bwwK=@UQaop`%UpxL6WwW*Av6BBo(g? zg@_PaQHwJhrkQ$X(hfREYP&69rv^JZkzu2AaJQTUd~ZM9avc9+Xsdnb($awlFFe`A zoIv>#^H5;R<{*`%R`kq_N89 zKb)PbtXDof`Uf>H{hDUr=EC^sBP#}&29io11DzeWQIw>T#cJqzf^+Y0cr-vF*f^pd z5=Sn`QEo!3FN?A?tq$Cs_s3ckTRa5sz*0Ck$PNe}h16=ruiFwyS}lM{z|JhBrNFud zt14COu}0Xe&&YbJItuCm6^D0ty2K;SRSa8>lcDwYUE&cYzQ#kFo#-pP!rKp_RKp!K veG|o>bfMY#wRQQeS0O&z!da`JJbiorvr?h!*xUOYj|%? { + assertThat(p.getAsJsonObject().has("compressedHash")).isTrue(); + assertThat(p.getAsJsonObject().has("compressedFilename")).isTrue(); + }); + } +} diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java b/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java new file mode 100644 index 00000000000..6d896f3a911 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java @@ -0,0 +1,31 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.tests.plugins; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + CompressPluginsTest.class, +}) +public class CompressedPluginsSuite { + +} -- 2.39.5