import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import javax.annotation.CheckForNull;
import org.apache.commons.lang.SystemUtils;
assertThat(childDir2).doesNotExist();
}
+ @Test
+ public void getPack200FilePath_transforms_jar_name() {
+ Path jarPath = Paths.get("plugin.jar");
+ assertThat(FileUtils.getPack200FilePath(jarPath)).isEqualTo(Paths.get("plugin.pack.gz"));
+ }
+
private void expectDirectoryCanNotBeNullNPE() {
expectedException.expect(NullPointerException.class);
expectedException.expectMessage("Directory can not be null");
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.jar.JarOutputStream;
import java.util.jar.Pack200;
import java.util.zip.GZIPInputStream;
/** Maximum loop count when creating temp directories. */
private static final int TEMP_DIR_ATTEMPTS = 10_000;
- private final File dir;
+ private final File cacheDir;
private final File tmpDir;
private final FileHashes hashes;
private final Logger logger;
FileCache(File dir, FileHashes fileHashes, Logger logger) {
this.hashes = fileHashes;
this.logger = logger;
- this.dir = createDir(dir, "user cache: ");
+ this.cacheDir = createDir(dir, "user cache: ");
logger.info(String.format("User cache: %s", dir.getAbsolutePath()));
this.tmpDir = createDir(new File(dir, "_tmp"), "temp dir");
}
}
public File getDir() {
- return dir;
+ return cacheDir;
}
/**
*/
@CheckForNull
public File get(String filename, String hash) {
- File cachedFile = new File(new File(dir, hash), filename);
+ File cachedFile = new File(new File(cacheDir, hash), filename);
if (cachedFile.exists()) {
return cachedFile;
}
void download(String filename, File toFile) throws IOException;
}
- public File get(String jarFilename, String hash, Downloader downloader) {
+ public File get(String filename, String hash, Downloader downloader) {
// Does not fail if another process tries to create the directory at the same time.
File hashDir = hashDir(hash);
- File targetFile = new File(hashDir, jarFilename);
+ File targetFile = new File(hashDir, filename);
if (!targetFile.exists()) {
- File tempPackedFile = newTempFile();
- File tempJarFile = newTempFile();
- String packedFileName = getPackedFileName(jarFilename);
- download(downloader, packedFileName, tempPackedFile);
-
- logger.debug("Unpacking plugin " + jarFilename);
-
- unpack200(tempPackedFile.toPath(), tempJarFile.toPath());
- logger.debug("Done");
- String downloadedHash = hashes.of(tempJarFile);
- // if (!hash.equals(downloadedHash)) {
- // throw new IllegalStateException("INVALID HASH: File " + tempJarFile.getAbsolutePath() + " was expected to have hash " + hash
- // + " but was downloaded with hash " + downloadedHash);
- // }
- mkdirQuietly(hashDir);
- renameQuietly(tempJarFile, targetFile);
-
+ cacheMiss(targetFile, hash, downloader);
}
return targetFile;
}
- private static String getPackedFileName(String jarName) {
- return jarName.substring(0, jarName.length() - 3) + "pack.gz";
+ private void cacheMiss(File targetFile, String expectedHash, Downloader downloader) {
+ File tempFile = newTempFile();
+ download(downloader, targetFile.getName(), tempFile);
+ String downloadedHash = hashes.of(tempFile);
+ if (!expectedHash.equals(downloadedHash)) {
+ throw new IllegalStateException("INVALID HASH: File " + tempFile.getAbsolutePath() + " was expected to have hash " + expectedHash
+ + " but was downloaded with hash " + downloadedHash);
+ }
+ mkdirQuietly(targetFile.getParentFile());
+ renameQuietly(tempFile, targetFile);
+ }
+
+ public File getCompressed(String filename, String hash, Downloader downloader) {
+ File hashDir = hashDir(hash);
+ File compressedFile = new File(hashDir, filename);
+ File jarFile = new File(compressedFile.getParentFile(), getUnpackedFileName(compressedFile.getName()));
+
+ if (!jarFile.exists()) {
+ if (!compressedFile.exists()) {
+ cacheMiss(compressedFile, hash, downloader);
+ }
+ File tempFile = newTempFile();
+ unpack200(compressedFile.toPath(), tempFile.toPath());
+ renameQuietly(tempFile, jarFile);
+ }
+ return jarFile;
+ }
+
+ private static String getUnpackedFileName(String packedName) {
+ return packedName.substring(0, packedName.length() - 7) + "jar";
}
- private static void unpack200(Path tempFile, Path targetFile) {
+ private void unpack200(Path compressedFile, Path jarFile) {
+ logger.debug("Unpacking plugin " + compressedFile);
Pack200.Unpacker unpacker = Pack200.newUnpacker();
try {
- try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(targetFile)));
- InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(tempFile)))) {
+ try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(jarFile)));
+ InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(compressedFile)))) {
unpacker.unpack(in, jarStream);
}
} catch (IOException e) {
}
private File hashDir(String hash) {
- return new File(dir, hash);
+ return new File(cacheDir, hash);
}
private static void mkdirQuietly(File hashDir) {
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
-import org.assertj.core.util.Files;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class FileCacheTest {
thrown.expectMessage("Fail to download");
cache.get("sonar-foo-plugin-1.5.jar", "ABCDE", downloader);
}
-
+
@Test
public void fail_create_hash_dir() throws IOException {
File file = tempFolder.newFile();
thrown.expectMessage("Unable to create user cache");
cache = new FileCache(file, fileHashes, mock(Logger.class));
}
-
+
@Test
public void fail_to_create_hash_dir() throws IOException {
when(fileHashes.of(any(File.class))).thenReturn("ABCDE");
-
+
File hashDir = new File(cache.getDir(), "ABCDE");
hashDir.createNewFile();
thrown.expect(IllegalStateException.class);
assertThat(FileUtils.readFileToString(cachedFile)).isEqualTo("body");
}
+ @Test
+ public void download_and_add_to_cache_compressed_file() throws IOException {
+ when(fileHashes.of(any(File.class))).thenReturn("ABCDE");
+
+ FileCache.Downloader downloader = new FileCache.Downloader() {
+ public void download(String filename, File toFile) throws IOException {
+ FileUtils.copyFile(compressedFile(), toFile);
+ }
+ };
+ File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader);
+ assertThat(cachedFile).isNotNull().exists().isFile();
+
+ assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar");
+ assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz");
+ assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir());
+ }
+
+ @Test
+ public void dont_download_compressed_file_if_jar_exists() throws IOException {
+ when(fileHashes.of(any(File.class))).thenReturn("ABCDE");
+ FileCache.Downloader downloader = mock(FileCache.Downloader.class);
+
+ File hashDir = new File(cache.getDir(), "ABCDE");
+ hashDir.mkdirs();
+ File jar = new File(new File(cache.getDir(), "ABCDE"), "sonar-foo-plugin-1.5.jar");
+ jar.createNewFile();
+ File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader);
+ assertThat(cachedFile).isNotNull().exists().isFile();
+
+ assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar");
+ assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar");
+ assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir());
+
+ verifyZeroInteractions(downloader);
+ }
+
+ @Test
+ public void dont_download_compressed_file_if_it_exists() throws IOException {
+ when(fileHashes.of(any(File.class))).thenReturn("ABCDE");
+ FileCache.Downloader downloader = mock(FileCache.Downloader.class);
+
+ File hashDir = new File(cache.getDir(), "ABCDE");
+ hashDir.mkdirs();
+ FileUtils.copyFile(compressedFile(), new File(hashDir, "sonar-foo-plugin-1.5.pack.gz"));
+ File cachedFile = cache.getCompressed("sonar-foo-plugin-1.5.pack.gz", "ABCDE", downloader);
+ assertThat(cachedFile).isNotNull().exists().isFile();
+
+ assertThat(cachedFile.getName()).isEqualTo("sonar-foo-plugin-1.5.jar");
+ assertThat(cachedFile.getParentFile().list()).containsOnly("sonar-foo-plugin-1.5.jar", "sonar-foo-plugin-1.5.pack.gz");
+ assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir());
+
+ verifyZeroInteractions(downloader);
+ }
+
@Test
public void download_corrupted_file() throws IOException {
thrown.expect(IllegalStateException.class);
assertThat(cachedFile.getParentFile().getParentFile()).isEqualTo(cache.getDir());
assertThat(FileUtils.readFileToString(cachedFile)).contains("downloaded by");
}
+
+ private File compressedFile() {
+ return new File("src/test/resources/test.pack.gz");
+ }
}
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
@VisibleForTesting
File download(final InstalledPlugin remote) {
try {
- return fileCache.get(remote.filename, remote.hash, new FileDownloader(remote.key));
+ if (remote.compressedFilename != null) {
+ return fileCache.getCompressed(remote.compressedFilename, remote.compressedHash, new FileDownloader(remote.key));
+ } else {
+ return fileCache.get(remote.filename, remote.hash, new FileDownloader(remote.key));
+ }
} catch (Exception e) {
throw new IllegalStateException("Fail to download plugin: " + remote.key, e);
}
String hash;
String filename;
long updatedAt;
+ @Nullable
+ String compressedHash;
+ @Nullable
+ String compressedFilename;
}
private class FileDownloader implements FileCache.Downloader {
assertThat(file).isEqualTo(pluginJar);
}
+ @Test
+ public void should_download_compressed_plugin() throws Exception {
+ File pluginJar = temp.newFile();
+ when(fileCache.getCompressed(eq("checkstyle-plugin.pack.gz"), eq("hash"), any(FileCache.Downloader.class))).thenReturn(pluginJar);
+
+ ScannerPluginInstaller underTest = new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate);
+
+ InstalledPlugin remote = new InstalledPlugin();
+ remote.key = "checkstyle";
+ remote.filename = "checkstyle-plugin.jar";
+ remote.hash = "fakemd5_1";
+ remote.compressedFilename = "checkstyle-plugin.pack.gz";
+ remote.compressedHash = "hash";
+ File file = underTest.download(remote);
+
+ assertThat(file).isEqualTo(pluginJar);
+ }
+
@Test
public void should_fail_to_get_plugin_index() {
WsTestUtil.mockException(wsClient, "/api/plugins/installed", new IllegalStateException());
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonarqube.tests.plugins;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.build.SonarScanner;
+import org.json.JSONException;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonarqube.qa.util.Tester;
+import org.sonarqube.ws.client.GetRequest;
+import org.sonarqube.ws.client.WsResponse;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static util.ItUtils.projectDir;
+import static util.ItUtils.xooPlugin;
+
+/**
+ * Checks the feature of compressing plugins with pack200 and making them available for the scanners.
+ */
+public class CompressPluginsTest {
+ @ClassRule
+ public static Orchestrator orchestrator = Orchestrator.builderEnv()
+ .addPlugin(xooPlugin())
+ .setServerProperty("sonar.pluginsCompression.enable", "true")
+ .build();
+
+ @Rule
+ public Tester tester = new Tester(orchestrator);
+
+ @Before
+ public void setUp() {
+ orchestrator.resetData();
+ }
+
+ @Test
+ public void dont_fail_analysis() {
+ SonarScanner scanner = SonarScanner.create(projectDir("shared/xoo-sample"));
+ orchestrator.executeBuild(scanner);
+ }
+
+ @Test
+ public void plugins_installed_ws_should_expose_compressed_plugin() throws JSONException {
+ WsResponse response = tester.wsClient().wsConnector().call(new GetRequest("api/plugins/installed"));
+ String content = response.content();
+ JsonParser parser = new JsonParser();
+ JsonObject root = parser.parse(content).getAsJsonObject();
+ JsonArray plugins = root.getAsJsonArray("plugins");
+ plugins.forEach(p -> {
+ assertThat(p.getAsJsonObject().has("compressedHash")).isTrue();
+ assertThat(p.getAsJsonObject().has("compressedFilename")).isTrue();
+ });
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonarqube.tests.plugins;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ CompressPluginsTest.class,
+})
+public class CompressedPluginsSuite {
+
+}
projects / sonarqube.git / commitdiff