From: Simon Brandhof Date: Thu, 10 May 2018 19:47:15 +0000 (+0200) Subject: SONAR-10591 scanner uses WS api/plugins/download X-Git-Tag: 7.5~1207 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=01e1c69fe35fac0678130f5784ee0b70422c8536;p=sonarqube.git SONAR-10591 scanner uses WS api/plugins/download --- diff --git a/sonar-core/src/main/java/org/sonar/core/platform/RemotePlugin.java b/sonar-core/src/main/java/org/sonar/core/platform/RemotePlugin.java deleted file mode 100644 index 54201e21861..00000000000 --- a/sonar-core/src/main/java/org/sonar/core/platform/RemotePlugin.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.core.platform; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang.StringUtils; - -/** - * @deprecated since 6.6 Used for deprecated deploy/plugin/index.txt - * - */ -@Deprecated -public class RemotePlugin { - private String pluginKey; - private boolean sonarLintSupported; - private RemotePluginFile file = null; - - public RemotePlugin(String pluginKey) { - this.pluginKey = pluginKey; - } - - public static RemotePlugin create(PluginInfo pluginInfo) { - RemotePlugin result = new RemotePlugin(pluginInfo.getKey()); - result.setFile(pluginInfo.getNonNullJarFile()); - result.setSonarLintSupported(pluginInfo.isSonarLintSupported()); - return result; - } - - public static RemotePlugin unmarshal(String row) { - String[] fields = StringUtils.split(row, ","); - RemotePlugin result = new RemotePlugin(fields[0]); - if (fields.length >= 3) { - result.setSonarLintSupported(StringUtils.equals("true", fields[1])); - String[] nameAndHash = StringUtils.split(fields[2], "|"); - result.setFile(nameAndHash[0], nameAndHash[1]); - } - return result; - } - - public String marshal() { - StringBuilder sb = new StringBuilder(); - sb.append(pluginKey) - .append(",") - .append(sonarLintSupported) - .append(",") - .append(file.getFilename()) - .append("|") - .append(file.getHash()); - return sb.toString(); - } - - public String getKey() { - return pluginKey; - } - - public RemotePlugin setFile(String filename, String hash) { - file = new RemotePluginFile(filename, hash); - return this; - } - - public RemotePlugin setSonarLintSupported(boolean sonarLintPlugin) { - this.sonarLintSupported = sonarLintPlugin; - return this; - } - - public RemotePlugin setFile(File f) { - try (FileInputStream fis = new FileInputStream(f)) { - return this.setFile(f.getName(), DigestUtils.md5Hex(fis)); - } catch (IOException e) { - throw new IllegalStateException("Fail to compute hash", e); - } - } - - public RemotePluginFile file() { - return file; - } - - public boolean isSonarLintSupported() { - return sonarLintSupported; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RemotePlugin that = (RemotePlugin) o; - return pluginKey.equals(that.pluginKey); - } - - @Override - public int hashCode() { - return pluginKey.hashCode(); - } -} diff --git a/sonar-core/src/test/java/org/sonar/core/platform/RemotePluginTest.java b/sonar-core/src/test/java/org/sonar/core/platform/RemotePluginTest.java deleted file mode 100644 index f67161eca13..00000000000 --- a/sonar-core/src/test/java/org/sonar/core/platform/RemotePluginTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.core.platform; - -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class RemotePluginTest { - @Test - public void shouldEqual() { - RemotePlugin clirr1 = new RemotePlugin("clirr"); - RemotePlugin clirr2 = new RemotePlugin("clirr"); - RemotePlugin checkstyle = new RemotePlugin("checkstyle"); - assertThat(clirr1).isEqualTo(clirr2); - assertThat(clirr1).isEqualTo(clirr1); - assertThat(clirr1).isNotEqualTo(checkstyle); - } - - @Test - public void shouldMarshalNotSonarLintByDefault() { - RemotePlugin clirr = new RemotePlugin("clirr").setFile("clirr-1.1.jar", "fakemd5"); - String text = clirr.marshal(); - assertThat(text).isEqualTo("clirr,false,clirr-1.1.jar|fakemd5"); - } - - @Test - public void shouldMarshalSonarLint() { - RemotePlugin clirr = new RemotePlugin("clirr").setFile("clirr-1.1.jar", "fakemd5").setSonarLintSupported(true); - String text = clirr.marshal(); - assertThat(text).isEqualTo("clirr,true,clirr-1.1.jar|fakemd5"); - } - - @Test - public void shouldUnmarshal() { - RemotePlugin clirr = RemotePlugin.unmarshal("clirr,true,clirr-1.1.jar|fakemd5"); - assertThat(clirr.getKey()).isEqualTo("clirr"); - assertThat(clirr.isSonarLintSupported()).isTrue(); - assertThat(clirr.file().getFilename()).isEqualTo("clirr-1.1.jar"); - assertThat(clirr.file().getHash()).isEqualTo("fakemd5"); - } -} diff --git a/sonar-scanner-engine/build.gradle b/sonar-scanner-engine/build.gradle index c7d662c821d..b58fbf87ce6 100644 --- a/sonar-scanner-engine/build.gradle +++ b/sonar-scanner-engine/build.gradle @@ -36,6 +36,7 @@ dependencies { compileOnly 'com.google.code.findbugs:jsr305' + testCompile 'com.squareup.okhttp3:mockwebserver' testCompile 'com.tngtech.java:junit-dataprovider' testCompile 'javax.servlet:javax.servlet-api' testCompile 'junit:junit' diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/FileCacheProvider.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/FileCacheProvider.java deleted file mode 100644 index 7deea6e36d6..00000000000 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/FileCacheProvider.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.bootstrap; - -import java.io.File; -import org.picocontainer.injectors.ProviderAdapter; -import org.sonar.api.config.Configuration; -import org.sonar.home.cache.FileCache; -import org.sonar.home.cache.FileCacheBuilder; - -public class FileCacheProvider extends ProviderAdapter { - private FileCache cache; - - public FileCache provide(Configuration settings) { - if (cache == null) { - File home = settings.get("sonar.userHome") - .map(File::new) - .orElse(null); - cache = new FileCacheBuilder(new Slf4jLogger()).setUserHome(home).build(); - } - return cache; - } -} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/GlobalContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/GlobalContainer.java index bdb8202809b..1aea964a478 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/GlobalContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/GlobalContainer.java @@ -93,7 +93,7 @@ public class GlobalContainer extends ComponentContainer { new GlobalTempFolderProvider(), DefaultHttpDownloader.class, UriReader.class, - new FileCacheProvider(), + PluginFiles.class, System2.INSTANCE, Clock.systemDefaultZone(), new MetricsRepositoryProvider(), diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginFiles.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginFiles.java new file mode 100644 index 00000000000..697908b40db --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/PluginFiles.java @@ -0,0 +1,237 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.nio.file.Files; +import java.util.Objects; +import java.util.Optional; +import java.util.jar.JarOutputStream; +import java.util.jar.Pack200; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.sonar.api.config.Configuration; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.scanner.bootstrap.ScannerPluginInstaller.InstalledPlugin; +import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.HttpException; +import org.sonarqube.ws.client.WsResponse; + +import static java.lang.String.format; + +public class PluginFiles { + + private static final Logger LOGGER = Loggers.get(PluginFiles.class); + private static final String MD5_HEADER = "Sonar-MD5"; + private static final String COMPRESSION_HEADER = "Sonar-Compression"; + private static final String PACK200 = "pack200"; + private static final String UNCOMPRESSED_MD5_HEADER = "Sonar-UncompressedMD5"; + + private final ScannerWsClient wsClient; + private final File cacheDir; + private final File tempDir; + + public PluginFiles(ScannerWsClient wsClient, Configuration configuration) { + this.wsClient = wsClient; + File home = locateHomeDir(configuration); + this.cacheDir = mkdir(new File(home, "cache"), "user cache"); + this.tempDir = mkdir(new File(home, "_tmp"), "temp dir"); + LOGGER.info("User cache: {}", cacheDir.getAbsolutePath()); + } + + public File createTempDir() { + try { + return Files.createTempDirectory(tempDir.toPath(), "plugins").toFile(); + } catch (IOException e) { + throw new IllegalStateException("Fail to create temp directory in " + tempDir, e); + } + } + + /** + * Get the JAR file of specified plugin. If not present in user local cache, + * then it's downloaded from server and added to cache. + * + * @return the file, or {@link Optional#empty()} if plugin not found (404 HTTP code) + * @throws IllegalStateException if the plugin can't be downloaded (not 404 nor 2xx HTTP codes) + * or can't be cached locally. + */ + public Optional get(InstalledPlugin plugin) { + // Does not fail if another process tries to create the directory at the same time. + File jarInCache = jarInCache(plugin.key, plugin.hash); + if (jarInCache.exists() && jarInCache.isFile()) { + return Optional.of(jarInCache); + } + return download(plugin); + } + + private Optional download(InstalledPlugin plugin) { + GetRequest request = new GetRequest("api/plugins/download") + .setParam("plugin", plugin.key) + .setParam("acceptCompressions", PACK200); + + File downloadedFile = newTempFile(); + LOGGER.debug("Download plugin '{}' to '{}'", plugin.key, downloadedFile); + + try (WsResponse response = wsClient.call(request)) { + Optional expectedMd5 = response.header(MD5_HEADER); + if (!expectedMd5.isPresent()) { + throw new IllegalStateException(format( + "Fail to download plugin [%s]. Request to %s did not return header %s", plugin.key, response.requestUrl(), MD5_HEADER)); + } + + downloadBinaryTo(plugin, downloadedFile, response); + + // verify integrity + String effectiveTempMd5 = computeMd5(downloadedFile); + if (!expectedMd5.get().equals(effectiveTempMd5)) { + throw new IllegalStateException(format( + "Fail to download plugin [%s]. File %s was expected to have checksum %s but had %s", plugin.key, downloadedFile, expectedMd5.get(), effectiveTempMd5)); + } + + // un-compress if needed + String cacheMd5; + File tempJar; + Optional compression = response.header(COMPRESSION_HEADER); + if (compression.isPresent() && PACK200.equals(compression.get())) { + tempJar = unpack200(plugin.key, downloadedFile); + cacheMd5 = response.header(UNCOMPRESSED_MD5_HEADER).orElseThrow(() -> new IllegalStateException(format( + "Fail to download plugin [%s]. Request to %s did not return header %s.", plugin.key, response.requestUrl(), UNCOMPRESSED_MD5_HEADER))); + } else { + tempJar = downloadedFile; + cacheMd5 = expectedMd5.get(); + } + + // put in cache + File jarInCache = jarInCache(plugin.key, cacheMd5); + mkdir(jarInCache.getParentFile()); + moveFile(tempJar, jarInCache); + return Optional.of(jarInCache); + + } catch (HttpException e) { + if (e.code() == HttpURLConnection.HTTP_NOT_FOUND) { + // Plugin was listed but not longer available. It has probably been + // uninstalled. + return Optional.empty(); + } + + // not 2xx nor 404 + throw new IllegalStateException(format("Fail to download plugin [%s]. Request to %s returned code %d.", plugin.key, e.url(), e.code())); + } + } + + private static void downloadBinaryTo(InstalledPlugin plugin, File downloadedFile, WsResponse response) { + try (InputStream stream = response.contentStream()) { + FileUtils.copyInputStreamToFile(stream, downloadedFile); + } catch (IOException e) { + throw new IllegalStateException(format("Fail to download plugin [%s] into %s", plugin.key, downloadedFile), e); + } + } + + private File jarInCache(String pluginKey, String hash) { + File hashDir = new File(cacheDir, hash); + File file = new File(hashDir, format("sonar-%s-plugin.jar", pluginKey)); + if (!file.getParentFile().toPath().equals(hashDir.toPath())) { + // vulnerability - attempt to create a file outside the cache directory + throw new IllegalStateException(format("Fail to download plugin [%s]. Key is not valid.", pluginKey)); + } + return file; + } + + private File newTempFile() { + try { + return File.createTempFile("fileCache", null, tempDir); + } catch (IOException e) { + throw new IllegalStateException("Fail to create temp file in " + tempDir, e); + } + } + + private File unpack200(String pluginKey, File compressedFile) { + LOGGER.debug("Unpacking plugin {}", pluginKey); + File jar = newTempFile(); + try (InputStream input = new GZIPInputStream(new BufferedInputStream(FileUtils.openInputStream(compressedFile))); + JarOutputStream output = new JarOutputStream(new BufferedOutputStream(FileUtils.openOutputStream(jar)))) { + Pack200.newUnpacker().unpack(input, output); + } catch (IOException e) { + throw new IllegalStateException(format("Fail to download plugin [%s]. Pack200 error.", pluginKey), e); + } + return jar; + } + + private static String computeMd5(File file) { + try (InputStream fis = new BufferedInputStream(FileUtils.openInputStream(file))) { + return DigestUtils.md5Hex(fis); + } catch (IOException e) { + throw new IllegalStateException("Fail to compute md5 of " + file, e); + } + } + + private static void moveFile(File sourceFile, File targetFile) { + boolean rename = sourceFile.renameTo(targetFile); + // Check if the file was cached by another process during download + if (!rename && !targetFile.exists()) { + LOGGER.warn("Unable to rename {} to {}", sourceFile.getAbsolutePath(), targetFile.getAbsolutePath()); + LOGGER.warn("A copy/delete will be tempted but with no guarantee of atomicity"); + try { + Files.move(sourceFile.toPath(), targetFile.toPath()); + } catch (IOException e) { + throw new IllegalStateException("Fail to move " + sourceFile.getAbsolutePath() + " to " + targetFile, e); + } + } + } + + private static void mkdir(File dir) { + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + throw new IllegalStateException("Fail to create cache directory: " + dir, e); + } + } + + private static File mkdir(File dir, String debugTitle) { + if (!dir.isDirectory() || !dir.exists()) { + LOGGER.debug("Create : {}", dir.getAbsolutePath()); + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + throw new IllegalStateException("Unable to create " + debugTitle + dir.getAbsolutePath(), e); + } + } + return dir; + } + + private static File locateHomeDir(Configuration configuration) { + return Stream.of( + configuration.get("sonar.userHome").orElse(null), + System.getenv("SONAR_USER_HOME"), + System.getProperty("user.home") + File.separator + ".sonar") + .filter(Objects::nonNull) + .findFirst() + .map(File::new) + .get(); + } +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java index fd9ce623721..ce739659d67 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginInstaller.java @@ -19,102 +19,95 @@ */ package org.sonar.scanner.bootstrap; -import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import java.io.File; -import java.io.IOException; -import java.io.InputStream; import java.io.Reader; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.annotation.Nullable; -import org.apache.commons.io.FileUtils; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.api.utils.log.Profiler; import org.sonar.core.platform.PluginInfo; -import org.sonar.home.cache.FileCache; import org.sonarqube.ws.client.GetRequest; -import org.sonarqube.ws.client.WsResponse; import static java.lang.String.format; /** * Downloads the plugins installed on server and stores them in a local user cache - * (see {@link FileCacheProvider}). */ public class ScannerPluginInstaller implements PluginInstaller { private static final Logger LOG = Loggers.get(ScannerPluginInstaller.class); - private static final String PLUGINS_WS_URL = "/api/plugins/installed"; + private static final String PLUGINS_WS_URL = "api/plugins/installed"; - private final FileCache fileCache; + private final PluginFiles pluginFiles; private final ScannerPluginPredicate pluginPredicate; private final ScannerWsClient wsClient; - public ScannerPluginInstaller(ScannerWsClient wsClient, FileCache fileCache, ScannerPluginPredicate pluginPredicate) { - this.fileCache = fileCache; + public ScannerPluginInstaller(PluginFiles pluginFiles, ScannerPluginPredicate pluginPredicate, ScannerWsClient wsClient) { + this.pluginFiles = pluginFiles; this.pluginPredicate = pluginPredicate; this.wsClient = wsClient; } @Override public Map installRemotes() { - return loadPlugins(listInstalledPlugins()); + Profiler profiler = Profiler.create(LOG).startInfo("Load/download plugins"); + try { + Map result = new HashMap<>(); + Loaded loaded = loadPlugins(result); + if (!loaded.ok) { + // retry once, a plugin may have been uninstalled during downloads + result.clear(); + loaded = loadPlugins(result); + if (!loaded.ok) { + throw new IllegalStateException(format("Fail to download plugin [%s]. Not found.", loaded.notFoundPlugin)); + } + } + return result; + } finally { + profiler.stopInfo(); + } } - private Map loadPlugins(InstalledPlugin[] remotePlugins) { - Map infosByKey = new HashMap<>(remotePlugins.length); - - Profiler profiler = Profiler.create(LOG).startInfo("Load/download plugins"); + private Loaded loadPlugins(Map result) { + for (InstalledPlugin plugin : listInstalledPlugins()) { + if (pluginPredicate.apply(plugin.key)) { + Optional jarFile = pluginFiles.get(plugin); + if (!jarFile.isPresent()) { + return new Loaded(false, plugin.key); + } - for (InstalledPlugin installedPlugin : remotePlugins) { - if (pluginPredicate.apply(installedPlugin.key)) { - File jarFile = download(installedPlugin); - PluginInfo info = PluginInfo.create(jarFile); - infosByKey.put(info.getKey(), new ScannerPlugin(installedPlugin.key, installedPlugin.updatedAt, info)); + PluginInfo info = PluginInfo.create(jarFile.get()); + result.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, info)); } } - profiler.stopInfo(); - return infosByKey; + return new Loaded(true, null); } /** * Returns empty on purpose. This method is used only by medium tests. - * @see org.sonar.scanner.mediumtest.ScannerMediumTester */ @Override public List installLocals() { return Collections.emptyList(); } - @VisibleForTesting - File download(final InstalledPlugin remote) { - try { - if (remote.compressedFilename != null) { - return fileCache.getCompressed(remote.compressedFilename, remote.compressedHash, new FileDownloader(remote.key)); - } else { - return fileCache.get(remote.filename, remote.hash, new FileDownloader(remote.key)); - } - } catch (Exception e) { - throw new IllegalStateException("Fail to download plugin: " + remote.key, e); - } - } - /** * Gets information about the plugins installed on server (filename, checksum) */ - @VisibleForTesting - InstalledPlugin[] listInstalledPlugins() { + private InstalledPlugin[] listInstalledPlugins() { Profiler profiler = Profiler.create(LOG).startInfo("Load plugins index"); GetRequest getRequest = new GetRequest(PLUGINS_WS_URL); InstalledPlugins installedPlugins; try (Reader reader = wsClient.call(getRequest).contentReader()) { installedPlugins = new Gson().fromJson(reader, InstalledPlugins.class); - } catch (IOException e) { - throw new IllegalStateException(e); + } catch (Exception e) { + throw new IllegalStateException("Fail to parse response of " + PLUGINS_WS_URL, e); } profiler.stopInfo(); @@ -128,34 +121,17 @@ public class ScannerPluginInstaller implements PluginInstaller { static class InstalledPlugin { String key; String hash; - String filename; long updatedAt; - @Nullable - String compressedHash; - @Nullable - String compressedFilename; } - private class FileDownloader implements FileCache.Downloader { - private String key; - - FileDownloader(String key) { - this.key = key; - } - - @Override - public void download(String filename, File toFile) throws IOException { - String url = format("/deploy/plugins/%s/%s", key, filename); - if (LOG.isDebugEnabled()) { - LOG.debug("Download plugin '{}' to '{}'", filename, toFile); - } else { - LOG.debug("Download '{}'", filename); - } + private static class Loaded { + private final boolean ok; + @Nullable + private final String notFoundPlugin; - WsResponse response = wsClient.call(new GetRequest(url)); - try (InputStream stream = response.contentStream()) { - FileUtils.copyInputStreamToFile(stream, toFile); - } + private Loaded(boolean ok, @Nullable String notFoundPlugin) { + this.ok = ok; + this.notFoundPlugin = notFoundPlugin; } } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java index c764619858f..89f449bbecb 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java @@ -28,17 +28,16 @@ import org.sonar.api.utils.ZipUtils; import org.sonar.core.platform.ExplodedPlugin; import org.sonar.core.platform.PluginInfo; import org.sonar.core.platform.PluginJarExploder; -import org.sonar.home.cache.FileCache; import static org.sonar.core.util.FileUtils.deleteQuietly; @ScannerSide public class ScannerPluginJarExploder extends PluginJarExploder { - private final FileCache fileCache; + private final PluginFiles pluginFiles; - public ScannerPluginJarExploder(FileCache fileCache) { - this.fileCache = fileCache; + public ScannerPluginJarExploder(PluginFiles pluginFiles) { + this.pluginFiles = pluginFiles; } @Override @@ -62,7 +61,7 @@ public class ScannerPluginJarExploder extends PluginJarExploder { try { // Recheck in case of concurrent processes if (!destDir.exists()) { - File tempDir = fileCache.createTempDir(); + File tempDir = pluginFiles.createTempDir(); ZipUtils.unzip(cachedFile, tempDir, newLibFilter()); FileUtils.moveDirectory(tempDir, destDir); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/WsTestUtil.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/WsTestUtil.java index 811a4511e26..b22ba78b812 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/WsTestUtil.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/WsTestUtil.java @@ -53,10 +53,17 @@ public class WsTestUtil { when(mock.call(any(WsRequest.class))).thenReturn(response); } - public static void mockReader(ScannerWsClient mock, String path, Reader reader) { + public static void mockReader(ScannerWsClient mock, String path, Reader reader, Reader... others) { WsResponse response = mock(WsResponse.class); when(response.contentReader()).thenReturn(reader); - when(mock.call(argThat(new RequestMatcher(path)))).thenReturn(response); + WsResponse[] otherResponses = new WsResponse[others.length]; + for (int i = 0; i < others.length; i++) { + WsResponse otherResponse = mock(WsResponse.class); + when(otherResponse.contentReader()).thenReturn(others[i]); + otherResponses[i] = otherResponse; + } + + when(mock.call(argThat(new RequestMatcher(path)))).thenReturn(response, otherResponses); } public static void mockException(ScannerWsClient mock, Exception e) { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/FileCacheProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/FileCacheProviderTest.java deleted file mode 100644 index bc5b731b84d..00000000000 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/FileCacheProviderTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.scanner.bootstrap; - -import java.io.File; -import java.io.IOException; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.config.internal.MapSettings; -import org.sonar.home.cache.FileCache; - -import static org.assertj.core.api.Assertions.assertThat; - -public class FileCacheProviderTest { - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Test - public void provide() { - FileCacheProvider provider = new FileCacheProvider(); - FileCache cache = provider.provide(new MapSettings().asConfig()); - - assertThat(cache).isNotNull(); - assertThat(cache.getDir()).isNotNull().exists(); - } - - @Test - public void keep_singleton_instance() { - FileCacheProvider provider = new FileCacheProvider(); - MapSettings settings = new MapSettings(); - FileCache cache1 = provider.provide(settings.asConfig()); - FileCache cache2 = provider.provide(settings.asConfig()); - - assertThat(cache1).isSameAs(cache2); - } - - @Test - public void honor_sonarUserHome() throws IOException { - FileCacheProvider provider = new FileCacheProvider(); - MapSettings settings = new MapSettings(); - File f = temp.newFolder(); - settings.appendProperty("sonar.userHome", f.getAbsolutePath()); - FileCache cache = provider.provide(settings.asConfig()); - - assertThat(cache.getDir()).isEqualTo(new File(f, "cache")); - } -} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/PluginFilesTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/PluginFilesTest.java new file mode 100644 index 00000000000..f3b306c7773 --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/PluginFilesTest.java @@ -0,0 +1,363 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.bootstrap; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Collections; +import java.util.Optional; +import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; +import java.util.jar.Pack200; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import javax.annotation.Nullable; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okio.Buffer; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.scanner.bootstrap.ScannerPluginInstaller.InstalledPlugin; +import org.sonarqube.ws.client.HttpConnector; +import org.sonarqube.ws.client.WsClientFactories; + +import static org.apache.commons.io.FileUtils.moveFile; +import static org.assertj.core.api.Assertions.assertThat; + +public class PluginFilesTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + @Rule + public MockWebServer server = new MockWebServer(); + + private File userHome; + private PluginFiles underTest; + + @Before + public void setUp() throws Exception { + HttpConnector connector = HttpConnector.newBuilder().url(server.url("/").toString()).build(); + GlobalAnalysisMode analysisMode = new GlobalAnalysisMode(new GlobalProperties(Collections.emptyMap())); + ScannerWsClient wsClient = new ScannerWsClient(WsClientFactories.getDefault().newClient(connector), false, analysisMode); + + userHome = temp.newFolder(); + MapSettings settings = new MapSettings(); + settings.setProperty("sonar.userHome", userHome.getAbsolutePath()); + + underTest = new PluginFiles(wsClient, settings.asConfig()); + } + + @Test + public void get_jar_from_cache_if_present() throws Exception { + FileAndMd5 jar = createFileInCache("foo"); + + File result = underTest.get(newInstalledPlugin("foo", jar.md5)).get(); + + verifySameContent(result, jar); + // no requests to server + assertThat(server.getRequestCount()).isEqualTo(0); + } + + @Test + public void download_and_add_jar_to_cache_if_missing() throws Exception { + FileAndMd5 tempJar = new FileAndMd5(); + enqueueDownload(tempJar); + + InstalledPlugin plugin = newInstalledPlugin("foo", tempJar.md5); + File result = underTest.get(plugin).get(); + + verifySameContent(result, tempJar); + HttpUrl requestedUrl = server.takeRequest().getRequestUrl(); + assertThat(requestedUrl.encodedPath()).isEqualTo("/api/plugins/download"); + assertThat(requestedUrl.encodedQuery()).isEqualTo("plugin=foo&acceptCompressions=pack200"); + + // get from cache on second call + result = underTest.get(plugin).get(); + verifySameContent(result, tempJar); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @Test + public void download_compressed_and_add_uncompressed_to_cache_if_missing() throws Exception { + FileAndMd5 jar = new FileAndMd5(); + enqueueCompressedDownload(jar, true); + + InstalledPlugin plugin = newInstalledPlugin("foo", jar.md5); + File result = underTest.get(plugin).get(); + + verifySameContentAfterCompression(jar.file, result); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getRequestUrl().queryParameter("acceptCompressions")).isEqualTo("pack200"); + + // get from cache on second call + result = underTest.get(plugin).get(); + verifySameContentAfterCompression(jar.file, result); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @Test + public void return_empty_if_plugin_not_found_on_server() { + server.enqueue(new MockResponse().setResponseCode(404)); + + InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); + Optional result = underTest.get(plugin); + + assertThat(result).isEmpty(); + } + + @Test + public void fail_if_integrity_of_download_is_not_valid() throws IOException { + FileAndMd5 tempJar = new FileAndMd5(); + enqueueDownload(tempJar.file, "invalid_hash"); + InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); + + expectISE("foo", "was expected to have checksum invalid_hash but had " + tempJar.md5); + + underTest.get(plugin); + } + + @Test + public void fail_if_integrity_of_compressed_download_is_not_valid() throws Exception { + FileAndMd5 jar = new FileAndMd5(); + enqueueCompressedDownload(jar, false); + + expectISE("foo", "was expected to have checksum invalid_hash but had "); + InstalledPlugin plugin = newInstalledPlugin("foo", jar.md5); + + underTest.get(plugin).get(); + } + + @Test + public void fail_if_md5_header_is_missing_from_response() throws IOException { + File tempJar = temp.newFile(); + enqueueDownload(tempJar, null); + InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); + + expectISE("foo", "did not return header Sonar-MD5"); + + underTest.get(plugin); + } + + @Test + public void fail_if_compressed_download_cannot_be_uncompressed() { + MockResponse response = new MockResponse().setBody("not binary"); + response.setHeader("Sonar-MD5", DigestUtils.md5Hex("not binary")); + response.setHeader("Sonar-UncompressedMD5", "abc"); + response.setHeader("Sonar-Compression", "pack200"); + server.enqueue(response); + + expectISE("foo", "Pack200 error"); + + InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); + underTest.get(plugin).get(); + } + + @Test + public void fail_if_server_returns_error() { + server.enqueue(new MockResponse().setResponseCode(500)); + InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); + + expectISE("foo", "returned code 500"); + + underTest.get(plugin); + } + + @Test + public void download_a_new_version_of_plugin_during_blue_green_switch() throws IOException { + FileAndMd5 tempJar = new FileAndMd5(); + enqueueDownload(tempJar); + + // expecting to download plugin foo with checksum "abc" + InstalledPlugin pluginV1 = newInstalledPlugin("foo", "abc"); + + File result = underTest.get(pluginV1).get(); + verifySameContent(result, tempJar); + + // new version of downloaded jar is put in cache with the new md5 + InstalledPlugin pluginV2 = newInstalledPlugin("foo", tempJar.md5); + result = underTest.get(pluginV2).get(); + verifySameContent(result, tempJar); + assertThat(server.getRequestCount()).isEqualTo(1); + + // v1 still requests server and downloads v2 + enqueueDownload(tempJar); + result = underTest.get(pluginV1).get(); + verifySameContent(result, tempJar); + assertThat(server.getRequestCount()).isEqualTo(2); + } + + @Test + public void fail_if_cached_file_is_outside_cache_dir() throws IOException { + FileAndMd5 tempJar = new FileAndMd5(); + enqueueDownload(tempJar); + + InstalledPlugin plugin = newInstalledPlugin("foo/bar", "abc"); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Fail to download plugin [foo/bar]. Key is not valid."); + + underTest.get(plugin); + } + + private FileAndMd5 createFileInCache(String pluginKey) throws IOException { + FileAndMd5 tempFile = new FileAndMd5(); + return moveToCache(pluginKey, tempFile); + } + + private FileAndMd5 moveToCache(String pluginKey, FileAndMd5 jar) throws IOException { + File jarInCache = new File(userHome, "cache/" + jar.md5 + "/sonar-" + pluginKey + "-plugin.jar"); + moveFile(jar.file, jarInCache); + return new FileAndMd5(jarInCache, jar.md5); + } + + /** + * Enqueue download of file with valid MD5 + */ + private void enqueueDownload(FileAndMd5 file) throws IOException { + enqueueDownload(file.file, file.md5); + } + + /** + * Enqueue download of file with a MD5 that may not be returned (null) or not valid + */ + private void enqueueDownload(File file, @Nullable String md5) throws IOException { + Buffer body = new Buffer(); + body.write(FileUtils.readFileToByteArray(file)); + MockResponse response = new MockResponse().setBody(body); + if (md5 != null) { + response.setHeader("Sonar-MD5", md5); + } + server.enqueue(response); + } + + /** + * Enqueue download of file with a MD5 that may not be returned (null) or not valid + */ + private void enqueueCompressedDownload(FileAndMd5 jar, boolean validMd5) throws IOException { + Buffer body = new Buffer(); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (JarInputStream in = new JarInputStream(new BufferedInputStream(Files.newInputStream(jar.file.toPath()))); + OutputStream output = new GZIPOutputStream(new BufferedOutputStream(bytes))) { + Pack200.newPacker().pack(in, output); + } + body.write(bytes.toByteArray()); + + MockResponse response = new MockResponse().setBody(body); + response.setHeader("Sonar-MD5", validMd5 ? DigestUtils.md5Hex(bytes.toByteArray()) : "invalid_hash"); + response.setHeader("Sonar-UncompressedMD5", jar.md5); + response.setHeader("Sonar-Compression", "pack200"); + server.enqueue(response); + } + + private static InstalledPlugin newInstalledPlugin(String pluginKey, String fileChecksum) { + InstalledPlugin plugin = new InstalledPlugin(); + plugin.key = pluginKey; + plugin.hash = fileChecksum; + return plugin; + } + + private static void verifySameContent(File file1, FileAndMd5 file2) { + assertThat(file1).isFile().exists(); + assertThat(file2.file).isFile().exists(); + assertThat(file1).hasSameContentAs(file2.file); + } + + /** + * Packing and unpacking a JAR generates a different file. + */ + private void verifySameContentAfterCompression(File file1, File file2) throws IOException { + assertThat(file1).isFile().exists(); + assertThat(file2).isFile().exists(); + assertThat(packAndUnpackJar(file1)).hasSameContentAs(packAndUnpackJar(file2)); + } + + private File packAndUnpackJar(File source) throws IOException { + File packed = temp.newFile(); + try (JarInputStream in = new JarInputStream(new BufferedInputStream(Files.newInputStream(source.toPath()))); + OutputStream out = new GZIPOutputStream(new BufferedOutputStream(Files.newOutputStream(packed.toPath())))) { + Pack200.newPacker().pack(in, out); + } + + File to = temp.newFile(); + try (InputStream input = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(packed.toPath()))); + JarOutputStream output = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(to.toPath())))) { + Pack200.newUnpacker().unpack(input, output); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + return to; + } + + private void expectISE(String pluginKey, String message) { + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(String item) { + return item.startsWith("Fail to download plugin [" + pluginKey + "]") && item.contains(message); + } + + @Override + public void describeTo(Description description) { + } + }); + } + + private class FileAndMd5 { + private final File file; + private final String md5; + + FileAndMd5(File file, String md5) { + this.file = file; + this.md5 = md5; + } + + FileAndMd5() throws IOException { + this.file = temp.newFile(); + FileUtils.write(this.file, RandomStringUtils.random(3)); + try (InputStream fis = FileUtils.openInputStream(this.file)) { + this.md5 = DigestUtils.md5Hex(fis); + } catch (IOException e) { + throw new IllegalStateException("Fail to compute md5 of " + this.file, e); + } + } + + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java index b99ab430412..ec5fe74846c 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest.java @@ -20,89 +20,136 @@ package org.sonar.scanner.bootstrap; import java.io.File; +import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import org.junit.Before; +import java.io.StringReader; +import java.util.Map; +import java.util.Optional; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import org.apache.commons.io.FileUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import org.sonar.home.cache.FileCache; import org.sonar.scanner.WsTestUtil; -import org.sonar.scanner.bootstrap.ScannerPluginInstaller.InstalledPlugin; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ScannerPluginInstallerTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - @Rule - public ExpectedException thrown = ExpectedException.none(); + public ExpectedException expectedException = ExpectedException.none(); - private FileCache fileCache = mock(FileCache.class); - private ScannerWsClient wsClient; private ScannerPluginPredicate pluginPredicate = mock(ScannerPluginPredicate.class); + private PluginFiles pluginFiles = mock(PluginFiles.class); + private ScannerWsClient wsClient = mock(ScannerWsClient.class); + private ScannerPluginInstaller underTest = new ScannerPluginInstaller(pluginFiles, pluginPredicate, wsClient); - @Before - public void setUp() { - wsClient = mock(ScannerWsClient.class); + @Test + public void download_installed_plugins() throws IOException { + WsTestUtil.mockReader(wsClient, "api/plugins/installed", new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json"))); + enqueueDownload("scmgit", "abc"); + enqueueDownload("java", "def"); + when(pluginPredicate.apply(any())).thenReturn(true); + + Map result = underTest.installRemotes(); + + assertThat(result.keySet()).containsExactlyInAnyOrder("scmgit", "java"); + ScannerPlugin gitPlugin = result.get("scmgit"); + assertThat(gitPlugin.getKey()).isEqualTo("scmgit"); + assertThat(gitPlugin.getInfo().getNonNullJarFile()).exists().isFile(); + assertThat(gitPlugin.getUpdatedAt()).isEqualTo(100L); + + ScannerPlugin javaPlugin = result.get("java"); + assertThat(javaPlugin.getKey()).isEqualTo("java"); + assertThat(javaPlugin.getInfo().getNonNullJarFile()).exists().isFile(); + assertThat(javaPlugin.getUpdatedAt()).isEqualTo(200L); } @Test - public void listRemotePlugins() { - WsTestUtil.mockReader(wsClient, "/api/plugins/installed", - new InputStreamReader(this.getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json"), StandardCharsets.UTF_8)); - ScannerPluginInstaller underTest = new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate); + public void filter_blacklisted_plugins() throws IOException { + WsTestUtil.mockReader(wsClient, "api/plugins/installed", new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json"))); + enqueueDownload("scmgit", "abc"); + enqueueDownload("java", "def"); + when(pluginPredicate.apply("scmgit")).thenReturn(true); + when(pluginPredicate.apply("java")).thenReturn(false); + + Map result = underTest.installRemotes(); - InstalledPlugin[] remotePlugins = underTest.listInstalledPlugins(); - assertThat(remotePlugins).extracting("key").containsOnly("scmgit", "java", "scmsvn"); + assertThat(result.keySet()).containsExactlyInAnyOrder("scmgit"); + verify(pluginFiles, times(1)).get(any()); } @Test - public void should_download_plugin() throws Exception { - File pluginJar = temp.newFile(); - when(fileCache.get(eq("checkstyle-plugin.jar"), eq("fakemd5_1"), any(FileCache.Downloader.class))).thenReturn(pluginJar); + public void fail_if_json_of_installed_plugins_is_not_valid() { + WsTestUtil.mockReader(wsClient, "api/plugins/installed", new StringReader("not json")); - ScannerPluginInstaller underTest = new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate); + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Fail to parse response of api/plugins/installed"); - InstalledPlugin remote = new InstalledPlugin(); - remote.key = "checkstyle"; - remote.filename = "checkstyle-plugin.jar"; - remote.hash = "fakemd5_1"; - File file = underTest.download(remote); + underTest.installRemotes(); + } - assertThat(file).isEqualTo(pluginJar); + @Test + public void reload_list_if_plugin_uninstalled_during_blue_green_switch() throws IOException { + WsTestUtil.mockReader(wsClient, "api/plugins/installed", + new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/blue-installed.json")), + new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/green-installed.json"))); + enqueueNotFoundDownload("scmgit", "abc"); + enqueueDownload("java", "def"); + enqueueDownload("cobol", "ghi"); + when(pluginPredicate.apply(any())).thenReturn(true); + + Map result = underTest.installRemotes(); + + assertThat(result.keySet()).containsExactlyInAnyOrder("java", "cobol"); } @Test - public void should_download_compressed_plugin() throws Exception { - File pluginJar = temp.newFile(); - when(fileCache.getCompressed(eq("checkstyle-plugin.pack.gz"), eq("hash"), any(FileCache.Downloader.class))).thenReturn(pluginJar); + public void fail_if_plugin_not_found_two_times() throws IOException { + WsTestUtil.mockReader(wsClient, "api/plugins/installed", + new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/blue-installed.json")), + new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/green-installed.json"))); + enqueueDownload("scmgit", "abc"); + enqueueDownload("cobol", "ghi"); + enqueueNotFoundDownload("java", "def"); + when(pluginPredicate.apply(any())).thenReturn(true); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Fail to download plugin [java]. Not found."); + + underTest.installRemotes(); + } - ScannerPluginInstaller underTest = new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate); + @Test + public void installLocals_always_returns_empty() { + // this method is used only by medium tests + assertThat(underTest.installLocals()).isEmpty(); + } - InstalledPlugin remote = new InstalledPlugin(); - remote.key = "checkstyle"; - remote.filename = "checkstyle-plugin.jar"; - remote.hash = "fakemd5_1"; - remote.compressedFilename = "checkstyle-plugin.pack.gz"; - remote.compressedHash = "hash"; - File file = underTest.download(remote); + private void enqueueDownload(String pluginKey, String pluginHash) throws IOException { + File jar = temp.newFile(); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().putValue("Plugin-Key", pluginKey); + try (JarOutputStream output = new JarOutputStream(FileUtils.openOutputStream(jar), manifest)) { - assertThat(file).isEqualTo(pluginJar); + } + doReturn(Optional.of(jar)).when(pluginFiles).get(argThat(p -> pluginKey.equals(p.key) && pluginHash.equals(p.hash))); } - @Test - public void should_fail_to_get_plugin_index() { - WsTestUtil.mockException(wsClient, "/api/plugins/installed", new IllegalStateException()); - thrown.expect(IllegalStateException.class); - - new ScannerPluginInstaller(wsClient, fileCache, pluginPredicate).installRemotes(); + private void enqueueNotFoundDownload(String pluginKey, String pluginHash) { + doReturn(Optional.empty()).when(pluginFiles).get(argThat(p -> pluginKey.equals(p.key) && pluginHash.equals(p.hash))); } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java index c8a92c3a3ca..e2ad16d2257 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java @@ -28,53 +28,55 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.core.platform.ExplodedPlugin; import org.sonar.core.platform.PluginInfo; -import org.sonar.home.cache.FileCache; -import org.sonar.home.cache.FileCacheBuilder; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ScannerPluginJarExploderTest { @ClassRule public static TemporaryFolder temp = new TemporaryFolder(); - private File userHome; + private File tempDir; private ScannerPluginJarExploder underTest; @Before public void setUp() throws IOException { - userHome = temp.newFolder(); - FileCache fileCache = new FileCacheBuilder(new Slf4jLogger()).setUserHome(userHome).build(); - underTest = new ScannerPluginJarExploder(fileCache); + tempDir = temp.newFolder(); + PluginFiles pluginFiles = mock(PluginFiles.class); + when(pluginFiles.createTempDir()).thenReturn(tempDir); + underTest = new ScannerPluginJarExploder(pluginFiles); } @Test public void copy_and_extract_libs() throws IOException { - File fileFromCache = getFileFromCache("sonar-checkstyle-plugin-2.8.jar"); - ExplodedPlugin exploded = underTest.explode(PluginInfo.create(fileFromCache)); + File jar = loadFile("sonar-checkstyle-plugin-2.8.jar"); + ExplodedPlugin exploded = underTest.explode(PluginInfo.create(jar)); assertThat(exploded.getKey()).isEqualTo("checkstyle"); assertThat(exploded.getMain()).isFile().exists(); assertThat(exploded.getLibs()).extracting(File::getName).containsExactlyInAnyOrder("antlr-2.7.6.jar", "checkstyle-5.1.jar", "commons-cli-1.0.jar"); - assertThat(new File(fileFromCache.getParent(), "sonar-checkstyle-plugin-2.8.jar")).exists(); - assertThat(new File(fileFromCache.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/META-INF/lib/checkstyle-5.1.jar")).exists(); + assertThat(new File(jar.getParent(), "sonar-checkstyle-plugin-2.8.jar")).exists(); + assertThat(new File(jar.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/META-INF/lib/checkstyle-5.1.jar")).exists(); } @Test public void extract_only_libs() throws IOException { - File fileFromCache = getFileFromCache("sonar-checkstyle-plugin-2.8.jar"); - underTest.explode(PluginInfo.create(fileFromCache)); + File jar = loadFile("sonar-checkstyle-plugin-2.8.jar"); - assertThat(new File(fileFromCache.getParent(), "sonar-checkstyle-plugin-2.8.jar")).exists(); - assertThat(new File(fileFromCache.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/META-INF/MANIFEST.MF")).doesNotExist(); - assertThat(new File(fileFromCache.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/org/sonar/plugins/checkstyle/CheckstyleVersion.class")).doesNotExist(); + underTest.explode(PluginInfo.create(jar)); + + assertThat(new File(jar.getParent(), "sonar-checkstyle-plugin-2.8.jar")).exists(); + assertThat(new File(jar.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/META-INF/MANIFEST.MF")).doesNotExist(); + assertThat(new File(jar.getParent(), "sonar-checkstyle-plugin-2.8.jar_unzip/org/sonar/plugins/checkstyle/CheckstyleVersion.class")).doesNotExist(); } - private File getFileFromCache(String filename) throws IOException { - File src = FileUtils.toFile(getClass().getResource(this.getClass().getSimpleName() + "/" + filename)); - File destFile = new File(new File(userHome, "" + filename.hashCode()), filename); - FileUtils.copyFile(src, destFile); - return destFile; + private File loadFile(String filename) throws IOException { + File src = FileUtils.toFile(getClass().getResource(getClass().getSimpleName() + "/" + filename)); + File dest = new File(temp.newFolder(), filename); + FileUtils.copyFile(src, dest); + return dest; } } diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/blue-installed.json b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/blue-installed.json new file mode 100644 index 00000000000..c33470d53c5 --- /dev/null +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/blue-installed.json @@ -0,0 +1,14 @@ +{ + "plugins": [ + { + "key": "scmgit", + "hash": "abc", + "updatedAt": 100 + }, + { + "key": "java", + "hash": "def", + "updatedAt": 200 + } + ] +} diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/green-installed.json b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/green-installed.json new file mode 100644 index 00000000000..1379b68eca3 --- /dev/null +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/green-installed.json @@ -0,0 +1,14 @@ +{ + "plugins": [ + { + "key": "java", + "hash": "def", + "updatedAt": 200 + }, + { + "key": "cobol", + "hash": "ghi", + "updatedAt": 300 + } + ] +} diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json index 2e0c49bf832..c33470d53c5 100644 --- a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json +++ b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/bootstrap/ScannerPluginInstallerTest/installed-plugins-ws.json @@ -2,51 +2,13 @@ "plugins": [ { "key": "scmgit", - "name": "Git", - "description": "Git SCM Provider.", - "version": "1.0", - "license": "GNU LGPL 3", - "organizationName": "SonarSource", - "organizationUrl": "http://www.sonarsource.com", - "homepageUrl": "https://redirect.sonarsource.com/plugins/scmgit.html", - "issueTrackerUrl": "http://jira.sonarsource.com/browse/SONARSCGIT", - "implementationBuild": "9ce9d330c313c296fab051317cc5ad4b26319e07", - "filename": "sonar-scm-git-plugin-1.0.jar", - "hash": "abcdef123456", - "sonarLintSupported": false, - "updatedAt": 123456789 + "hash": "abc", + "updatedAt": 100 }, { "key": "java", - "name": "Java", - "description": "SonarQube rule engine.", - "version": "3.0", - "license": "GNU LGPL 3", - "organizationName": "SonarSource", - "organizationUrl": "http://www.sonarsource.com", - "homepageUrl": "https://redirect.sonarsource.com/plugins/java.html", - "issueTrackerUrl": "http://jira.sonarsource.com/browse/SONARJAVA", - "implementationBuild": "65396a609ddface8b311a6a665aca92a7da694f1", - "filename": "sonar-java-plugin-3.0.jar", - "hash": "abcdef123456", - "sonarLintSupported": true, - "updatedAt": 123456789 - }, - { - "key": "scmsvn", - "name": "SVN", - "description": "SVN SCM Provider.", - "version": "1.0", - "license": "GNU LGPL 3", - "organizationName": "SonarSource", - "organizationUrl": "http://www.sonarsource.com", - "homepageUrl": "https://redirect.sonarsource.com/plugins/scmsvn.html", - "issueTrackerUrl": "http://jira.sonarsource.com/browse/SONARSCSVN", - "implementationBuild": "213fc8a8b582ff530b12dd4a59a6512be1071234", - "filename": "sonar-scm-svn-plugin-1.0.jar", - "hash": "abcdef123456", - "sonarLintSupported": false, - "updatedAt": 123456789 + "hash": "def", + "updatedAt": 200 } ] -} \ No newline at end of file +} diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/BaseResponse.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/BaseResponse.java index df154463a1d..bbb268f1be0 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/BaseResponse.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/BaseResponse.java @@ -42,7 +42,7 @@ abstract class BaseResponse implements WsResponse { public boolean hasContent() { return code() != HTTP_NO_CONTENT; } - + @Override public void close() { // override if needed diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/LocalWsConnector.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/LocalWsConnector.java index 2a9209af4e1..823e364ae03 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/LocalWsConnector.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/LocalWsConnector.java @@ -140,5 +140,13 @@ class LocalWsConnector implements WsConnector { public String content() { return new String(bytes, UTF_8); } + + /** + * Not implemented yet + */ + @Override + public Optional header(String name) { + return Optional.empty(); + } } } diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/MockWsResponse.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/MockWsResponse.java index 7003b8f238c..789d61850d3 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/MockWsResponse.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/MockWsResponse.java @@ -27,6 +27,9 @@ import java.io.Reader; import java.io.StringReader; import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.apache.commons.io.IOUtils; import org.sonarqube.ws.MediaTypes; @@ -34,10 +37,12 @@ import static java.util.Objects.requireNonNull; public class MockWsResponse extends BaseResponse { + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private int code = HttpURLConnection.HTTP_OK; private String requestUrl; private byte[] content; - private String contentType; + private final Map headers = new HashMap<>(); @Override public int code() { @@ -51,12 +56,16 @@ public class MockWsResponse extends BaseResponse { @Override public String contentType() { - requireNonNull(contentType); - return contentType; + return requireNonNull(headers.get(CONTENT_TYPE_HEADER)); + } + + @Override + public Optional header(String name) { + return Optional.ofNullable(headers.get(name)); } public MockWsResponse setContentType(String contentType) { - this.contentType = contentType; + headers.put(CONTENT_TYPE_HEADER, contentType); return this; } diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/OkHttpResponse.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/OkHttpResponse.java index 239d4ebb609..9c0e1ec9968 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/OkHttpResponse.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/OkHttpResponse.java @@ -19,12 +19,12 @@ */ package org.sonarqube.ws.client; -import okhttp3.Response; -import okhttp3.ResponseBody; - import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.util.Optional; +import okhttp3.Response; +import okhttp3.ResponseBody; class OkHttpResponse extends BaseResponse { @@ -49,6 +49,11 @@ class OkHttpResponse extends BaseResponse { return okResponse.header("Content-Type"); } + @Override + public Optional header(String name) { + return Optional.ofNullable(okResponse.header(name)); + } + /** * Get stream of bytes */ diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/WsResponse.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/WsResponse.java index aeb297b399a..8847e040782 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/WsResponse.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/WsResponse.java @@ -22,6 +22,7 @@ package org.sonarqube.ws.client; import java.io.Closeable; import java.io.InputStream; import java.io.Reader; +import java.util.Optional; /** * @since 5.3 @@ -51,6 +52,8 @@ public interface WsResponse extends Closeable { String contentType(); + Optional header(String name); + boolean hasContent(); InputStream contentStream(); diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java b/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java deleted file mode 100644 index f2601bdaa1a..00000000000 --- a/tests/src/test/java/org/sonarqube/tests/plugins/CompressPluginsTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonarqube.tests.plugins; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.sonar.orchestrator.Orchestrator; -import com.sonar.orchestrator.build.SonarScanner; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.sonarqube.qa.util.Tester; -import org.sonarqube.ws.client.GetRequest; -import org.sonarqube.ws.client.WsResponse; - -import static org.assertj.core.api.Assertions.assertThat; -import static util.ItUtils.newOrchestratorBuilder; -import static util.ItUtils.projectDir; -import static util.ItUtils.xooPlugin; - -/** - * Checks the feature of compressing plugins with pack200 and making them available for the scanners. - */ -public class CompressPluginsTest { - @ClassRule - public static Orchestrator orchestrator = newOrchestratorBuilder() - .addPlugin(xooPlugin()) - .setServerProperty("sonar.pluginsCompression.enable", "true") - .build(); - - @Rule - public Tester tester = new Tester(orchestrator); - - @Before - public void setUp() { - orchestrator.resetData(); - } - - @Test - public void dont_fail_analysis() { - SonarScanner scanner = SonarScanner.create(projectDir("shared/xoo-sample")); - orchestrator.executeBuild(scanner); - } - - @Test - public void plugins_installed_ws_should_expose_compressed_plugin() { - WsResponse response = tester.wsClient().wsConnector().call(new GetRequest("api/plugins/installed")); - String content = response.content(); - JsonParser parser = new JsonParser(); - JsonObject root = parser.parse(content).getAsJsonObject(); - JsonArray plugins = root.getAsJsonArray("plugins"); - plugins.forEach(p -> { - assertThat(p.getAsJsonObject().has("compressedHash")).isTrue(); - assertThat(p.getAsJsonObject().has("compressedFilename")).isTrue(); - }); - } -} diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java b/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java deleted file mode 100644 index a35b2b033a0..00000000000 --- a/tests/src/test/java/org/sonarqube/tests/plugins/CompressedPluginsSuite.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2018 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonarqube.tests.plugins; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -@RunWith(Suite.class) -@Suite.SuiteClasses({ - CompressPluginsTest.class, -}) -public class CompressedPluginsSuite { - -} diff --git a/tests/src/test/java/org/sonarqube/tests/plugins/PluginsTest.java b/tests/src/test/java/org/sonarqube/tests/plugins/PluginsTest.java index 2dffe968d4e..137ed11e92d 100644 --- a/tests/src/test/java/org/sonarqube/tests/plugins/PluginsTest.java +++ b/tests/src/test/java/org/sonarqube/tests/plugins/PluginsTest.java @@ -120,6 +120,10 @@ public class PluginsTest { installPlugin(builder, "com.sonarsource.license", "sonar-dev-license-plugin"); builder.activateLicense(); + + // use compression of plugin JARs, just to check that it does not fail + builder.setServerProperty("sonar.pluginsCompression.enable", "true"); + ORCHESTRATOR = builder.build(); ORCHESTRATOR.start(); }