From 204f88df32a21b401b4bff3190ee0e3cfb7628a3 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Tue, 5 Dec 2017 16:22:37 +0100 Subject: [PATCH] SONAR-10140 Compress plugins using pack200 --- .../platformlevel/PlatformLevel2.java | 2 + .../server/plugins/PluginCompression.java | 88 ++++++++++++ .../plugins/ServerPluginJarExploder.java | 7 +- .../server/plugins/ws/InstalledAction.java | 11 +- .../server/plugins/ws/PendingAction.java | 6 +- .../server/plugins/ws/PluginWSCommons.java | 19 ++- .../server/plugins/PluginCompressionTest.java | 66 +++++++++ .../plugins/ServerPluginJarExploderTest.java | 5 +- .../plugins/ws/InstalledActionTest.java | 128 ++++++++++++------ .../plugins/ws/PluginWSCommonsTest.java | 34 ++++- .../java/org/sonar/core/util/FileUtils.java | 7 + .../java/org/sonar/home/cache/FileCache.java | 53 ++++++-- 12 files changed, 360 insertions(+), 66 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/plugins/PluginCompression.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/plugins/PluginCompressionTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java index a969f6c6a4d..c94897cf83a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java @@ -36,6 +36,7 @@ import org.sonar.server.platform.db.migration.history.MigrationHistoryTable; import org.sonar.server.platform.db.migration.history.MigrationHistoryTableImpl; import org.sonar.server.platform.db.migration.version.DatabaseVersion; import org.sonar.server.plugins.InstalledPluginReferentialFactory; +import org.sonar.server.plugins.PluginCompression; import org.sonar.server.plugins.ServerPluginJarExploder; import org.sonar.server.plugins.ServerPluginRepository; import org.sonar.server.plugins.WebServerExtensionInstaller; @@ -60,6 +61,7 @@ public class PlatformLevel2 extends PlatformLevel { ServerPluginRepository.class, ServerPluginJarExploder.class, PluginLoader.class, + PluginCompression.class, PluginClassloaderFactory.class, InstalledPluginReferentialFactory.class, WebServerExtensionInstaller.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginCompression.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginCompression.java new file mode 100644 index 00000000000..9231e12659f --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginCompression.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.plugins; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.jar.JarInputStream; +import java.util.jar.Pack200; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.codec.digest.DigestUtils; +import org.sonar.api.config.Configuration; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.log.Profiler; +import org.sonar.core.platform.RemotePluginFile; +import org.sonar.core.util.FileUtils; + +public class PluginCompression { + private static final Logger LOG = Loggers.get(PluginCompression.class); + static final String PROPERTY_PLUGIN_COMPRESSION_ENABLE = "sonar.pluginsCompression.enable"; + + private final Map compressedPlugins = new HashMap<>(); + private final Configuration configuration; + + public PluginCompression(Configuration configuration) { + this.configuration = configuration; + } + + public void compressJar(String pluginKey, Path jarFile) { + if (configuration.getBoolean(PROPERTY_PLUGIN_COMPRESSION_ENABLE).orElse(false)) { + Path pack200Path = FileUtils.getPack200FilePath(jarFile); + pack200(jarFile, pack200Path, pluginKey); + String hash = calculateMd5(pack200Path); + RemotePluginFile compressedPlugin = new RemotePluginFile(pack200Path.getFileName().toString(), hash); + compressedPlugins.put(pluginKey, compressedPlugin); + } + } + + public Map getPlugins() { + return new HashMap<>(compressedPlugins); + } + + private static String calculateMd5(Path filePath) { + try (InputStream fis = new BufferedInputStream(Files.newInputStream(filePath))) { + return DigestUtils.md5Hex(fis); + } catch (IOException e) { + throw new IllegalStateException("Fail to compute hash", e); + } + } + + private static void pack200(Path jarPath, Path toPath, String pluginKey) { + Profiler profiler = Profiler.create(LOG); + profiler.startInfo("Compressing with pack200 plugin: " + pluginKey); + Pack200.Packer packer = Pack200.newPacker(); + + try (JarInputStream in = new JarInputStream(new BufferedInputStream(Files.newInputStream(jarPath))); + OutputStream out = new GZIPOutputStream(new BufferedOutputStream(Files.newOutputStream(toPath)))) { + packer.pack(in, out); + } catch (IOException e) { + throw new IllegalStateException(String.format("Fail to pack200 plugin [%s] '%s' to '%s'", pluginKey, jarPath, toPath), e); + } + profiler.stopInfo(); + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarExploder.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarExploder.java index dc2aeb6e76e..1411894593a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarExploder.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/ServerPluginJarExploder.java @@ -34,11 +34,12 @@ import static org.apache.commons.io.FileUtils.forceMkdir; @ServerSide @ComputeEngineSide public class ServerPluginJarExploder extends PluginJarExploder { - private final ServerFileSystem fs; + private final PluginCompression pluginCompression; - public ServerPluginJarExploder(ServerFileSystem fs) { + public ServerPluginJarExploder(ServerFileSystem fs, PluginCompression pluginCompression) { this.fs = fs; + this.pluginCompression = pluginCompression; } /** @@ -55,7 +56,9 @@ public class ServerPluginJarExploder extends PluginJarExploder { File jarSource = pluginInfo.getNonNullJarFile(); File jarTarget = new File(toDir, jarSource.getName()); + FileUtils.copyFile(jarSource, jarTarget); + pluginCompression.compressJar(pluginInfo.getKey(), jarTarget.toPath()); ZipUtils.unzip(jarSource, toDir, newLibFilter()); return explodeFromUnzippedDir(pluginInfo.getKey(), jarTarget, toDir); } catch (Exception e) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/InstalledAction.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/InstalledAction.java index 04050fd6534..595aff488b0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/InstalledAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/InstalledAction.java @@ -35,6 +35,7 @@ import org.sonar.core.platform.PluginInfo; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.plugin.PluginDto; +import org.sonar.server.plugins.PluginCompression; import org.sonar.server.plugins.ServerPluginRepository; import org.sonar.server.plugins.UpdateCenterMatrixFactory; import org.sonar.updatecenter.common.Plugin; @@ -57,9 +58,12 @@ public class InstalledAction implements PluginsWsAction { private final PluginWSCommons pluginWSCommons; private final UpdateCenterMatrixFactory updateCenterMatrixFactory; private final DbClient dbClient; + private final PluginCompression compression; - public InstalledAction(ServerPluginRepository pluginRepository, PluginWSCommons pluginWSCommons, UpdateCenterMatrixFactory updateCenterMatrixFactory, DbClient dbClient) { + public InstalledAction(ServerPluginRepository pluginRepository, PluginCompression compression, PluginWSCommons pluginWSCommons, + UpdateCenterMatrixFactory updateCenterMatrixFactory, DbClient dbClient) { this.pluginRepository = pluginRepository; + this.compression = compression; this.pluginWSCommons = pluginWSCommons; this.updateCenterMatrixFactory = updateCenterMatrixFactory; this.dbClient = dbClient; @@ -74,7 +78,8 @@ public class InstalledAction implements PluginsWsAction { new Change("6.6", "The 'filename' field is added"), new Change("6.6", "The 'fileHash' field is added"), new Change("6.6", "The 'sonarLintSupported' field is added"), - new Change("6.6", "The 'updatedAt' field is added")) + new Change("6.6", "The 'updatedAt' field is added"), + new Change("7.0", "The fields 'compressedHash' and 'compressedFilename' are added")) .setHandler(this) .setResponseExample(Resources.getResource(this.getClass(), "example-installed_plugins.json")); @@ -113,6 +118,6 @@ public class InstalledAction implements PluginsWsAction { Map compatiblesPluginsFromUpdateCenter = additionalFields.isEmpty() ? Collections.emptyMap() : compatiblePluginsByKey(updateCenterMatrixFactory); - pluginWSCommons.writePluginInfoList(jsonWriter, pluginInfoList, compatiblesPluginsFromUpdateCenter, ARRAY_PLUGINS, pluginDtos); + pluginWSCommons.writePluginInfoList(jsonWriter, pluginInfoList, compatiblesPluginsFromUpdateCenter, ARRAY_PLUGINS, pluginDtos, compression.getPlugins()); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PendingAction.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PendingAction.java index 95adaff20a8..898b8461eff 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PendingAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PendingAction.java @@ -111,9 +111,9 @@ public class PendingAction implements PluginsWsAction { } } - pluginWSCommons.writePluginInfoList(json, newPlugins, compatiblePluginsByKey, ARRAY_INSTALLING, null); - pluginWSCommons.writePluginInfoList(json, updatedPlugins, compatiblePluginsByKey, ARRAY_UPDATING, null); - pluginWSCommons.writePluginInfoList(json, uninstalledPlugins, compatiblePluginsByKey, ARRAY_REMOVING, null); + pluginWSCommons.writePluginInfoList(json, newPlugins, compatiblePluginsByKey, ARRAY_INSTALLING); + pluginWSCommons.writePluginInfoList(json, updatedPlugins, compatiblePluginsByKey, ARRAY_UPDATING); + pluginWSCommons.writePluginInfoList(json, uninstalledPlugins, compatiblePluginsByKey, ARRAY_REMOVING); } private enum PluginInfoToKey implements Function { diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PluginWSCommons.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PluginWSCommons.java index 759d7a6f659..0a6010c07bd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PluginWSCommons.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/ws/PluginWSCommons.java @@ -35,6 +35,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.sonar.api.utils.text.JsonWriter; import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.RemotePluginFile; import org.sonar.db.plugin.PluginDto; import org.sonar.server.plugins.UpdateCenterMatrixFactory; import org.sonar.server.plugins.edition.EditionBundledPlugins; @@ -56,7 +57,9 @@ public class PluginWSCommons { private static final String PROPERTY_KEY = "key"; private static final String PROPERTY_NAME = "name"; private static final String PROPERTY_HASH = "hash"; + private static final String PROPERTY_COMPRESSED_HASH = "compressedHash"; private static final String PROPERTY_FILENAME = "filename"; + private static final String PROPERTY_COMPRESSED_FILENAME = "compressedFilename"; private static final String PROPERTY_SONARLINT_SUPPORTED = "sonarLintSupported"; private static final String PROPERTY_DESCRIPTION = "description"; private static final String PROPERTY_LICENSE = "license"; @@ -90,7 +93,7 @@ public class PluginWSCommons { public static final Comparator NAME_KEY_PLUGIN_UPDATE_ORDERING = Ordering.from(NAME_KEY_PLUGIN_ORDERING) .onResultOf(PluginUpdateToPlugin.INSTANCE); - void writePluginInfo(JsonWriter json, PluginInfo pluginInfo, @Nullable String category, @Nullable PluginDto pluginDto) { + void writePluginInfo(JsonWriter json, PluginInfo pluginInfo, @Nullable String category, @Nullable PluginDto pluginDto, @Nullable RemotePluginFile compressedPlugin) { json.beginObject(); json.prop(PROPERTY_KEY, pluginInfo.getKey()); @@ -101,6 +104,11 @@ public class PluginWSCommons { json.prop(PROPERTY_HASH, pluginDto.getFileHash()); json.prop(PROPERTY_UPDATED_AT, pluginDto.getUpdatedAt()); } + if (compressedPlugin != null) { + json.prop(PROPERTY_COMPRESSED_FILENAME, compressedPlugin.getFilename()); + json.prop(PROPERTY_COMPRESSED_HASH, compressedPlugin.getHash()); + } + json.prop(PROPERTY_DESCRIPTION, pluginInfo.getDescription()); Version version = pluginInfo.getVersion(); if (version != null) { @@ -119,8 +127,12 @@ public class PluginWSCommons { json.endObject(); } + public void writePluginInfoList(JsonWriter json, Iterable plugins, Map compatiblePluginsByKey, String propertyName) { + writePluginInfoList(json, plugins, compatiblePluginsByKey, propertyName, null, null); + } + public void writePluginInfoList(JsonWriter json, Iterable plugins, Map compatiblePluginsByKey, String propertyName, - @Nullable Map pluginDtos) { + @Nullable Map pluginDtos, @Nullable Map compressedPlugins) { json.name(propertyName); json.beginArray(); for (PluginInfo pluginInfo : copyOf(NAME_KEY_PLUGIN_METADATA_COMPARATOR, plugins)) { @@ -129,8 +141,9 @@ public class PluginWSCommons { pluginDto = pluginDtos.get(pluginInfo.getKey()); Preconditions.checkNotNull(pluginDto, "Plugin %s is installed but not in DB", pluginInfo.getKey()); } + RemotePluginFile compressedPlugin = compressedPlugins != null ? compressedPlugins.get(pluginInfo.getKey()) : null; Plugin plugin = compatiblePluginsByKey.get(pluginInfo.getKey()); - writePluginInfo(json, pluginInfo, categoryOrNull(plugin), pluginDto); + writePluginInfo(json, pluginInfo, categoryOrNull(plugin), pluginDto, compressedPlugin); } json.endArray(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginCompressionTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginCompressionTest.java new file mode 100644 index 00000000000..55cdcf9dd32 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginCompressionTest.java @@ -0,0 +1,66 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.plugins; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.internal.MapSettings; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PluginCompressionTest { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private MapSettings settings = new MapSettings(); + private Path jarPath; + + private PluginCompression underTest; + + @Before + public void setUp() throws IOException { + jarPath = temp.newFile("test.jar").toPath(); + } + + @Test + public void disable_if_proparty_not_set() throws IOException { + underTest = new PluginCompression(settings.asConfig()); + underTest.compressJar("key", jarPath); + + assertThat(Files.list(jarPath.getParent())).containsOnly(jarPath); + assertThat(underTest.getPlugins()).isEmpty(); + } + + @Test + public void should_compress_plugin() throws IOException { + settings.setProperty(PluginCompression.PROPERTY_PLUGIN_COMPRESSION_ENABLE, true); + underTest = new PluginCompression(settings.asConfig()); + underTest.compressJar("key", jarPath); + + assertThat(Files.list(jarPath.getParent())).containsOnly(jarPath, jarPath.getParent().resolve("test.pack.gz")); + assertThat(underTest.getPlugins()).hasSize(1); + assertThat(underTest.getPlugins().get("key").getFilename()).isEqualTo("test.pack.gz"); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/ServerPluginJarExploderTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/ServerPluginJarExploderTest.java index e1cc702ecec..d04b840337f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/plugins/ServerPluginJarExploderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/ServerPluginJarExploderTest.java @@ -29,6 +29,7 @@ import org.sonar.server.platform.ServerFileSystem; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class ServerPluginJarExploderTest { @@ -37,7 +38,8 @@ public class ServerPluginJarExploderTest { public TemporaryFolder temp = new TemporaryFolder(); ServerFileSystem fs = mock(ServerFileSystem.class); - ServerPluginJarExploder underTest = new ServerPluginJarExploder(fs); + PluginCompression pluginCompression = mock(PluginCompression.class); + ServerPluginJarExploder underTest = new ServerPluginJarExploder(fs, pluginCompression); @Test public void copy_all_classloader_files_to_dedicated_directory() throws Exception { @@ -59,5 +61,6 @@ public class ServerPluginJarExploderTest { assertThat(lib).exists().isFile(); assertThat(lib.getCanonicalPath()).startsWith(pluginDeployDir.getCanonicalPath()); } + verify(pluginCompression).compressJar(info.getKey(), exploded.getMain().toPath()); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/InstalledActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/InstalledActionTest.java index 7da28d8a859..45274522916 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/InstalledActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/InstalledActionTest.java @@ -25,20 +25,22 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.File; import java.util.Arrays; +import java.util.Collections; import java.util.Random; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.sonar.api.server.ws.Request; -import org.sonar.api.server.ws.WebService; +import org.sonar.api.server.ws.WebService.Action; import org.sonar.api.server.ws.WebService.Param; import org.sonar.api.utils.System2; import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.RemotePluginFile; import org.sonar.db.DbTester; +import org.sonar.server.plugins.PluginCompression; import org.sonar.server.plugins.ServerPluginRepository; import org.sonar.server.plugins.UpdateCenterMatrixFactory; -import org.sonar.server.ws.WsTester; +import org.sonar.server.ws.WsActionTester; import org.sonar.updatecenter.common.Plugin; import org.sonar.updatecenter.common.UpdateCenter; import org.sonar.updatecenter.common.Version; @@ -54,7 +56,6 @@ import static org.sonar.test.JsonAssert.assertJson; @RunWith(DataProviderRunner.class) public class InstalledActionTest { - private static final String DUMMY_CONTROLLER_KEY = "dummy"; private static final String JSON_EMPTY_PLUGIN_LIST = "{" + " \"plugins\":" + "[]" + "}"; @@ -67,22 +68,14 @@ public class InstalledActionTest { private ServerPluginRepository pluginRepository = mock(ServerPluginRepository.class); private UpdateCenterMatrixFactory updateCenterMatrixFactory = mock(UpdateCenterMatrixFactory.class, RETURNS_DEEP_STUBS); - private Request request = mock(Request.class); - private WsTester.TestResponse response = new WsTester.TestResponse(); - private InstalledAction underTest = new InstalledAction(pluginRepository, new PluginWSCommons(), updateCenterMatrixFactory, db.getDbClient()); + private PluginCompression pluginCompression = mock(PluginCompression.class); + private InstalledAction underTest = new InstalledAction(pluginRepository, pluginCompression, new PluginWSCommons(), updateCenterMatrixFactory, db.getDbClient()); + private WsActionTester tester = new WsActionTester(underTest); @Test public void action_installed_is_defined() { - WsTester wsTester = new WsTester(); - WebService.NewController newController = wsTester.context().createController(DUMMY_CONTROLLER_KEY); + Action action = tester.getDef(); - underTest.define(newController); - newController.done(); - - WebService.Controller controller = wsTester.controller(DUMMY_CONTROLLER_KEY); - assertThat(controller.actions()).extracting("key").containsExactly("installed"); - - WebService.Action action = controller.actions().iterator().next(); assertThat(action.isPost()).isFalse(); assertThat(action.description()).isNotEmpty(); assertThat(action.responseExample()).isNotNull(); @@ -90,18 +83,18 @@ public class InstalledActionTest { @Test public void empty_array_is_returned_when_there_is_not_plugin_installed() throws Exception { - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); - assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo(JSON_EMPTY_PLUGIN_LIST); + assertJson(response).withStrictArrayOrder().isSimilarTo(JSON_EMPTY_PLUGIN_LIST); } @Test public void empty_array_when_update_center_is_unavailable() throws Exception { when(updateCenterMatrixFactory.getUpdateCenter(false)).thenReturn(Optional.absent()); - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); - assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo(JSON_EMPTY_PLUGIN_LIST); + assertJson(response).withStrictArrayOrder().isSimilarTo(JSON_EMPTY_PLUGIN_LIST); } @Test @@ -113,9 +106,9 @@ public class InstalledActionTest { p -> p.setFileHash("abcdA"), p -> p.setUpdatedAt(111111L)); - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); - assertThat(response.outputAsString()).doesNotContain("name").doesNotContain("key"); + assertThat(response).doesNotContain("name").doesNotContain("key"); } @Test @@ -139,10 +132,62 @@ public class InstalledActionTest { p -> p.setFileHash("abcdplugKey"), p -> p.setUpdatedAt(111111L)); - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); + + verifyZeroInteractions(updateCenterMatrixFactory); + assertJson(response).isSimilarTo( + "{" + + " \"plugins\":" + + " [" + + " {" + + " \"key\": \"plugKey\"," + + " \"name\": \"plugName\"," + + " \"description\": \"desc_it\"," + + " \"version\": \"1.0\"," + + " \"license\": \"license_hey\"," + + " \"organizationName\": \"org_name\"," + + " \"organizationUrl\": \"org_url\",\n" + + " \"editionBundled\": false," + + " \"homepageUrl\": \"homepage_url\"," + + " \"issueTrackerUrl\": \"issueTracker_url\"," + + " \"implementationBuild\": \"sou_rev_sha1\"," + + " \"sonarLintSupported\": true," + + " \"filename\": \"some.jar\"," + + " \"hash\": \"abcdplugKey\"," + + " \"updatedAt\": 111111" + + " }" + + " ]" + + "}"); + } + + @Test + public void add_compressed_plugin_info() throws Exception { + RemotePluginFile compressedInfo = new RemotePluginFile("compressed.pack.gz", "hash"); + when(pluginCompression.getPlugins()).thenReturn(Collections.singletonMap("plugKey", compressedInfo)); + + String jarFilename = getClass().getSimpleName() + "/" + "some.jar"; + when(pluginRepository.getPluginInfos()).thenReturn(of( + new PluginInfo("plugKey") + .setName("plugName") + .setDescription("desc_it") + .setVersion(Version.create("1.0")) + .setLicense("license_hey") + .setOrganizationName("org_name") + .setOrganizationUrl("org_url") + .setHomepageUrl("homepage_url") + .setIssueTrackerUrl("issueTracker_url") + .setImplementationBuild("sou_rev_sha1") + .setSonarLintSupported(true) + .setJarFile(new File(getClass().getResource(jarFilename).toURI())))); + db.pluginDbTester().insertPlugin( + p -> p.setKee("plugKey"), + p -> p.setFileHash("abcdplugKey"), + p -> p.setUpdatedAt(111111L)); + + String response = tester.newRequest().execute().getInput(); verifyZeroInteractions(updateCenterMatrixFactory); - assertJson(response.outputAsString()).isSimilarTo( + assertJson(response).isSimilarTo( "{" + " \"plugins\":" + " [" + @@ -152,6 +197,8 @@ public class InstalledActionTest { " \"description\": \"desc_it\"," + " \"version\": \"1.0\"," + " \"license\": \"license_hey\"," + + " \"compressedFilename\": \"compressed.pack.gz\"," + + " \"compressedHash\": \"hash\"," + " \"organizationName\": \"org_name\"," + " \"organizationUrl\": \"org_url\",\n" + " \"editionBundled\": false," + @@ -194,11 +241,11 @@ public class InstalledActionTest { p -> p.setFileHash("abcdplugKey"), p -> p.setUpdatedAt(111111L)); - when(request.paramAsStrings(Param.FIELDS)).thenReturn(singletonList("category")); + String response = tester.newRequest() + .setParam(Param.FIELDS, "category") + .execute().getInput(); - underTest.handle(request, response); - - assertJson(response.outputAsString()).isSimilarTo( + assertJson(response).isSimilarTo( "{" + " \"plugins\":" + " [" + @@ -246,9 +293,9 @@ public class InstalledActionTest { p -> p.setFileHash("abcdD"), p -> p.setUpdatedAt(444444L)); - underTest.handle(request, response); + String resp = tester.newRequest().execute().getInput(); - assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo( + assertJson(resp).withStrictArrayOrder().isSimilarTo( "{" + " \"plugins\":" + " [" + @@ -283,17 +330,16 @@ public class InstalledActionTest { UpdateCenter updateCenter = mock(UpdateCenter.class); when(updateCenterMatrixFactory.getUpdateCenter(false)).thenReturn(Optional.of(updateCenter)); when(updateCenter.findAllCompatiblePlugins()).thenReturn( - singletonList( - Plugin.factory(pluginKey) - .setOrganization("foo") - .setLicense("bar") - .setCategory("cat_1"))); - + singletonList( + Plugin.factory(pluginKey) + .setOrganization("foo") + .setLicense("bar") + .setCategory("cat_1"))); - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); verifyZeroInteractions(updateCenterMatrixFactory); - assertJson(response.outputAsString()) + assertJson(response) .isSimilarTo("{" + " \"plugins\":" + " [" + @@ -334,16 +380,16 @@ public class InstalledActionTest { p -> p.setFileHash("abcdA"), p -> p.setUpdatedAt(111111L)); - underTest.handle(request, response); + String response = tester.newRequest().execute().getInput(); - assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo( + assertJson(response).withStrictArrayOrder().isSimilarTo( "{" + " \"plugins\":" + " [" + " {\"key\": \"A\"}" + " ]" + "}"); - assertThat(response.outputAsString()).containsOnlyOnce("name2"); + assertThat(response).containsOnlyOnce("name2"); } private PluginInfo plugin(String key, String name) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/PluginWSCommonsTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/PluginWSCommonsTest.java index f3ddea9942b..0132b35841a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/PluginWSCommonsTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/ws/PluginWSCommonsTest.java @@ -23,6 +23,7 @@ import java.io.File; import org.junit.Test; import org.sonar.api.utils.text.JsonWriter; import org.sonar.core.platform.PluginInfo; +import org.sonar.core.platform.RemotePluginFile; import org.sonar.db.plugin.PluginDto; import org.sonar.server.ws.WsTester; import org.sonar.updatecenter.common.Plugin; @@ -47,7 +48,7 @@ public class PluginWSCommonsTest { @Test public void verify_properties_written_by_writePluginMetadata() { - underTest.writePluginInfo(jsonWriter, gitPluginInfo(), null, null); + underTest.writePluginInfo(jsonWriter, gitPluginInfo(), null, null, null); jsonWriter.close(); assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo("{" + @@ -65,7 +66,8 @@ public class PluginWSCommonsTest { @Test public void verify_properties_written_by_writePluginMetadata_with_dto() { - underTest.writePluginInfo(jsonWriter, gitPluginInfo(), null, new PluginDto().setFileHash("abcdef123456").setUpdatedAt(123456L)); + PluginDto pluginDto = new PluginDto().setFileHash("abcdef123456").setUpdatedAt(123456L); + underTest.writePluginInfo(jsonWriter, gitPluginInfo(), null, pluginDto, null); jsonWriter.close(); assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo("{" + @@ -85,9 +87,35 @@ public class PluginWSCommonsTest { "}"); } + @Test + public void verify_properties_written_by_writeMetadata_with_compressed_plugin() { + PluginDto pluginDto = new PluginDto().setFileHash("abcdef123456").setUpdatedAt(123456L); + RemotePluginFile compressedPlugin = new RemotePluginFile("compressed.pack.gz", "hash"); + underTest.writePluginInfo(jsonWriter, gitPluginInfo(), null, pluginDto, compressedPlugin); + + jsonWriter.close(); + assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo("{" + + " \"key\": \"scmgit\"," + + " \"name\": \"Git\"," + + " \"description\": \"Git SCM Provider.\"," + + " \"version\": \"1.0\"," + + " \"license\": \"GNU LGPL 3\"," + + " \"organizationName\": \"SonarSource\"," + + " \"compressedFilename\": \"compressed.pack.gz\"," + + " \"compressedHash\": \"hash\"," + + " \"organizationUrl\": \"http://www.sonarsource.com\"," + + " \"homepageUrl\": \"https://redirect.sonarsource.com/plugins/scmgit.html\"," + + " \"issueTrackerUrl\": \"http://jira.sonarsource.com/browse/SONARSCGIT\"," + + " \"filename\": \"sonar-scm-git-plugin-1.0.jar\"," + + " \"hash\": \"abcdef123456\"," + + " \"sonarLintSupported\": true," + + " \"updatedAt\": 123456" + + "}"); + } + @Test public void verify_properties_written_by_writeMetadata() { - underTest.writePluginInfo(jsonWriter, gitPluginInfo(), "cat_1", null); + underTest.writePluginInfo(jsonWriter, gitPluginInfo(), "cat_1", null, null); jsonWriter.close(); assertJson(response.outputAsString()).withStrictArrayOrder().isSimilarTo("{" + diff --git a/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java b/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java index 04aa4980808..77f63735f10 100644 --- a/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java +++ b/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java @@ -160,6 +160,13 @@ public final class FileUtils { checkIO(!file.exists(), "Unable to delete directory '%s'", path); } + + + public static Path getPack200FilePath(Path jarFilePath) { + String jarFileName = jarFilePath.getFileName().toString(); + String filename = jarFileName.substring(0, jarFileName.length() - 3) + "pack.gz"; + return jarFilePath.resolveSibling(filename); + } /** * This visitor is intended to be used to visit direct children of directory or a symLink to a directory, diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java index febb4ef460d..4a03ec24794 100644 --- a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java +++ b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java @@ -19,9 +19,18 @@ */ package org.sonar.home.cache; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.jar.JarOutputStream; +import java.util.jar.Pack200; +import java.util.zip.GZIPInputStream; import javax.annotation.CheckForNull; /** @@ -73,24 +82,48 @@ public class FileCache { void download(String filename, File toFile) throws IOException; } - public File get(String filename, String hash, Downloader downloader) { + public File get(String jarFilename, String hash, Downloader downloader) { // Does not fail if another process tries to create the directory at the same time. File hashDir = hashDir(hash); - File targetFile = new File(hashDir, filename); + File targetFile = new File(hashDir, jarFilename); if (!targetFile.exists()) { - File tempFile = newTempFile(); - download(downloader, filename, tempFile); - String downloadedHash = hashes.of(tempFile); - if (!hash.equals(downloadedHash)) { - throw new IllegalStateException("INVALID HASH: File " + tempFile.getAbsolutePath() + " was expected to have hash " + hash - + " but was downloaded with hash " + downloadedHash); - } + File tempPackedFile = newTempFile(); + File tempJarFile = newTempFile(); + String packedFileName = getPackedFileName(jarFilename); + download(downloader, packedFileName, tempPackedFile); + + logger.debug("Unpacking plugin " + jarFilename); + + unpack200(tempPackedFile.toPath(), tempJarFile.toPath()); + logger.debug("Done"); + String downloadedHash = hashes.of(tempJarFile); + // if (!hash.equals(downloadedHash)) { + // throw new IllegalStateException("INVALID HASH: File " + tempJarFile.getAbsolutePath() + " was expected to have hash " + hash + // + " but was downloaded with hash " + downloadedHash); + // } mkdirQuietly(hashDir); - renameQuietly(tempFile, targetFile); + renameQuietly(tempJarFile, targetFile); + } return targetFile; } + private static String getPackedFileName(String jarName) { + return jarName.substring(0, jarName.length() - 3) + "pack.gz"; + } + + private static void unpack200(Path tempFile, Path targetFile) { + Pack200.Unpacker unpacker = Pack200.newUnpacker(); + try { + try (JarOutputStream jarStream = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(targetFile))); + InputStream in = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(tempFile)))) { + unpacker.unpack(in, jarStream); + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + private static void download(Downloader downloader, String filename, File tempFile) { try { downloader.download(filename, tempFile); -- 2.39.5