From 3ec1e2d419f88b5449e782589cab65442d76885f Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Mon, 16 Oct 2017 17:36:02 +0200 Subject: [PATCH] SONAR-9946 download in temp dir to support download thread killed --- .../plugins/AbstractPluginDownloader.java | 151 ------------------ .../server/plugins/PluginDownloader.java | 121 +++++++++++++- .../edition/EditionPluginDownloader.java | 66 +++++++- .../server/plugins/PluginDownloaderTest.java | 14 +- .../edition/EditionPluginDownloaderTest.java | 51 ++++-- 5 files changed, 225 insertions(+), 178 deletions(-) delete mode 100644 server/sonar-server/src/main/java/org/sonar/server/plugins/AbstractPluginDownloader.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/AbstractPluginDownloader.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/AbstractPluginDownloader.java deleted file mode 100644 index e29cd75f341..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/AbstractPluginDownloader.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2017 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.plugins; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import org.apache.commons.io.FileUtils; -import org.picocontainer.Startable; -import org.sonar.api.utils.HttpDownloader; -import org.sonar.api.utils.log.Logger; -import org.sonar.api.utils.log.Loggers; -import org.sonar.core.platform.PluginInfo; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.updatecenter.common.Release; - -import static org.apache.commons.io.FileUtils.copyFile; -import static org.apache.commons.io.FileUtils.copyFileToDirectory; -import static org.apache.commons.io.FileUtils.forceMkdir; -import static org.apache.commons.io.FileUtils.toFile; -import static org.apache.commons.lang.StringUtils.substringAfterLast; -import static org.sonar.core.util.FileUtils.deleteQuietly; - -public class AbstractPluginDownloader implements Startable { - private static final Logger LOG = Loggers.get(AbstractPluginDownloader.class); - - private static final String TMP_SUFFIX = "tmp"; - private static final String PLUGIN_EXTENSION = "jar"; - - private final HttpDownloader downloader; - private final File downloadDir; - - protected AbstractPluginDownloader(File downloadDir, HttpDownloader downloader) { - this.downloadDir = downloadDir; - this.downloader = downloader; - } - - /** - * Deletes the temporary files remaining from previous downloads - */ - @Override - public void start() { - try { - forceMkdir(downloadDir); - cleanTempFiles(); - } catch (IOException e) { - throw new IllegalStateException("Fail to create the directory: " + downloadDir, e); - } - } - - @Override - public void stop() { - // Nothing to do - } - - public void cancelDownloads() { - try { - if (downloadDir.exists()) { - org.sonar.core.util.FileUtils.cleanDirectory(downloadDir); - } - } catch (IOException e) { - throw new IllegalStateException("Fail to clean the plugin downloads directory: " + downloadDir, e); - } - } - - boolean hasDownloads() { - return !getDownloadedPluginFilenames().isEmpty(); - } - - List getDownloadedPluginFilenames() { - List names = new ArrayList<>(); - for (File file : listPlugins(this.downloadDir)) { - names.add(file.getName()); - } - return names; - } - - /** - * @return the list of download plugins as {@link PluginInfo} instances - */ - public Collection getDownloadedPlugins() { - return listPlugins(this.downloadDir) - .stream() - .map(PluginInfo::create) - .collect(MoreCollectors.toList()); - } - - protected void download(Release release) { - try { - downloadRelease(release); - } catch (Exception e) { - String message = String.format("Fail to download the plugin (%s, version %s) from %s (error is : %s)", - release.getArtifact().getKey(), release.getVersion().getName(), release.getDownloadUrl(), e.getMessage()); - LOG.debug(message, e); - throw new IllegalStateException(message, e); - } - } - - private void downloadRelease(Release release) throws URISyntaxException, IOException { - String url = release.getDownloadUrl(); - - URI uri = new URI(url); - if (url.startsWith("file:")) { - // used for tests - File file = toFile(uri.toURL()); - copyFileToDirectory(file, downloadDir); - } else { - String filename = substringAfterLast(uri.getPath(), "/"); - if (!filename.endsWith("." + PLUGIN_EXTENSION)) { - filename = release.getKey() + "-" + release.getVersion() + "." + PLUGIN_EXTENSION; - } - File targetFile = new File(downloadDir, filename); - File tempFile = new File(downloadDir, filename + "." + TMP_SUFFIX); - downloader.download(uri, tempFile); - copyFile(tempFile, targetFile); - deleteQuietly(tempFile); - } - } - - protected void cleanTempFiles() { - Collection tempFiles = FileUtils.listFiles(downloadDir, new String[] {TMP_SUFFIX}, false); - for (File tempFile : tempFiles) { - deleteQuietly(tempFile); - } - } - - private static Collection listPlugins(File dir) { - return FileUtils.listFiles(dir, new String[] {PLUGIN_EXTENSION}, false); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginDownloader.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginDownloader.java index a2e087ae19d..a55b4e8d042 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginDownloader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/PluginDownloader.java @@ -20,25 +20,100 @@ package org.sonar.server.plugins; import com.google.common.base.Optional; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.picocontainer.Startable; import org.sonar.api.utils.HttpDownloader; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.core.platform.PluginInfo; +import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.platform.ServerFileSystem; import org.sonar.updatecenter.common.Release; import org.sonar.updatecenter.common.UpdateCenter; import org.sonar.updatecenter.common.Version; +import static org.apache.commons.io.FileUtils.copyFile; +import static org.apache.commons.io.FileUtils.copyFileToDirectory; +import static org.apache.commons.io.FileUtils.forceMkdir; +import static org.apache.commons.io.FileUtils.toFile; +import static org.apache.commons.lang.StringUtils.substringAfterLast; +import static org.sonar.core.util.FileUtils.deleteQuietly; import static org.sonar.server.ws.WsUtils.checkRequest; /** * Downloads plugins from update center. Files are copied in the directory extensions/downloads and then * moved to extensions/plugins after server restart. */ -public class PluginDownloader extends AbstractPluginDownloader { +public class PluginDownloader implements Startable { + + private static final Logger LOG = Loggers.get(PluginDownloader.class); + private static final String TMP_SUFFIX = "tmp"; + private static final String PLUGIN_EXTENSION = "jar"; + private final UpdateCenterMatrixFactory updateCenterMatrixFactory; + private final HttpDownloader downloader; + private final File downloadDir; - public PluginDownloader(UpdateCenterMatrixFactory updateCenterMatrixFactory, HttpDownloader downloader, ServerFileSystem fileSystem) { - super(fileSystem.getDownloadedPluginsDir(), downloader); + public PluginDownloader(UpdateCenterMatrixFactory updateCenterMatrixFactory, HttpDownloader downloader, + ServerFileSystem fileSystem) { this.updateCenterMatrixFactory = updateCenterMatrixFactory; + this.downloader = downloader; + this.downloadDir = fileSystem.getDownloadedPluginsDir(); + } + + /** + * Deletes the temporary files remaining from previous downloads + */ + @Override + public void start() { + try { + forceMkdir(downloadDir); + for (File tempFile : listTempFile(this.downloadDir)) { + deleteQuietly(tempFile); + } + } catch (IOException e) { + throw new IllegalStateException("Fail to create the directory: " + downloadDir, e); + } + } + + @Override + public void stop() { + // Nothing to do + } + + public void cancelDownloads() { + try { + if (downloadDir.exists()) { + org.sonar.core.util.FileUtils.cleanDirectory(downloadDir); + } + } catch (IOException e) { + throw new IllegalStateException("Fail to clean the plugin downloads directory: " + downloadDir, e); + } + } + + public List getDownloadedPluginFilenames() { + List names = new ArrayList<>(); + for (File file : listPlugins(this.downloadDir)) { + names.add(file.getName()); + } + return names; + } + + /** + * @return the list of download plugins as {@link PluginInfo} instances + */ + public Collection getDownloadedPlugins() { + return listPlugins(this.downloadDir) + .stream() + .map(PluginInfo::create) + .collect(MoreCollectors.toList()); } public void download(String pluginKey, Version version) { @@ -46,9 +121,45 @@ public class PluginDownloader extends AbstractPluginDownloader { if (updateCenter.isPresent()) { List installablePlugins = updateCenter.get().findInstallablePlugins(pluginKey, version); checkRequest(!installablePlugins.isEmpty(), "Error while downloading plugin '%s' with version '%s'. No compatible plugin found.", pluginKey, version.getName()); - for (Release r : installablePlugins) { - super.download(r); + for (Release release : installablePlugins) { + try { + downloadRelease(release); + } catch (Exception e) { + String message = String.format("Fail to download the plugin (%s, version %s) from %s (error is : %s)", + release.getArtifact().getKey(), release.getVersion().getName(), release.getDownloadUrl(), e.getMessage()); + LOG.debug(message, e); + throw new IllegalStateException(message, e); + } } } } + + private void downloadRelease(Release release) throws URISyntaxException, IOException { + String url = release.getDownloadUrl(); + + URI uri = new URI(url); + if (url.startsWith("file:")) { + // used for tests + File file = toFile(uri.toURL()); + copyFileToDirectory(file, downloadDir); + } else { + String filename = substringAfterLast(uri.getPath(), "/"); + if (!filename.endsWith("." + PLUGIN_EXTENSION)) { + filename = release.getKey() + "-" + release.getVersion() + "." + PLUGIN_EXTENSION; + } + File targetFile = new File(downloadDir, filename); + File tempFile = new File(downloadDir, filename + "." + TMP_SUFFIX); + downloader.download(uri, tempFile); + copyFile(tempFile, targetFile); + deleteQuietly(tempFile); + } + } + + private static Collection listTempFile(File dir) { + return FileUtils.listFiles(dir, new String[] {TMP_SUFFIX}, false); + } + + private static Collection listPlugins(File dir) { + return FileUtils.listFiles(dir, new String[] {PLUGIN_EXTENSION}, false); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/plugins/edition/EditionPluginDownloader.java b/server/sonar-server/src/main/java/org/sonar/server/plugins/edition/EditionPluginDownloader.java index 6ff88874acf..db9943f0050 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/plugins/edition/EditionPluginDownloader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/plugins/edition/EditionPluginDownloader.java @@ -20,22 +20,41 @@ package org.sonar.server.plugins.edition; import com.google.common.base.Optional; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashSet; import java.util.Set; import org.sonar.api.utils.HttpDownloader; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.core.util.FileUtils; import org.sonar.server.platform.ServerFileSystem; -import org.sonar.server.plugins.AbstractPluginDownloader; import org.sonar.server.plugins.UpdateCenterMatrixFactory; import org.sonar.updatecenter.common.Release; import org.sonar.updatecenter.common.UpdateCenter; import org.sonar.updatecenter.common.Version; -public class EditionPluginDownloader extends AbstractPluginDownloader { +import static org.apache.commons.io.FileUtils.toFile; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + +public class EditionPluginDownloader { + private static final Logger LOG = Loggers.get(EditionPluginDownloader.class); + private static final String PLUGIN_EXTENSION = "jar"; + private final UpdateCenterMatrixFactory updateCenterMatrixFactory; + private final Path tmpDir; + private final Path downloadDir; + private final HttpDownloader downloader; public EditionPluginDownloader(UpdateCenterMatrixFactory updateCenterMatrixFactory, HttpDownloader downloader, ServerFileSystem fileSystem) { - super(fileSystem.getEditionDownloadedPluginsDir(), downloader); + this.downloadDir = fileSystem.getEditionDownloadedPluginsDir().toPath(); this.updateCenterMatrixFactory = updateCenterMatrixFactory; + this.downloader = downloader; + this.tmpDir = downloadDir.resolveSibling(downloadDir.getFileName() + "_tmp"); } public void installEdition(Set pluginKeys) { @@ -47,13 +66,48 @@ public class EditionPluginDownloader extends AbstractPluginDownloader { pluginsToInstall.addAll(updateCenter.get().findInstallablePlugins(pluginKey, Version.create(""))); } + FileUtils.deleteQuietly(tmpDir); + Files.createDirectories(tmpDir); + for (Release r : pluginsToInstall) { - super.download(r); + download(r); } + + FileUtils.deleteQuietly(downloadDir); + Files.move(tmpDir, downloadDir); } } catch (Exception e) { - cleanTempFiles(); - throw e; + FileUtils.deleteQuietly(tmpDir); + throw new IllegalStateException("Failed to install edition", e); + } + } + + protected void download(Release release) { + try { + downloadRelease(release); + } catch (Exception e) { + String message = String.format("Fail to download the plugin (%s, version %s) from %s (error is : %s)", + release.getArtifact().getKey(), release.getVersion().getName(), release.getDownloadUrl(), e.getMessage()); + LOG.debug(message, e); + throw new IllegalStateException(message, e); + } + } + + private void downloadRelease(Release release) throws URISyntaxException, IOException { + String url = release.getDownloadUrl(); + + URI uri = new URI(url); + if (url.startsWith("file:")) { + // used for tests + File file = toFile(uri.toURL()); + Files.copy(file.toPath(), tmpDir); + } else { + String filename = substringAfterLast(uri.getPath(), "/"); + if (!filename.endsWith("." + PLUGIN_EXTENSION)) { + filename = release.getKey() + "-" + release.getVersion() + "." + PLUGIN_EXTENSION; + } + Path targetFile = tmpDir.resolve(filename); + downloader.download(uri, targetFile.toFile()); } } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginDownloaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginDownloaderTest.java index 4325e0f8d25..d6b7bd1bc69 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginDownloaderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/PluginDownloaderTest.java @@ -196,7 +196,7 @@ public class PluginDownloaderTest { pluginDownloader.start(); pluginDownloader.download("foo", create("1.0")); verify(httpDownloader, never()).download(any(URI.class), any(File.class)); - assertThat(pluginDownloader.hasDownloads()).isTrue(); + assertThat(noDownloadedFiles()).isGreaterThan(0); } @Test @@ -237,7 +237,7 @@ public class PluginDownloaderTest { @Test public void read_download_folder() throws Exception { pluginDownloader.start(); - assertThat(pluginDownloader.getDownloadedPluginFilenames()).hasSize(0); + assertThat(noDownloadedFiles()).isZero(); copyFileToDirectory(TestProjectUtils.jarOf("test-base-plugin"), downloadDir); @@ -259,7 +259,7 @@ public class PluginDownloaderTest { File file2 = new File(downloadDir, "file2.jar"); file2.createNewFile(); - assertThat(pluginDownloader.getDownloadedPluginFilenames()).hasSize(2); + assertThat(noDownloadedFiles()).isEqualTo(2); } @Test @@ -270,9 +270,13 @@ public class PluginDownloaderTest { file2.createNewFile(); pluginDownloader.start(); - assertThat(pluginDownloader.hasDownloads()).isTrue(); + assertThat(noDownloadedFiles()).isGreaterThan(0); pluginDownloader.cancelDownloads(); - assertThat(pluginDownloader.hasDownloads()).isFalse(); + assertThat(noDownloadedFiles()).isZero(); + } + + private int noDownloadedFiles() { + return downloadDir.listFiles((file, name) -> name.endsWith(".jar")).length; } // SONAR-5011 diff --git a/server/sonar-server/src/test/java/org/sonar/server/plugins/edition/EditionPluginDownloaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/plugins/edition/EditionPluginDownloaderTest.java index b2288f96358..6f3545e272d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/plugins/edition/EditionPluginDownloaderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/plugins/edition/EditionPluginDownloaderTest.java @@ -20,11 +20,13 @@ package org.sonar.server.plugins.edition; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Collections; +import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,12 +36,15 @@ import org.mockito.MockitoAnnotations; import org.sonar.api.utils.HttpDownloader; import org.sonar.server.platform.ServerFileSystem; import org.sonar.server.plugins.UpdateCenterMatrixFactory; +import org.sonar.updatecenter.common.Plugin; import org.sonar.updatecenter.common.Release; import org.sonar.updatecenter.common.UpdateCenter; import org.sonar.updatecenter.common.Version; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -57,32 +62,55 @@ public class EditionPluginDownloaderTest { private HttpDownloader httpDownloader; private File downloadDir; + private File tmpDir; private EditionPluginDownloader downloader; @Before public void setUp() throws IOException { MockitoAnnotations.initMocks(this); downloadDir = temp.newFolder(); + tmpDir = new File(downloadDir.getParentFile(), downloadDir.getName() + "_tmp"); when(updateCenterMatrixFactory.getUpdateCenter(anyBoolean())).thenReturn(Optional.of(updateCenter)); when(fs.getEditionDownloadedPluginsDir()).thenReturn(downloadDir); downloader = new EditionPluginDownloader(updateCenterMatrixFactory, httpDownloader, fs); } @Test - public void download_plugin_to_tmp_and_rename_it() throws IOException, URISyntaxException { - String url = "http://host/plugin.jar"; - Release release = createRelease("pluginKey", "1.0", url); + public void download_plugin_to_tmp() throws IOException, URISyntaxException { + List releases = ImmutableList.of(createRelease("plugin1", "1.0", "http://host/plugin1.jar"), + createRelease("plugin2", "1.0", "http://host/plugin2.jar")); - // mock file downloaded - File tmp = new File(downloadDir, "plugin.jar.tmp"); - tmp.createNewFile(); + when(updateCenter.findInstallablePlugins("plugins", Version.create(""))).thenReturn(releases); + downloader.installEdition(Collections.singleton("plugins")); - when(updateCenter.findInstallablePlugins("pluginKey", Version.create(""))).thenReturn(Collections.singletonList(release)); - downloader.installEdition(Collections.singleton("pluginKey")); + verify(httpDownloader).download(new URI("http://host/plugin1.jar"), new File(tmpDir, "plugin1.jar")); + verify(httpDownloader).download(new URI("http://host/plugin2.jar"), new File(tmpDir, "plugin2.jar")); - verify(httpDownloader).download(new URI(url), tmp); - assertThat(tmp).doesNotExist(); - assertThat(new File(downloadDir, "plugin.jar")).isFile(); + assertThat(downloadDir).isDirectory(); + assertThat(tmpDir).doesNotExist(); + } + + @Test + public void dont_write_download_dir_if_download_fails() throws URISyntaxException { + List releases = ImmutableList.of(createRelease("plugin1", "1.0", "http://host/plugin1.jar"), + createRelease("plugin2", "1.0", "http://host/plugin2.jar")); + + doThrow(new IllegalStateException("error")).when(httpDownloader).download(new URI("http://host/plugin1.jar"), new File(tmpDir, "plugin1.jar")); + + when(updateCenter.findInstallablePlugins("plugins", Version.create(""))).thenReturn(releases); + + try { + downloader.installEdition(Collections.singleton("plugins")); + fail("expecting exception"); + } catch (IllegalStateException e) { + + } + + verify(httpDownloader).download(new URI("http://host/plugin1.jar"), new File(tmpDir, "plugin1.jar")); + verify(httpDownloader).download(new URI("http://host/plugin2.jar"), new File(tmpDir, "plugin2.jar")); + + assertThat(downloadDir.list()).isEmpty(); + assertThat(tmpDir).doesNotExist(); } private static Release createRelease(String key, String version, String url) { @@ -90,6 +118,7 @@ public class EditionPluginDownloaderTest { when(release.getKey()).thenReturn(key); when(release.getVersion()).thenReturn(Version.create(version)); when(release.getDownloadUrl()).thenReturn(url); + when(release.getArtifact()).thenReturn(Plugin.factory(key)); return release; } } -- 2.39.5