From 8f1df5c561f9bd1a94aaf95dcda0b474bab472d2 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Fri, 29 May 2015 11:29:09 +0200 Subject: [PATCH] SONAR-6577 Offline mode in preview mode --- .../batch/bootstrap/GlobalContainer.java | 1 + .../bootstrap/PersistentCacheProvider.java | 51 ++++ .../sonar/batch/bootstrap/ServerClient.java | 59 +++- .../repository/DefaultServerIssuesLoader.java | 15 +- .../batch/repository/user/UserRepository.java | 29 +- .../PersistentCacheProviderTest.java | 62 ++++ .../batch/bootstrap/ServerClientTest.java | 68 ++++- sonar-home/pom.xml | 18 -- .../java/org/sonar/home/cache/FileCache.java | 8 +- .../java/org/sonar/home/cache/FileHashes.java | 10 +- .../org/sonar/home/cache/PersistentCache.java | 265 ++++++++++++++++++ .../home/cache/PersistentCacheBuilder.java | 73 +++++ .../cache/PersistentCacheBuilderTest.java | 62 ++++ .../sonar/home/cache/PersistentCacheTest.java | 132 +++++++++ 14 files changed, 793 insertions(+), 60 deletions(-) create mode 100644 sonar-batch/src/main/java/org/sonar/batch/bootstrap/PersistentCacheProvider.java create mode 100644 sonar-batch/src/test/java/org/sonar/batch/bootstrap/PersistentCacheProviderTest.java create mode 100644 sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java create mode 100644 sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java create mode 100644 sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java create mode 100644 sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java index f24fe52fbe8..facf6c58a3a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/GlobalContainer.java @@ -112,6 +112,7 @@ public class GlobalContainer extends ComponentContainer { DefaultHttpDownloader.class, UriReader.class, new FileCacheProvider(), + new PersistentCacheProvider(), System2.INSTANCE, DefaultI18n.class, Durations.class, diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PersistentCacheProvider.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PersistentCacheProvider.java new file mode 100644 index 00000000000..eca9407bce9 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/PersistentCacheProvider.java @@ -0,0 +1,51 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.bootstrap; + +import org.sonar.home.cache.PersistentCacheBuilder; +import org.picocontainer.injectors.ProviderAdapter; + +import java.nio.file.Paths; + +import org.sonar.home.cache.PersistentCache; + +public class PersistentCacheProvider extends ProviderAdapter { + private PersistentCache cache; + + public PersistentCache provide(BootstrapProperties props) { + if (cache == null) { + PersistentCacheBuilder builder = new PersistentCacheBuilder(); + + String forceUpdate = props.property("sonar.forceUpdate"); + + if ("true".equals(forceUpdate)) { + builder.forceUpdate(true); + } + + String home = props.property("sonar.userHome"); + if (home != null) { + builder.setSonarHome(Paths.get(home)); + } + + cache = builder.build(); + } + return cache; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java index db2d261b1bb..c0574deddd6 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java @@ -19,6 +19,7 @@ */ package org.sonar.batch.bootstrap; +import org.sonar.home.cache.PersistentCache; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -39,6 +40,7 @@ import org.sonar.core.util.DefaultHttpDownloader; import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -47,6 +49,7 @@ import java.net.URI; import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Callable; /** * Replace the deprecated org.sonar.batch.ServerMetadata @@ -59,11 +62,15 @@ public class ServerClient { private static final String GET = "GET"; private BootstrapProperties props; + private PersistentCache cache; private DefaultHttpDownloader.BaseHttpDownloader downloader; + private DefaultAnalysisMode mode; - public ServerClient(BootstrapProperties settings, EnvironmentInformation env) { + public ServerClient(BootstrapProperties settings, EnvironmentInformation env, PersistentCache cache, DefaultAnalysisMode mode) { this.props = settings; this.downloader = new DefaultHttpDownloader.BaseHttpDownloader(settings.properties(), env.toString()); + this.cache = cache; + this.mode = mode; } public String getURL() { @@ -102,31 +109,63 @@ public class ServerClient { } public String request(String pathStartingWithSlash, String requestMethod, boolean wrapHttpException, @Nullable Integer timeoutMillis) { - InputSupplier inputSupplier = doRequest(pathStartingWithSlash, requestMethod, timeoutMillis); + final byte[] buf = load(pathStartingWithSlash, requestMethod, wrapHttpException, timeoutMillis); try { - return IOUtils.toString(inputSupplier.getInput(), "UTF-8"); - } catch (HttpDownloader.HttpException e) { - throw wrapHttpException ? handleHttpException(e) : e; - } catch (IOException e) { + return new String(buf, "UTF-8"); + } catch (UnsupportedEncodingException e) { throw new IllegalStateException(String.format("Unable to request: %s", pathStartingWithSlash), e); } } public InputSupplier doRequest(String pathStartingWithSlash, String requestMethod, @Nullable Integer timeoutMillis) { + final byte[] buf = load(pathStartingWithSlash, requestMethod, false, timeoutMillis); + + return new InputSupplier() { + @Override + public InputStream getInput() throws IOException { + return new ByteArrayInputStream(buf); + } + }; + } + + private byte[] load(String pathStartingWithSlash, String requestMethod, boolean wrapHttpException, @Nullable Integer timeoutMillis) { Preconditions.checkArgument(pathStartingWithSlash.startsWith("/"), "Path must start with slash /"); String path = StringEscapeUtils.escapeHtml(pathStartingWithSlash); - URI uri = URI.create(getURL() + path); + try { + if (GET.equals(requestMethod) && mode.isPreview()) { + return cache.get(uri.toString(), new HttpValueLoader(uri, requestMethod, timeoutMillis)); + } else { + return new HttpValueLoader(uri, requestMethod, timeoutMillis).call(); + } + } catch (HttpDownloader.HttpException e) { + throw wrapHttpException ? handleHttpException(e) : e; + } catch (Exception e) { + throw new IllegalStateException(String.format("Unable to request: %s", uri), e); + } + } + + private class HttpValueLoader implements Callable { + private URI uri; + private String requestMethod; + private Integer timeoutMillis; + + public HttpValueLoader(URI uri, String requestMethod, Integer timeoutMillis) { + this.uri = uri; + this.requestMethod = requestMethod; + this.timeoutMillis = timeoutMillis; + } + + @Override + public byte[] call() throws Exception { InputSupplier inputSupplier; if (Strings.isNullOrEmpty(getLogin())) { inputSupplier = downloader.newInputSupplier(uri, requestMethod, timeoutMillis); } else { inputSupplier = downloader.newInputSupplier(uri, requestMethod, getLogin(), getPassword(), timeoutMillis); } - return inputSupplier; - } catch (Exception e) { - throw new IllegalStateException(String.format("Unable to request: %s", uri), e); + return IOUtils.toByteArray(inputSupplier.getInput()); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java index 8193d3a3ebe..e71e9eb54cb 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java @@ -38,18 +38,23 @@ public class DefaultServerIssuesLoader implements ServerIssuesLoader { @Override public void load(String componentKey, Function consumer, boolean incremental) { - InputSupplier request = serverClient.doRequest("/batch/issues?key=" + ServerClient.encodeForUrl(componentKey), "GET", null); - try (InputStream is = request.getInput()) { + try { + InputSupplier request = serverClient.doRequest("/batch/issues?key=" + ServerClient.encodeForUrl(componentKey), "GET", null); + parseIssues(request, consumer); + } catch (HttpDownloader.HttpException e) { + throw serverClient.handleHttpException(e); + } + } + + private static void parseIssues(InputSupplier input, Function consumer) { + try (InputStream is = input.getInput()) { ServerIssue previousIssue = ServerIssue.parseDelimitedFrom(is); while (previousIssue != null) { consumer.apply(previousIssue); previousIssue = ServerIssue.parseDelimitedFrom(is); } - } catch (HttpDownloader.HttpException e) { - throw serverClient.handleHttpException(e); } catch (IOException e) { throw new IllegalStateException("Unable to get previous issues", e); } } - } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java b/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java index 099b9470903..7208086c308 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java @@ -46,24 +46,35 @@ public class UserRepository { if (userLogins.isEmpty()) { return Collections.emptyList(); } - InputSupplier request = serverClient.doRequest("/batch/users?logins=" + Joiner.on(',').join(Lists.transform(userLogins, new Function() { - @Override - public String apply(String input) { - return ServerClient.encodeForUrl(input); - } - })), "GET", null); + + try { + InputSupplier request = serverClient.doRequest("/batch/users?logins=" + Joiner.on(',').join(Lists.transform(userLogins, new Function() { + @Override + public String apply(String input) { + return ServerClient.encodeForUrl(input); + } + })), "GET", null); + + return parseUsers(request); + + } catch (HttpDownloader.HttpException e) { + throw serverClient.handleHttpException(e); + } + } + + private static Collection parseUsers(InputSupplier input) { List users = new ArrayList<>(); - try (InputStream is = request.getInput()) { + + try (InputStream is = input.getInput()) { BatchInput.User user = BatchInput.User.parseDelimitedFrom(is); while (user != null) { users.add(user); user = BatchInput.User.parseDelimitedFrom(is); } - } catch (HttpDownloader.HttpException e) { - throw serverClient.handleHttpException(e); } catch (IOException e) { throw new IllegalStateException("Unable to get user details from server", e); } + return users; } diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PersistentCacheProviderTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PersistentCacheProviderTest.java new file mode 100644 index 00000000000..fb07230e741 --- /dev/null +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/PersistentCacheProviderTest.java @@ -0,0 +1,62 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.bootstrap; + +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.junit.Before; + +import static org.mockito.Mockito.when; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; + +public class PersistentCacheProviderTest { + private PersistentCacheProvider provider = null; + + @Mock + private BootstrapProperties props = null; + + @Before + public void prepare() { + MockitoAnnotations.initMocks(this); + provider = new PersistentCacheProvider(); + } + + @Test + public void test_singleton() { + assertThat(provider.provide(props)).isEqualTo(provider.provide(props)); + } + + @Test + public void test_cache_dir() { + assertThat(provider.provide(props).getBaseDirectory().toFile()).exists().isDirectory(); + } + + @Test + public void test_forceUpdate() { + // normally don't force update + assertThat(provider.provide(props).isForceUpdate()).isFalse(); + + when(props.property("sonar.forceUpdate")).thenReturn("true"); + provider = new PersistentCacheProvider(); + assertThat(provider.provide(props).isForceUpdate()).isTrue(); + } +} diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/ServerClientTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/ServerClientTest.java index b50d3643887..beab35ba672 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/ServerClientTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/ServerClientTest.java @@ -19,7 +19,9 @@ */ package org.sonar.batch.bootstrap; -import com.google.common.io.Files; +import org.junit.Before; +import org.sonar.home.cache.PersistentCacheBuilder; +import org.sonar.home.cache.PersistentCache; import org.apache.commons.io.IOUtils; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; @@ -39,6 +41,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.apache.commons.io.IOUtils.write; @@ -52,23 +55,71 @@ public class ServerClientTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); @Rule + public TemporaryFolder cacheTmp = new TemporaryFolder(); + @Rule public ExpectedException thrown = ExpectedException.none(); - MockHttpServer server = null; - BootstrapProperties bootstrapProps = mock(BootstrapProperties.class); + private MockHttpServer server = null; + private BootstrapProperties bootstrapProps = mock(BootstrapProperties.class); + private DefaultAnalysisMode mode = null; + + @Before + public void setUp() { + mode = mock(DefaultAnalysisMode.class); + when(mode.isPreview()).thenReturn(true); + } + @After public void stopServer() { if (server != null) { server.stop(); } } + + @Test + public void dont_cache_post_request() throws Exception { + server = new MockHttpServer(); + server.start(); + server.setMockResponseData("this is the content"); + + assertThat(newServerClient().request("/foo", "POST")).isEqualTo("this is the content"); + + // cache never accessed, so not even the .lock should be there + assertThat(getNumFilesInCache()).isEqualTo(0); + } + + @Test + public void dont_cache_non_preview_mode() throws Exception { + server = new MockHttpServer(); + server.start(); + server.setMockResponseData("this is the content"); + + when(mode.isPreview()).thenReturn(false); + assertThat(newServerClient().request("/foo")).isEqualTo("this is the content"); + + // cache never accessed, so not even the .lock should be there + assertThat(getNumFilesInCache()).isEqualTo(0); + } + + @Test + public void cache_preview_mode() throws Exception { + server = new MockHttpServer(); + server.start(); + server.setMockResponseData("this is the content"); + + assertThat(newServerClient().request("/foo")).isEqualTo("this is the content"); + + //should have the .lock and one request cached + assertThat(getNumFilesInCache()).isEqualTo(2); + } @Test public void should_remove_url_ending_slash() { BootstrapProperties settings = mock(BootstrapProperties.class); when(settings.property("sonar.host.url")).thenReturn("http://localhost:8080/sonar/"); - ServerClient client = new ServerClient(settings, new EnvironmentInformation("Junit", "4")); + PersistentCache ps = new PersistentCacheBuilder().setSonarHome(cacheTmp.getRoot().toPath()).build(); + ServerClient client = new ServerClient(settings, new EnvironmentInformation("Junit", "4"), ps, mode); assertThat(client.getURL()).isEqualTo("http://localhost:8080/sonar"); } @@ -99,7 +150,7 @@ public class ServerClientTest { File file = temp.newFile(); newServerClient().download("/foo", file); - assertThat(Files.toString(file, StandardCharsets.UTF_8)).isEqualTo("this is the content"); + assertThat(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)).isEqualTo("this is the content"); } @Test @@ -153,7 +204,12 @@ public class ServerClientTest { private ServerClient newServerClient() { when(bootstrapProps.property("sonar.host.url")).thenReturn("http://localhost:" + server.getPort()); - return new ServerClient(bootstrapProps, new EnvironmentInformation("Junit", "4")); + PersistentCache ps = new PersistentCacheBuilder().setSonarHome(cacheTmp.getRoot().toPath()).build(); + return new ServerClient(bootstrapProps, new EnvironmentInformation("Junit", "4"), ps, mode); + } + + private int getNumFilesInCache() { + return new File(cacheTmp.getRoot(), "ws_cache").listFiles().length; } static class MockHttpServer { diff --git a/sonar-home/pom.xml b/sonar-home/pom.xml index 4acab99b362..10b477a495c 100644 --- a/sonar-home/pom.xml +++ b/sonar-home/pom.xml @@ -14,10 +14,6 @@ Access the user home directory that contains cache of files - - commons-io - commons-io - org.slf4j slf4j-api @@ -28,20 +24,6 @@ jsr305 provided - - org.codehaus.sonar - sonar-plugin-api - - - jfree - jcommon - - - jfree - jfreechart - - - org.codehaus.sonar diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java index 63a1168639d..3b681edcff1 100644 --- a/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java +++ b/sonar-home/src/main/java/org/sonar/home/cache/FileCache.java @@ -19,13 +19,13 @@ */ package org.sonar.home.cache; -import org.apache.commons.io.FileUtils; import org.sonar.home.log.Log; import javax.annotation.CheckForNull; import java.io.File; import java.io.IOException; +import java.nio.file.Files; /** * This class is responsible for managing Sonar batch file cache. You can put file into cache and @@ -108,7 +108,7 @@ public class FileCache { log.warn(String.format("Unable to rename %s to %s", sourceFile.getAbsolutePath(), targetFile.getAbsolutePath())); log.warn(String.format("A copy/delete will be tempted but with no garantee of atomicity")); try { - FileUtils.moveFile(sourceFile, targetFile); + Files.move(sourceFile.toPath(), targetFile.toPath()); } catch (IOException e) { throw new IllegalStateException("Fail to move " + sourceFile.getAbsolutePath() + " to " + targetFile, e); } @@ -121,7 +121,7 @@ public class FileCache { private void mkdirQuietly(File hashDir) { try { - FileUtils.forceMkdir(hashDir); + Files.createDirectories(hashDir.toPath()); } catch (IOException e) { throw new IllegalStateException("Fail to create cache directory: " + hashDir, e); } @@ -151,7 +151,7 @@ public class FileCache { if (!dir.isDirectory() || !dir.exists()) { log.debug("Create : " + dir.getAbsolutePath()); try { - FileUtils.forceMkdir(dir); + Files.createDirectories(dir.toPath()); } catch (IOException e) { throw new IllegalStateException("Unable to create " + debugTitle + dir.getAbsolutePath(), e); } diff --git a/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java b/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java index 18a7a5fe912..549f8d10f63 100644 --- a/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java +++ b/sonar-home/src/main/java/org/sonar/home/cache/FileHashes.java @@ -19,8 +19,6 @@ */ package org.sonar.home.cache; -import org.apache.commons.io.IOUtils; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -49,16 +47,12 @@ public class FileHashes { * Computes the hash of given stream. The stream is closed by this method. */ public String of(InputStream input) { - try { + try(InputStream is = input) { MessageDigest digest = MessageDigest.getInstance("MD5"); - byte[] hash = digest(input, digest); + byte[] hash = digest(is, digest); return toHex(hash); - } catch (Exception e) { throw new IllegalStateException("Fail to compute hash", e); - - } finally { - IOUtils.closeQuietly(input); } } diff --git a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java new file mode 100644 index 00000000000..f5493d3d6a0 --- /dev/null +++ b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCache.java @@ -0,0 +1,265 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.home.cache; + +import org.sonar.home.log.Log; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.Callable; + +import static java.nio.file.StandardOpenOption.*; + +public class PersistentCache { + private static final Charset ENCODING = StandardCharsets.UTF_8; + private static final String DIGEST_ALGO = "MD5"; + private static final String LOCK_FNAME = ".lock"; + + private Path baseDir; + + // eviction strategy is to expire entries after modification once a time duration has elapsed + private final long defaultDurationToExpireMs; + private final Log log; + private final boolean forceUpdate; + + public PersistentCache(Path baseDir, long defaultDurationToExpireMs, Log log, boolean forceUpdate) { + this.baseDir = baseDir; + this.defaultDurationToExpireMs = defaultDurationToExpireMs; + this.log = log; + this.forceUpdate = forceUpdate; + + log.info("cache: " + baseDir + " default expiration time (ms): " + defaultDurationToExpireMs); + + if (forceUpdate) { + log.debug("cache: forcing update"); + } + + try { + Files.createDirectories(baseDir); + } catch (IOException e) { + throw new IllegalStateException("failed to create cache dir", e); + } + } + + public Path getBaseDirectory() { + return baseDir; + } + + public boolean isForceUpdate() { + return forceUpdate; + } + + @CheckForNull + public synchronized String getString(@Nonnull String obj, @Nullable final Callable valueLoader) throws Exception { + byte[] cached = get(obj, new Callable() { + @Override + public byte[] call() throws Exception { + String s = valueLoader.call(); + if (s != null) { + return s.getBytes(ENCODING); + } + return null; + } + }); + + if (cached == null) { + return null; + } + + return new String(cached, ENCODING); + } + + @CheckForNull + public synchronized byte[] get(@Nonnull String obj, @Nullable Callable valueLoader) throws Exception { + String key = getKey(obj); + log.debug("cache: " + obj + " -> " + key); + + try (FileLock l = lock()) { + if (!forceUpdate) { + byte[] cached = getCache(key); + + if (cached != null) { + log.debug("cache hit for " + obj); + return cached; + } + + log.debug("cache miss for " + obj); + } else { + log.debug("cache force update for " + obj); + } + + if (valueLoader != null) { + byte[] value = valueLoader.call(); + if (value != null) { + putCache(key, value); + } + return value; + } + } + + return null; + } + + /** + * Deletes all cache entries + */ + public synchronized void clear() { + log.info("cache: clearing"); + try (FileLock l = lock()) { + deleteCacheEntries(createClearFilter()); + } catch (IOException e) { + log.error("Error clearing cache", e); + } + } + + /** + * Deletes cache entries that are no longer valid according to the default expiration time period. + */ + public synchronized void clean() { + log.info("cache: cleaning"); + try (FileLock l = lock()) { + deleteCacheEntries(createCleanFilter()); + } catch (IOException e) { + log.error("Error cleaning cache", e); + } + } + + private FileLock lock() throws IOException { + FileChannel ch = FileChannel.open(getLockPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + return ch.lock(); + } + + private String getKey(String uri) { + try { + MessageDigest digest = MessageDigest.getInstance(DIGEST_ALGO); + digest.update(uri.getBytes(StandardCharsets.UTF_8)); + return byteArrayToHex(digest.digest()); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("Couldn't create hash", e); + } + } + + private void deleteCacheEntries(DirectoryStream.Filter filter) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(baseDir, filter)) { + for (Path p : stream) { + try { + Files.delete(p); + } catch (Exception e) { + log.error("Error deleting " + p, e); + } + } + } + } + + private DirectoryStream.Filter createClearFilter() throws IOException { + return new DirectoryStream.Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return !LOCK_FNAME.equals(entry.getFileName().toString()); + } + }; + } + + private DirectoryStream.Filter createCleanFilter() throws IOException { + return new DirectoryStream.Filter() { + @Override + public boolean accept(Path entry) throws IOException { + if (LOCK_FNAME.equals(entry.getFileName().toString())) { + return false; + } + + return isCacheEntryExpired(entry, PersistentCache.this.defaultDurationToExpireMs); + } + }; + } + + private void putCache(String key, byte[] value) throws UnsupportedEncodingException, IOException { + Path cachePath = getCacheEntryPath(key); + Files.write(cachePath, value, CREATE, WRITE, TRUNCATE_EXISTING); + } + + private byte[] getCache(String key) throws IOException { + Path cachePath = getCacheEntryPath(key); + + if (!validateCacheEntry(cachePath, this.defaultDurationToExpireMs)) { + return null; + } + + return Files.readAllBytes(cachePath); + } + + private boolean validateCacheEntry(Path cacheEntryPath, long durationToExpireMs) throws IOException { + if (!Files.exists(cacheEntryPath)) { + return false; + } + + if (isCacheEntryExpired(cacheEntryPath, durationToExpireMs)) { + log.debug("cache: expiring entry"); + Files.delete(cacheEntryPath); + return false; + } + + return true; + } + + private boolean isCacheEntryExpired(Path cacheEntryPath, long durationToExpireMs) throws IOException { + BasicFileAttributes attr = Files.readAttributes(cacheEntryPath, BasicFileAttributes.class); + long modTime = attr.lastModifiedTime().toMillis(); + + long age = System.currentTimeMillis() - modTime; + + if (age > durationToExpireMs) { + return true; + } + + return false; + } + + private Path getLockPath() { + return baseDir.resolve(LOCK_FNAME); + } + + private Path getCacheEntryPath(String key) { + return baseDir.resolve(key); + } + + private static String byteArrayToHex(byte[] a) { + StringBuilder sb = new StringBuilder(a.length * 2); + for (byte b : a) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } +} diff --git a/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java new file mode 100644 index 00000000000..c8fcf06d4d0 --- /dev/null +++ b/sonar-home/src/main/java/org/sonar/home/cache/PersistentCacheBuilder.java @@ -0,0 +1,73 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.home.cache; + +import org.sonar.home.log.StandardLog; + +import org.sonar.home.log.Log; + +import javax.annotation.Nullable; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +public class PersistentCacheBuilder { + private boolean forceUpdate = false; + private Path cachePath = null; + private Log log = new StandardLog(); + private String name = "ws_cache"; + + public PersistentCache build() { + if (cachePath == null) { + setSonarHome(findHome()); + } + + return new PersistentCache(cachePath, TimeUnit.MILLISECONDS.convert(1L, TimeUnit.DAYS), log, forceUpdate); + } + + public PersistentCacheBuilder setLog(Log log) { + this.log = log; + return this; + } + + public PersistentCacheBuilder setSonarHome(@Nullable Path p) { + if (p != null) { + this.cachePath = p.resolve(name); + } + return this; + } + + public PersistentCacheBuilder forceUpdate(boolean update) { + this.forceUpdate = update; + return this; + } + + private static Path findHome() { + String home = System.getenv("SONAR_USER_HOME"); + + if (home != null) { + return Paths.get(home); + } + + home = System.getProperty("user.home"); + return Paths.get(home, ".sonar"); + } +} diff --git a/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java new file mode 100644 index 00000000000..027e2d68924 --- /dev/null +++ b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheBuilderTest.java @@ -0,0 +1,62 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.home.cache; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +public class PersistentCacheBuilderTest { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void user_home_property_can_be_null() { + PersistentCache cache = new PersistentCacheBuilder().setSonarHome(null).build(); + assertTrue(Files.isDirectory(cache.getBaseDirectory())); + assertThat(cache.getBaseDirectory().getFileName().toString()).isEqualTo("ws_cache"); + } + + @Test + public void set_user_home() { + PersistentCache cache = new PersistentCacheBuilder().setSonarHome(temp.getRoot().toPath()).build(); + + assertThat(cache.getBaseDirectory().getParent().toString()).isEqualTo(temp.getRoot().toPath().toString()); + assertTrue(Files.isDirectory(cache.getBaseDirectory())); + } + + @Test + public void read_system_env() { + System.setProperty("user.home", temp.getRoot().getAbsolutePath()); + + PersistentCache cache = new PersistentCacheBuilder().build(); + assertTrue(Files.isDirectory(cache.getBaseDirectory())); + assertThat(cache.getBaseDirectory().getFileName().toString()).isEqualTo("ws_cache"); + + String expectedSonarHome = temp.getRoot().toPath().resolve(".sonar").toString(); + assertThat(cache.getBaseDirectory().getParent().toString()).isEqualTo(expectedSonarHome); + } +} diff --git a/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java new file mode 100644 index 00000000000..8127e41885d --- /dev/null +++ b/sonar-home/src/test/java/org/sonar/home/cache/PersistentCacheTest.java @@ -0,0 +1,132 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.home.cache; + +import org.sonar.home.log.Slf4jLog; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.util.concurrent.Callable; + +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mock; +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Before; +import org.junit.Test; + +public class PersistentCacheTest { + private final static String URI = "key1"; + private final static String VALUE = "cache content"; + private final static long CACHE_EXPIRE = 1000; + private PersistentCache cache = null; + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private Slf4jLog log = new Slf4jLog(FileCacheTest.class); + + @Before + public void setUp() { + cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, false); + } + + @Test + public void testCacheMiss() throws Exception { + assertCacheHit(false); + } + + @Test + public void testNullLoader() throws Exception { + assertThat(cache.get(URI, null)).isNull(); + assertCacheHit(false); + } + + @Test + public void testNullValue() throws Exception { + // mocks have their methods returning null by default + Callable c = mock(Callable.class); + assertThat(cache.get(URI, c)).isNull(); + verify(c).call(); + assertCacheHit(false); + } + + @Test + public void testClean() throws Exception { + assertCacheHit(false); + cache.clear(); + assertCacheHit(false); + } + + @Test + public void testCacheHit() throws Exception { + assertCacheHit(false); + assertCacheHit(true); + } + + @Test + public void testForceUpdate() throws Exception { + cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, true); + + assertCacheHit(false); + assertCacheHit(false); + assertCacheHit(false); + + // with forceUpdate, it should still have cached the last call + cache = new PersistentCache(tmp.getRoot().toPath(), CACHE_EXPIRE, log, false); + assertCacheHit(true); + } + + @Test + public void testExpiration() throws Exception { + assertCacheHit(false); + Thread.sleep(CACHE_EXPIRE); + assertCacheHit(false); + } + + private void assertCacheHit(boolean hit) throws Exception { + CacheFillerString c = new CacheFillerString(); + assertThat(cache.getString(URI, c)).isEqualTo(VALUE); + assertThat(c.wasCalled).isEqualTo(!hit); + } + + private class CacheFillerString implements Callable { + public boolean wasCalled = false; + + @Override + public String call() throws Exception { + wasCalled = true; + return VALUE; + } + } + + /** + * WSCache should be transparent regarding exceptions: if an exception is thrown by the value loader, it should pass through + * the cache to the original caller using the cache. + * @throws Exception + */ + @Test(expected = ArithmeticException.class) + public void testExceptions() throws Exception { + Callable c = mock(Callable.class); + when(c.call()).thenThrow(ArithmeticException.class); + cache.get(URI, c); + } + +} -- 2.39.5