From d5f10d28f6a996af9c34c4757e7dcdf446b43af9 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Fri, 31 Jul 2015 14:44:08 +0200 Subject: [PATCH] SONAR-6770 Improve logging of cache usage --- .../batch/bootstrap/AbstractServerLoader.java | 31 ++++++ .../batch/bootstrap/BatchPluginInstaller.java | 36 ++++-- .../sonar/batch/bootstrap/ServerClient.java | 2 +- .../org/sonar/batch/bootstrap/WSLoader.java | 104 ++++++++++++------ .../sonar/batch/bootstrap/WSLoaderResult.java | 41 +++++++ .../DefaultServerLineHashesLoader.java | 12 +- .../issue/tracking/ServerIssueRepository.java | 84 ++++++++------ .../batch/mediumtest/BatchMediumTester.java | 21 +++- .../DefaultGlobalRepositoriesLoader.java | 10 +- .../DefaultProjectRepositoriesLoader.java | 14 ++- .../repository/DefaultServerIssuesLoader.java | 10 +- .../repository/GlobalRepositoriesLoader.java | 2 + .../GlobalRepositoriesProvider.java | 11 +- .../repository/ProjectRepositoriesLoader.java | 2 + .../ProjectRepositoriesProvider.java | 11 +- .../batch/repository/ServerIssuesLoader.java | 2 +- .../batch/repository/user/UserRepository.java | 31 ++++-- .../sonar/batch/rule/DefaultRulesLoader.java | 11 +- .../org/sonar/batch/rule/RulesLoader.java | 2 + .../org/sonar/batch/rule/RulesProvider.java | 14 +++ .../bootstrap/BatchPluginInstallerTest.java | 11 +- .../sonar/batch/bootstrap/WSLoaderTest.java | 47 ++++---- .../DefaultServerLineHashesLoaderTest.java | 7 +- .../DefaultProjectRepositoriesLoaderTest.java | 13 ++- .../DefaultServerIssuesLoaderTest.java | 5 +- .../repository/user/UserRepositoryTest.java | 5 +- .../batch/rule/DefaultRulesLoaderTest.java | 16 ++- 27 files changed, 393 insertions(+), 162 deletions(-) create mode 100644 sonar-batch/src/main/java/org/sonar/batch/bootstrap/AbstractServerLoader.java create mode 100644 sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoaderResult.java diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/AbstractServerLoader.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/AbstractServerLoader.java new file mode 100644 index 00000000000..d7883e74b9a --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/AbstractServerLoader.java @@ -0,0 +1,31 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.bootstrap; + +public class AbstractServerLoader { + protected Boolean loadedFromCache = null; + + public boolean loadedFromCache() { + if (loadedFromCache == null) { + throw new IllegalStateException("Didn't load"); + } + return loadedFromCache; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginInstaller.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginInstaller.java index b6744b5ff4a..fb278f7ea85 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginInstaller.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchPluginInstaller.java @@ -19,8 +19,6 @@ */ package org.sonar.batch.bootstrap; -import com.google.common.io.Files; - import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -54,18 +52,25 @@ public class BatchPluginInstaller implements PluginInstaller { private final WSLoader wsLoader; private final FileCache fileCache; private final BatchPluginPredicate pluginPredicate; + private final ServerClient serverClient; - public BatchPluginInstaller(WSLoader wsLoader, FileCache fileCache, BatchPluginPredicate pluginPredicate) { + public BatchPluginInstaller(WSLoader wsLoader, ServerClient serverClient, FileCache fileCache, BatchPluginPredicate pluginPredicate) { this.wsLoader = wsLoader; this.fileCache = fileCache; this.pluginPredicate = pluginPredicate; + this.serverClient = serverClient; } @Override public Map installRemotes() { + return loadPlugins(listRemotePlugins()); + } + + private Map loadPlugins(List remotePlugins) { Map infosByKey = new HashMap<>(); - List remotePlugins = listRemotePlugins(); + Profiler profiler = Profiler.create(LOG).startDebug("Load plugins"); + for (RemotePlugin remotePlugin : remotePlugins) { if (pluginPredicate.apply(remotePlugin.getKey())) { File jarFile = download(remotePlugin); @@ -73,6 +78,7 @@ public class BatchPluginInstaller implements PluginInstaller { infosByKey.put(info.getKey(), info); } } + profiler.stopDebug(); return infosByKey; } @@ -100,7 +106,7 @@ public class BatchPluginInstaller implements PluginInstaller { LOG.info("Download {}", file.getFilename()); } - Files.write(wsLoader.load(url), toFile); + serverClient.download(url, toFile); } }); @@ -115,10 +121,8 @@ public class BatchPluginInstaller implements PluginInstaller { @VisibleForTesting List listRemotePlugins() { try { - Profiler profiler = Profiler.create(LOG).startInfo("Load plugins index"); - String indexContent = wsLoader.loadString(PLUGINS_INDEX_URL); - profiler.stopInfo(); - String[] rows = StringUtils.split(indexContent, CharUtils.LF); + String pluginIndex = loadPluginIndex(); + String[] rows = StringUtils.split(pluginIndex, CharUtils.LF); List result = Lists.newArrayList(); for (String row : rows) { result.add(RemotePlugin.unmarshal(row)); @@ -129,4 +133,18 @@ public class BatchPluginInstaller implements PluginInstaller { throw new IllegalStateException("Fail to load plugin index: " + PLUGINS_INDEX_URL, e); } } + + private String loadPluginIndex() { + Profiler profiler = Profiler.create(LOG).startInfo("Load plugins index"); + WSLoaderResult wsResult = wsLoader.loadString(PLUGINS_INDEX_URL); + + if (wsResult.isFromCache()) { + profiler.stopInfo("Load plugins index (done from cache)"); + } else { + profiler.stopInfo(); + } + + return wsResult.get(); + } + } diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java index 197544db075..a205a01c81b 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/ServerClient.java @@ -107,7 +107,7 @@ public class ServerClient { /** * @throws IllegalStateException on I/O error, not limited to the network connection and if HTTP response code > 400 and wrapHttpException is true - * @throws HttpException if HTTP response code > 400 and wrapHttpException is false + * @throws HttpDownloader.HttpException if HTTP response code > 400 and wrapHttpException is false */ public InputStream load(String pathStartingWithSlash, String requestMethod, boolean wrapHttpException, @Nullable Integer connectTimeoutMs, @Nullable Integer readTimeoutMs) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java index 26246bc3d9a..c37863f1a5c 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoader.java @@ -19,6 +19,9 @@ */ package org.sonar.batch.bootstrap; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + import javax.annotation.Nonnull; import org.sonar.api.utils.HttpDownloader; @@ -34,6 +37,7 @@ import static org.sonar.batch.bootstrap.WSLoader.LoadStrategy.*; import org.sonar.home.cache.PersistentCache; public class WSLoader { + private static final Logger LOG = Loggers.get(WSLoader.class); private static final String FAIL_MSG = "Server is not accessible and data is not cached"; private static final int CONNECT_TIMEOUT = 5000; private static final int READ_TIMEOUT = 10000; @@ -65,16 +69,20 @@ public class WSLoader { this(false, cache, client); } - public ByteSource loadSource(String id) { - return ByteSource.wrap(load(id)); + @Nonnull + public WSLoaderResult loadSource(String id) { + WSLoaderResult byteResult = load(id); + return new WSLoaderResult(ByteSource.wrap(byteResult.get()), byteResult.isFromCache()); } - public String loadString(String id) { - return new String(load(id), StandardCharsets.UTF_8); + @Nonnull + public WSLoaderResult loadString(String id) { + WSLoaderResult byteResult = load(id); + return new WSLoaderResult(new String(byteResult.get(), StandardCharsets.UTF_8), byteResult.isFromCache()); } @Nonnull - public byte[] load(String id) { + public WSLoaderResult load(String id) { if (loadStrategy == CACHE_FIRST) { return loadFromCacheFirst(id); } else { @@ -99,6 +107,7 @@ public class WSLoader { } private void switchToOffline() { + LOG.debug("server not available - switching to offline mode"); serverStatus = NOT_ACCESSIBLE; } @@ -110,71 +119,94 @@ public class WSLoader { return serverStatus == NOT_ACCESSIBLE; } - @Nonnull - private byte[] loadFromCacheFirst(String id) { - byte[] cached = loadFromCache(id); - if (cached != null) { - return cached; + private void updateCache(String id, byte[] value) { + if (cacheEnabled) { + try { + cache.put(client.getURI(id).toString(), value); + } catch (IOException e) { + LOG.warn("Error saving to WS cache", e); + } } + } + @Nonnull + private WSLoaderResult loadFromCacheFirst(String id) { try { - return loadFromServer(id); - } catch (Exception e) { - if (e.getCause() instanceof HttpDownloader.HttpException) { - throw e; + return loadFromCache(id); + } catch (NotAvailableException cacheNotAvailable) { + try { + return loadFromServer(id); + } catch (NotAvailableException serverNotAvailable) { + throw new IllegalStateException(FAIL_MSG, serverNotAvailable.getCause()); } - throw new IllegalStateException(FAIL_MSG, e); } } @Nonnull - private byte[] loadFromServerFirst(String id) { + private WSLoaderResult loadFromServerFirst(String id) { try { return loadFromServer(id); - } catch (Exception serverException) { - if (serverException.getCause() instanceof HttpDownloader.HttpException) { - // http exceptions should always be thrown (no fallback) - throw serverException; - } - byte[] cached = loadFromCache(id); - if (cached != null) { - return cached; + } catch (NotAvailableException serverNotAvailable) { + try { + return loadFromCache(id); + } catch (NotAvailableException cacheNotAvailable) { + throw new IllegalStateException(FAIL_MSG, serverNotAvailable.getCause()); } - throw new IllegalStateException(FAIL_MSG, serverException); } } - private byte[] loadFromCache(String id) { + @Nonnull + private WSLoaderResult loadFromCache(String id) throws NotAvailableException { if (!cacheEnabled) { - return null; + throw new NotAvailableException("cache disabled"); } try { - return cache.get(client.getURI(id).toString(), null); + byte[] result = cache.get(client.getURI(id).toString(), null); + if (result == null) { + throw new NotAvailableException("resource not cached"); + } + return new WSLoaderResult(result, true); } catch (IOException e) { + // any exception on the cache should fail fast throw new IllegalStateException(e); } } - private byte[] loadFromServer(String id) { + @Nonnull + private WSLoaderResult loadFromServer(String id) throws NotAvailableException { if (isOffline()) { - throw new IllegalStateException("Server is not accessible"); + throw new NotAvailableException("Server not available"); } try { InputStream is = client.load(id, REQUEST_METHOD, true, CONNECT_TIMEOUT, READ_TIMEOUT); switchToOnline(); byte[] value = IOUtils.toByteArray(is); - if (cacheEnabled) { - cache.put(client.getURI(id).toString(), value); - } - return value; + updateCache(client.getURI(id).toString(), value); + return new WSLoaderResult(value, false); } catch (IllegalStateException e) { + if (e.getCause() instanceof HttpDownloader.HttpException) { + // fail fast if it could connect but there was a application-level error + throw e; + } switchToOffline(); - throw e; + throw new NotAvailableException(e); } catch (Exception e) { - switchToOffline(); + // fail fast throw new IllegalStateException(e); } } + + private class NotAvailableException extends Exception { + private static final long serialVersionUID = 1L; + + public NotAvailableException(String message) { + super(message); + } + + public NotAvailableException(Throwable cause) { + super(cause); + } + } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoaderResult.java b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoaderResult.java new file mode 100644 index 00000000000..8615a122647 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/bootstrap/WSLoaderResult.java @@ -0,0 +1,41 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.bootstrap; + +import javax.annotation.Nonnull; + +public class WSLoaderResult { + private T result; + private boolean fromCache; + + public WSLoaderResult(T result, boolean fromCache) { + this.result = result; + this.fromCache = fromCache; + } + + @Nonnull + public T get() { + return result; + } + + public boolean isFromCache() { + return fromCache; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoader.java index 121783e1da5..e704b5ee621 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoader.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.issue.tracking; -import org.sonar.batch.util.BatchUtils; +import org.sonar.batch.bootstrap.WSLoaderResult; +import org.sonar.batch.util.BatchUtils; import org.sonar.batch.bootstrap.WSLoader; import com.google.common.base.Splitter; import com.google.common.collect.Iterators; @@ -45,10 +46,15 @@ public class DefaultServerLineHashesLoader implements ServerLineHashesLoader { Profiler profiler = Profiler.createIfDebug(Loggers.get(getClass())) .addContext("file", fileKey) .startDebug("Load line hashes"); + WSLoaderResult result = wsLoader.loadString("/api/sources/hash?key=" + BatchUtils.encodeForUrl(fileKey)); try { - return wsLoader.loadString("/api/sources/hash?key=" + BatchUtils.encodeForUrl(fileKey)); + return result.get(); } finally { - profiler.stopDebug(); + if (result.isFromCache()) { + profiler.stopDebug(); + } else { + profiler.stopDebug("Load line hashes (done from cache)"); + } } } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueRepository.java b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueRepository.java index 7b5aa5faee8..4607cacc61c 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueRepository.java +++ b/sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueRepository.java @@ -20,10 +20,13 @@ package org.sonar.batch.issue.tracking; import com.google.common.base.Function; -import java.util.ArrayList; + import java.util.Collections; +import java.util.LinkedList; import java.util.List; + import javax.annotation.Nullable; + import org.sonar.api.batch.AnalysisMode; import org.sonar.api.batch.BatchSide; import org.sonar.api.batch.InstantiationStrategy; @@ -73,25 +76,8 @@ public class ServerIssueRepository { Profiler profiler = Profiler.create(LOG).startInfo("Load server issues"); this.issuesCache = caches.createCache("previousIssues"); caches.registerValueCoder(ServerIssue.class, new ServerIssueValueCoder()); - previousIssuesLoader.load(reactor.getRoot().getKeyWithBranch(), new Function() { - - @Override - public Void apply(@Nullable ServerIssue issue) { - if (issue == null) { - return null; - } - String componentKey = ComponentKeys.createEffectiveKey(issue.getModuleKey(), issue.hasPath() ? issue.getPath() : null); - BatchComponent r = resourceCache.get(componentKey); - if (r == null) { - // Deleted resource - issuesCache.put(0, issue.getKey(), issue); - } else { - issuesCache.put(r.batchId(), issue.getKey(), issue); - } - return null; - } - }, false); - profiler.stopDebug(); + boolean fromCache = previousIssuesLoader.load(reactor.getRoot().getKeyWithBranch(), new SaveIssueConsumer(), false); + stopDebug(profiler, "Load server issues", fromCache); } public Iterable byComponent(BatchComponent component) { @@ -104,25 +90,55 @@ public class ServerIssueRepository { return Collections.emptyList(); } Profiler profiler = Profiler.create(LOG).startInfo("Load server issues for " + component.resource().getPath()); - final List result = new ArrayList<>(); - previousIssuesLoader.load(component.key(), new Function() { - - @Override - public Void apply(@Nullable ServerIssue issue) { - if (issue == null) { - return null; - } - result.add(issue); - return null; - } - }, true); - profiler.stopDebug(); - return result; + ServerIssueConsumer consumer = new ServerIssueConsumer(); + boolean fromCache = previousIssuesLoader.load(component.key(), consumer, true); + stopDebug(profiler, "Load server issues for " + component.resource().getPath(), fromCache); + return consumer.issueList; } else { return issuesCache.values(component.batchId()); } } + private void stopDebug(Profiler profiler, String msg, boolean fromCache) { + if (fromCache) { + profiler.stopDebug(msg + " (done from cache)"); + } else { + profiler.stopDebug(msg + " (done)"); + } + } + + private class SaveIssueConsumer implements Function { + + @Override + public Void apply(@Nullable ServerIssue issue) { + if (issue == null) { + return null; + } + String componentKey = ComponentKeys.createEffectiveKey(issue.getModuleKey(), issue.hasPath() ? issue.getPath() : null); + BatchComponent r = resourceCache.get(componentKey); + if (r == null) { + // Deleted resource + issuesCache.put(0, issue.getKey(), issue); + } else { + issuesCache.put(r.batchId(), issue.getKey(), issue); + } + return null; + } + } + + private static class ServerIssueConsumer implements Function { + List issueList = new LinkedList<>(); + + @Override + public Void apply(@Nullable ServerIssue issue) { + if (issue == null) { + return null; + } + issueList.add(issue); + return null; + } + } + public Iterable issuesOnMissingComponents() { if (analysisMode.isIncremental()) { throw new UnsupportedOperationException("Only issues of analyzed components are loaded in incremental mode"); diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java index 88690a3bfcb..09c41cf4dcb 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java +++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/BatchMediumTester.java @@ -181,7 +181,7 @@ public class BatchMediumTester { projectRefProvider.addFileData(moduleKey, path, fileData); return this; } - + public BatchMediumTesterBuilder setLastBuildDate(Date d) { projectRefProvider.setLastAnalysisDate(d); return this; @@ -292,6 +292,10 @@ public class BatchMediumTester { return rules; } + @Override + public boolean loadedFromCache() { + return false; + } } private static class FakeGlobalRepositoriesLoader implements GlobalRepositoriesLoader { @@ -325,6 +329,11 @@ public class BatchMediumTester { metricId++; return this; } + + @Override + public boolean loadedFromCache() { + return true; + } } private static class FakeProjectRepositoriesLoader implements ProjectRepositoriesLoader { @@ -351,12 +360,16 @@ public class BatchMediumTester { ref.addFileData(moduleKey, path, fileData); return this; } - + public FakeProjectRepositoriesLoader setLastAnalysisDate(Date d) { ref.setLastAnalysisDate(d); return this; } + @Override + public boolean loadedFromCache() { + return true; + } } private static class FakeServerIssuesLoader implements ServerIssuesLoader { @@ -368,13 +381,13 @@ public class BatchMediumTester { } @Override - public void load(String componentKey, Function consumer, boolean incremental) { + public boolean load(String componentKey, Function consumer, boolean incremental) { for (ServerIssue serverIssue : serverIssues) { if (!incremental || ComponentKeys.createEffectiveKey(serverIssue.getModuleKey(), serverIssue.hasPath() ? serverIssue.getPath() : null).equals(componentKey)) { consumer.apply(serverIssue); } } - + return false; } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultGlobalRepositoriesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultGlobalRepositoriesLoader.java index beb47749d6f..4d0a24c772f 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultGlobalRepositoriesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultGlobalRepositoriesLoader.java @@ -19,11 +19,13 @@ */ package org.sonar.batch.repository; -import org.sonar.batch.bootstrap.WSLoader; +import org.sonar.batch.bootstrap.AbstractServerLoader; +import org.sonar.batch.bootstrap.WSLoaderResult; +import org.sonar.batch.bootstrap.WSLoader; import org.sonar.batch.protocol.input.GlobalRepositories; -public class DefaultGlobalRepositoriesLoader implements GlobalRepositoriesLoader { +public class DefaultGlobalRepositoriesLoader extends AbstractServerLoader implements GlobalRepositoriesLoader { private static final String BATCH_GLOBAL_URL = "/batch/global"; @@ -35,7 +37,9 @@ public class DefaultGlobalRepositoriesLoader implements GlobalRepositoriesLoader @Override public GlobalRepositories load() { - return GlobalRepositories.fromJson(wsLoader.loadString(BATCH_GLOBAL_URL)); + WSLoaderResult result = wsLoader.loadString(BATCH_GLOBAL_URL); + super.loadedFromCache = result.isFromCache(); + return GlobalRepositories.fromJson(result.get()); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoader.java index 564ce0f3658..acb50733362 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoader.java @@ -19,6 +19,9 @@ */ package org.sonar.batch.repository; +import org.sonar.batch.bootstrap.AbstractServerLoader; + +import org.sonar.batch.bootstrap.WSLoaderResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.bootstrap.ProjectReactor; @@ -30,10 +33,9 @@ import org.sonar.batch.protocol.input.ProjectRepositories; import org.sonar.batch.rule.ModuleQProfiles; import org.sonar.batch.util.BatchUtils; -public class DefaultProjectRepositoriesLoader implements ProjectRepositoriesLoader { +public class DefaultProjectRepositoriesLoader extends AbstractServerLoader implements ProjectRepositoriesLoader { private static final Logger LOG = LoggerFactory.getLogger(DefaultProjectRepositoriesLoader.class); - private static final String BATCH_PROJECT_URL = "/batch/project"; private final WSLoader wsLoader; @@ -54,11 +56,17 @@ public class DefaultProjectRepositoriesLoader implements ProjectRepositoriesLoad url += "&profile=" + BatchUtils.encodeForUrl(taskProperties.properties().get(ModuleQProfiles.SONAR_PROFILE_PROP)); } url += "&preview=" + globalMode.isPreview(); - ProjectRepositories projectRepositories = ProjectRepositories.fromJson(wsLoader.loadString(url)); + ProjectRepositories projectRepositories = ProjectRepositories.fromJson(load(url)); validateProjectRepositories(projectRepositories, reactor.getRoot().getKey()); return projectRepositories; } + private String load(String resource) { + WSLoaderResult result = wsLoader.loadString(resource); + super.loadedFromCache = result.isFromCache(); + return result.get(); + } + private static void validateProjectRepositories(ProjectRepositories projectRepositories, String projectKey) { if (projectRepositories.qProfiles().isEmpty()) { throw MessageException.of("No quality profiles has been found this project, you probably don't have any language plugin suitable for this analysis."); diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java index b6d9723bb55..a7f74979682 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/DefaultServerIssuesLoader.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.repository; -import org.sonar.batch.util.BatchUtils; +import org.sonar.batch.bootstrap.WSLoaderResult; +import org.sonar.batch.util.BatchUtils; import com.google.common.io.ByteSource; import org.sonar.batch.bootstrap.WSLoader; import com.google.common.base.Function; @@ -38,9 +39,10 @@ public class DefaultServerIssuesLoader implements ServerIssuesLoader { } @Override - public void load(String componentKey, Function consumer, boolean incremental) { - ByteSource request = wsLoader.loadSource("/batch/issues?key=" + BatchUtils.encodeForUrl(componentKey)); - parseIssues(request, consumer); + public boolean load(String componentKey, Function consumer, boolean incremental) { + WSLoaderResult result = wsLoader.loadSource("/batch/issues?key=" + BatchUtils.encodeForUrl(componentKey)); + parseIssues(result.get(), consumer); + return result.isFromCache(); } private static void parseIssues(ByteSource input, Function consumer) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesLoader.java index e7f56510fdc..d0d88161c45 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesLoader.java @@ -25,4 +25,6 @@ public interface GlobalRepositoriesLoader { GlobalRepositories load(); + boolean loadedFromCache(); + } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesProvider.java b/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesProvider.java index ba8dc3c7542..3bdab15c126 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesProvider.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/GlobalRepositoriesProvider.java @@ -28,14 +28,19 @@ import org.sonar.batch.protocol.input.GlobalRepositories; public class GlobalRepositoriesProvider extends ProviderAdapter { private static final Logger LOG = Loggers.get(GlobalRepositoriesProvider.class); - + private static final String LOG_MSG = "Load global repositories"; private GlobalRepositories globalReferentials; public GlobalRepositories provide(GlobalRepositoriesLoader loader) { if (globalReferentials == null) { - Profiler profiler = Profiler.create(LOG).startInfo("Load global repositories"); + Profiler profiler = Profiler.create(LOG).startInfo(LOG_MSG); globalReferentials = loader.load(); - profiler.stopInfo(); + + if (loader.loadedFromCache()) { + profiler.stopInfo(LOG_MSG + " (done from cache)"); + } else { + profiler.stopInfo(); + } } return globalReferentials; } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesLoader.java index 1c445fe6870..1dbff7350b4 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesLoader.java @@ -26,5 +26,7 @@ import org.sonar.batch.protocol.input.ProjectRepositories; public interface ProjectRepositoriesLoader { ProjectRepositories load(ProjectReactor reactor, AnalysisProperties taskProperties); + + boolean loadedFromCache(); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesProvider.java b/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesProvider.java index 6e7bbf532e5..fe54870a567 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesProvider.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/ProjectRepositoriesProvider.java @@ -30,15 +30,22 @@ import org.sonar.batch.protocol.input.ProjectRepositories; public class ProjectRepositoriesProvider extends ProviderAdapter { + private static final String LOG_MSG = "Load project repositories"; private static final Logger LOG = Loggers.get(ProjectRepositoriesProvider.class); private ProjectRepositories projectReferentials; public ProjectRepositories provide(ProjectRepositoriesLoader loader, ProjectReactor reactor, AnalysisProperties taskProps, AnalysisMode analysisMode) { if (projectReferentials == null) { - Profiler profiler = Profiler.create(LOG).startInfo("Load project repositories"); + Profiler profiler = Profiler.create(LOG).startInfo(LOG_MSG); projectReferentials = loader.load(reactor, taskProps); - profiler.stopInfo(); + + if (loader.loadedFromCache()) { + profiler.stopInfo(LOG_MSG + " (done from cache)"); + } else { + profiler.stopInfo(); + } + if (analysisMode.isPreview() && projectReferentials.lastAnalysisDate() == null) { LOG.warn("No analysis has been found on the server for this project. All issues will be marked as 'new'."); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/ServerIssuesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/repository/ServerIssuesLoader.java index eae1d2fb3ea..250648b3bcf 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/ServerIssuesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/ServerIssuesLoader.java @@ -24,6 +24,6 @@ import org.sonar.batch.protocol.input.BatchInput.ServerIssue; public interface ServerIssuesLoader { - void load(String componentKey, Function consumer, boolean incremental); + boolean load(String componentKey, Function consumer, boolean incremental); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java b/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java index c6f1d8ae58c..a01f86d5ed7 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java +++ b/sonar-batch/src/main/java/org/sonar/batch/repository/user/UserRepository.java @@ -19,8 +19,12 @@ */ package org.sonar.batch.repository.user; -import org.sonar.batch.util.BatchUtils; +import org.sonar.batch.bootstrap.WSLoaderResult; +import org.sonar.api.utils.log.Profiler; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.batch.util.BatchUtils; import org.sonar.batch.bootstrap.WSLoader; import com.google.common.io.ByteSource; import com.google.common.base.Function; @@ -36,8 +40,8 @@ import java.util.Collections; import java.util.List; public class UserRepository { - - private WSLoader wsLoader; + private static final Logger LOG = Loggers.get(UserRepository.class); + private final WSLoader wsLoader; public UserRepository(WSLoader wsLoader) { this.wsLoader = wsLoader; @@ -47,15 +51,22 @@ public class UserRepository { if (userLogins.isEmpty()) { return Collections.emptyList(); } + Profiler profiler = Profiler.create(LOG).startDebug("Load user repository"); + WSLoaderResult result = wsLoader.loadSource("/batch/users?logins=" + Joiner.on(',').join(Lists.transform(userLogins, new UserEncodingFunction()))); + if (result.isFromCache()) { + profiler.stopInfo("Load user repository (done from cache)"); + } else { + profiler.stopInfo(); + } - ByteSource source = wsLoader.loadSource("/batch/users?logins=" + Joiner.on(',').join(Lists.transform(userLogins, new Function() { - @Override - public String apply(String input) { - return BatchUtils.encodeForUrl(input); - } - }))); + return parseUsers(result.get()); + } - return parseUsers(source); + private static class UserEncodingFunction implements Function { + @Override + public String apply(String input) { + return BatchUtils.encodeForUrl(input); + } } private static Collection parseUsers(ByteSource input) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/rule/DefaultRulesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/rule/DefaultRulesLoader.java index f9cb7c426fe..ae4492bed7b 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/rule/DefaultRulesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/rule/DefaultRulesLoader.java @@ -19,6 +19,9 @@ */ package org.sonar.batch.rule; +import org.sonar.batch.bootstrap.AbstractServerLoader; + +import org.sonar.batch.bootstrap.WSLoaderResult; import org.sonarqube.ws.Rules.ListResponse.Rule; import com.google.common.io.ByteSource; import org.sonarqube.ws.Rules.ListResponse; @@ -29,7 +32,7 @@ import java.util.List; import org.sonar.batch.bootstrap.WSLoader; -public class DefaultRulesLoader implements RulesLoader { +public class DefaultRulesLoader extends AbstractServerLoader implements RulesLoader { private static final String RULES_SEARCH_URL = "/api/rules/list"; private final WSLoader wsLoader; @@ -40,11 +43,13 @@ public class DefaultRulesLoader implements RulesLoader { @Override public List load() { - ListResponse list = loadFromSource(wsLoader.loadSource(RULES_SEARCH_URL)); + WSLoaderResult result = wsLoader.loadSource(RULES_SEARCH_URL); + ListResponse list = loadFromSource(result.get()); + super.loadedFromCache = result.isFromCache(); return list.getRulesList(); } - private ListResponse loadFromSource(ByteSource input) { + private static ListResponse loadFromSource(ByteSource input) { try (InputStream is = input.openStream()) { return ListResponse.parseFrom(is); } catch (IOException e) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/rule/RulesLoader.java b/sonar-batch/src/main/java/org/sonar/batch/rule/RulesLoader.java index a177cd72cb5..1e0c5df7cff 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/rule/RulesLoader.java +++ b/sonar-batch/src/main/java/org/sonar/batch/rule/RulesLoader.java @@ -25,4 +25,6 @@ import org.sonarqube.ws.Rules.ListResponse.Rule; public interface RulesLoader { List load(); + + boolean loadedFromCache(); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/rule/RulesProvider.java b/sonar-batch/src/main/java/org/sonar/batch/rule/RulesProvider.java index cafc1ebf65f..df0c231ed3d 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/rule/RulesProvider.java +++ b/sonar-batch/src/main/java/org/sonar/batch/rule/RulesProvider.java @@ -19,6 +19,11 @@ */ package org.sonar.batch.rule; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +import org.sonar.api.utils.log.Profiler; + import java.util.List; import org.sonarqube.ws.Rules.ListResponse.Rule; @@ -29,6 +34,8 @@ import org.sonar.api.batch.rule.internal.NewRule; import org.sonar.api.batch.rule.Rules; public class RulesProvider extends ProviderAdapter { + private static final Logger LOG = Loggers.get(RulesProvider.class); + private static final String LOG_MSG = "Load server rules"; private Rules singleton = null; public Rules provide(RulesLoader ref) { @@ -39,6 +46,7 @@ public class RulesProvider extends ProviderAdapter { } private static Rules load(RulesLoader ref) { + Profiler profiler = Profiler.create(LOG).startInfo(LOG_MSG); List loadedRules = ref.load(); RulesBuilder builder = new RulesBuilder(); @@ -48,6 +56,12 @@ public class RulesProvider extends ProviderAdapter { newRule.setInternalKey(r.getInternalKey()); } + if (ref.loadedFromCache()) { + profiler.stopInfo(LOG_MSG + " (done from cache)"); + } else { + profiler.stopInfo(); + } + return builder.build(); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginInstallerTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginInstallerTest.java index 78995ef7a74..e42962aca94 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginInstallerTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/BatchPluginInstallerTest.java @@ -45,15 +45,16 @@ public class BatchPluginInstallerTest { public ExpectedException thrown = ExpectedException.none(); FileCache fileCache = mock(FileCache.class); + ServerClient serverClient = mock(ServerClient.class); BatchPluginPredicate pluginPredicate = mock(BatchPluginPredicate.class); @Test public void listRemotePlugins() { WSLoader wsLoader = mock(WSLoader.class); - when(wsLoader.load("/deploy/plugins/index.txt")).thenReturn("checkstyle\nsqale".getBytes()); - when(wsLoader.loadString("/deploy/plugins/index.txt")).thenReturn("checkstyle\nsqale"); - BatchPluginInstaller installer = new BatchPluginInstaller(wsLoader, fileCache, pluginPredicate); + when(wsLoader.load("/deploy/plugins/index.txt")).thenReturn(new WSLoaderResult("checkstyle\nsqale".getBytes(), true)); + when(wsLoader.loadString("/deploy/plugins/index.txt")).thenReturn(new WSLoaderResult("checkstyle\nsqale", true)); + BatchPluginInstaller installer = new BatchPluginInstaller(wsLoader, serverClient, fileCache, pluginPredicate); List remotePlugins = installer.listRemotePlugins(); assertThat(remotePlugins).extracting("key").containsOnly("checkstyle", "sqale"); @@ -65,7 +66,7 @@ public class BatchPluginInstallerTest { when(fileCache.get(eq("checkstyle-plugin.jar"), eq("fakemd5_1"), any(FileCache.Downloader.class))).thenReturn(pluginJar); WSLoader wsLoader = mock(WSLoader.class); - BatchPluginInstaller installer = new BatchPluginInstaller(wsLoader, fileCache, pluginPredicate); + BatchPluginInstaller installer = new BatchPluginInstaller(wsLoader, serverClient, fileCache, pluginPredicate); RemotePlugin remote = new RemotePlugin("checkstyle").setFile("checkstyle-plugin.jar", "fakemd5_1"); File file = installer.download(remote); @@ -80,6 +81,6 @@ public class BatchPluginInstallerTest { WSLoader wsLoader = mock(WSLoader.class); doThrow(new IllegalStateException()).when(wsLoader).load("/deploy/plugins/index.txt"); - new BatchPluginInstaller(wsLoader, fileCache, pluginPredicate).installRemotes(); + new BatchPluginInstaller(wsLoader, serverClient, fileCache, pluginPredicate).installRemotes(); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/WSLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/WSLoaderTest.java index c4571c9e8e6..31c70e85d4b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/bootstrap/WSLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/bootstrap/WSLoaderTest.java @@ -74,10 +74,9 @@ public class WSLoaderTest { when(client.load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt())).thenThrow(new IllegalStateException()); WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.SERVER_FIRST); - loader.setCacheEnabled(true); - assertThat(loader.loadString(ID)).isEqualTo(cacheValue); - assertThat(loader.loadString(ID)).isEqualTo(cacheValue); + assertResult(loader.loadString(ID), cacheValue, true); + assertResult(loader.loadString(ID), cacheValue, true); // only try once the server verify(client, times(1)).load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt()); @@ -89,9 +88,8 @@ public class WSLoaderTest { when(cache.get(ID, null)).thenReturn(null); WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.CACHE_FIRST); - loader.setCacheEnabled(true); - loader.load(ID); + assertResult(loader.load(ID), serverValue.getBytes(), false); InOrder inOrder = Mockito.inOrder(client, cache); inOrder.verify(cache).get(ID, null); @@ -103,8 +101,8 @@ public class WSLoaderTest { when(client.load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt())).thenThrow(new IllegalStateException()); WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.SERVER_FIRST); - loader.setCacheEnabled(true); - assertThat(loader.loadString(ID)).isEqualTo(cacheValue); + + assertResult(loader.loadString(ID), cacheValue, true); InOrder inOrder = Mockito.inOrder(client, cache); inOrder.verify(client).load(eq(ID), anyString(), anyBoolean(), anyInt(), anyInt()); @@ -123,9 +121,9 @@ public class WSLoaderTest { public void test_throw_cache_exception_fallback() throws IOException { when(client.load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt())).thenThrow(new IllegalStateException()); when(cache.get(ID, null)).thenThrow(new NullPointerException()); + WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.SERVER_FIRST); - loader.setCacheEnabled(true); loader.load(ID); } @@ -133,9 +131,9 @@ public class WSLoaderTest { @Test(expected = IllegalStateException.class) public void test_throw_cache_exception() throws IOException { when(cache.get(ID, null)).thenThrow(new IllegalStateException()); + WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.CACHE_FIRST); - loader.setCacheEnabled(true); loader.load(ID); } @@ -144,34 +142,21 @@ public class WSLoaderTest { public void test_throw_http_exceptions() { HttpDownloader.HttpException httpException = mock(HttpDownloader.HttpException.class); IllegalStateException wrapperException = new IllegalStateException(httpException); - + when(client.load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt())).thenThrow(wrapperException); - + WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.SERVER_FIRST); - + try { loader.load(ID); - } catch(IllegalStateException e) { + } catch (IllegalStateException e) { // cache should not be used verifyNoMoreInteractions(cache); throw e; } } - @Test - public void test_server_not_accessible() throws IOException { - when(client.load(anyString(), anyString(), anyBoolean(), anyInt(), anyInt())).thenThrow(new IllegalStateException()); - WSLoader loader = new WSLoader(true, cache, client); - loader.setStrategy(LoadStrategy.SERVER_FIRST); - loader.load(ID); - loader.load(ID); - - // only try once from server - verify(client, times(1)).load(eq(ID), anyString(), anyBoolean(), anyInt(), anyInt()); - verify(cache, times(2)).get(ID, null); - } - @Test public void test_change_strategy() throws IOException { WSLoader loader = new WSLoader(true, cache, client); @@ -190,7 +175,7 @@ public class WSLoaderTest { public void test_server_strategy() throws IOException { WSLoader loader = new WSLoader(true, cache, client); loader.setStrategy(LoadStrategy.SERVER_FIRST); - loader.load(ID); + assertResult(loader.load(ID), serverValue.getBytes(), false); // should not fetch from cache verify(cache).put(ID, serverValue.getBytes()); @@ -209,6 +194,12 @@ public class WSLoaderTest { @Test public void test_string() { WSLoader loader = new WSLoader(cache, client); - assertThat(loader.loadString(ID)).isEqualTo(serverValue); + assertResult(loader.loadString(ID), serverValue, false); + } + + private void assertResult(WSLoaderResult result, T expected, boolean fromCache) { + assertThat(result).isNotNull(); + assertThat(result.get()).isEqualTo(expected); + assertThat(result.isFromCache()).isEqualTo(fromCache); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoaderTest.java index 991e61dc463..d1880ef7d3c 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/issue/tracking/DefaultServerLineHashesLoaderTest.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.issue.tracking; -import org.sonar.batch.bootstrap.WSLoader; +import org.sonar.batch.bootstrap.WSLoaderResult; +import org.sonar.batch.bootstrap.WSLoader; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -48,7 +49,7 @@ public class DefaultServerLineHashesLoaderTest { @Test public void should_download_source_from_ws_if_preview_mode() { WSLoader wsLoader = mock(WSLoader.class); - when(wsLoader.loadString(anyString())).thenReturn("ae12\n\n43fb"); + when(wsLoader.loadString(anyString())).thenReturn(new WSLoaderResult("ae12\n\n43fb", true)); ServerLineHashesLoader lastSnapshots = new DefaultServerLineHashesLoader(wsLoader); @@ -60,7 +61,7 @@ public class DefaultServerLineHashesLoaderTest { @Test public void should_download_source_with_space_from_ws_if_preview_mode() { WSLoader server = mock(WSLoader.class); - when(server.loadString(anyString())).thenReturn("ae12\n\n43fb"); + when(server.loadString(anyString())).thenReturn(new WSLoaderResult("ae12\n\n43fb", true)); ServerLineHashesLoader lastSnapshots = new DefaultServerLineHashesLoader(server); diff --git a/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoaderTest.java index 63a69e0c600..7a17f5f5e56 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultProjectRepositoriesLoaderTest.java @@ -19,8 +19,12 @@ */ package org.sonar.batch.repository; +import org.sonar.batch.bootstrap.WSLoaderResult; + import com.google.common.collect.Maps; + import java.util.Date; + import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,7 +38,6 @@ import org.sonar.batch.bootstrap.WSLoader; import org.sonar.batch.protocol.input.ProjectRepositories; import org.sonar.batch.protocol.input.QProfile; import org.sonar.batch.rule.ModuleQProfiles; - import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -58,7 +61,7 @@ public class DefaultProjectRepositoriesLoaderTest { globalMode = mock(GlobalMode.class); loader = new DefaultProjectRepositoriesLoader(wsLoader, globalMode); loader = spy(loader); - when(wsLoader.loadString(anyString())).thenReturn("{}"); + when(wsLoader.loadString(anyString())).thenReturn(new WSLoaderResult("{}", true)); taskProperties = new AnalysisProperties(Maps.newHashMap(), ""); } @@ -98,15 +101,15 @@ public class DefaultProjectRepositoriesLoaderTest { thrown.expectMessage("No quality profiles has been found this project, you probably don't have any language plugin suitable for this analysis."); reactor = new ProjectReactor(ProjectDefinition.create().setKey("foo")); - when(wsLoader.loadString(anyString())).thenReturn(new ProjectRepositories().toJson()); + when(wsLoader.loadString(anyString())).thenReturn(new WSLoaderResult(new ProjectRepositories().toJson(), true)); loader.load(reactor, taskProperties); } - private void addQualityProfile(){ + private void addQualityProfile() { ProjectRepositories projectRepositories = new ProjectRepositories(); projectRepositories.addQProfile(new QProfile("key", "name", "language", new Date())); - when(wsLoader.loadString(anyString())).thenReturn(projectRepositories.toJson()); + when(wsLoader.loadString(anyString())).thenReturn(new WSLoaderResult(projectRepositories.toJson(), true)); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultServerIssuesLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultServerIssuesLoaderTest.java index d1686d63372..3b370f6d08a 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultServerIssuesLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/repository/DefaultServerIssuesLoaderTest.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.repository; -import com.google.common.io.ByteSource; +import org.sonar.batch.bootstrap.WSLoaderResult; +import com.google.common.io.ByteSource; import org.sonar.batch.bootstrap.WSLoader; import com.google.common.base.Function; import org.junit.Before; @@ -50,7 +51,7 @@ public class DefaultServerIssuesLoaderTest { @Test public void loadFromWs() throws Exception { ByteSource bs = mock(ByteSource.class); - when(wsLoader.loadSource("/batch/issues?key=foo")).thenReturn(bs); + when(wsLoader.loadSource("/batch/issues?key=foo")).thenReturn(new WSLoaderResult(bs, true)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); diff --git a/sonar-batch/src/test/java/org/sonar/batch/repository/user/UserRepositoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/repository/user/UserRepositoryTest.java index 6910f2dc462..0057f208038 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/repository/user/UserRepositoryTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/repository/user/UserRepositoryTest.java @@ -19,8 +19,9 @@ */ package org.sonar.batch.repository.user; -import com.google.common.io.ByteSource; +import org.sonar.batch.bootstrap.WSLoaderResult; +import com.google.common.io.ByteSource; import org.sonar.batch.bootstrap.WSLoader; import org.junit.Test; import org.sonar.batch.protocol.input.BatchInput; @@ -48,7 +49,7 @@ public class UserRepositoryTest { builder.setLogin("sbrandhof").setName("Simon").build().writeDelimitedTo(out); ByteSource source = mock(ByteSource.class); - when(wsLoader.loadSource("/batch/users?logins=fmallet,sbrandhof")).thenReturn(source); + when(wsLoader.loadSource("/batch/users?logins=fmallet,sbrandhof")).thenReturn(new WSLoaderResult(source, true)); when(source.openStream()).thenReturn(new ByteArrayInputStream(out.toByteArray())); assertThat(userRepo.loadFromWs(Arrays.asList("fmallet", "sbrandhof"))).extracting("login", "name").containsOnly(tuple("fmallet", "Freddy Mallet"), tuple("sbrandhof", "Simon")); diff --git a/sonar-batch/src/test/java/org/sonar/batch/rule/DefaultRulesLoaderTest.java b/sonar-batch/src/test/java/org/sonar/batch/rule/DefaultRulesLoaderTest.java index 4f2f85e8567..0f2cc51d76d 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/rule/DefaultRulesLoaderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/rule/DefaultRulesLoaderTest.java @@ -21,6 +21,9 @@ package org.sonar.batch.rule; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + +import org.sonar.batch.bootstrap.WSLoaderResult; + import org.sonarqube.ws.Rules.ListResponse.Rule; import com.google.common.io.ByteSource; import com.google.common.io.Resources; @@ -38,9 +41,20 @@ public class DefaultRulesLoaderTest { public void testParseServerResponse() throws IOException { WSLoader wsLoader = mock(WSLoader.class); ByteSource source = Resources.asByteSource(this.getClass().getResource("DefaultRulesLoader/response.protobuf")); - when(wsLoader.loadSource(anyString())).thenReturn(source); + when(wsLoader.loadSource(anyString())).thenReturn(new WSLoaderResult(source, true)); DefaultRulesLoader loader = new DefaultRulesLoader(wsLoader); List ruleList = loader.load(); assertThat(ruleList).hasSize(318); } + + @Test + public void testLoadedFromCache() { + WSLoader wsLoader = mock(WSLoader.class); + ByteSource source = Resources.asByteSource(this.getClass().getResource("DefaultRulesLoader/response.protobuf")); + when(wsLoader.loadSource(anyString())).thenReturn(new WSLoaderResult(source, true)); + DefaultRulesLoader loader = new DefaultRulesLoader(wsLoader); + loader.load(); + + assertThat(loader.loadedFromCache()).isTrue(); + } } -- 2.39.5