From 136e364e7c851a13d84dea3864f4c66cdf09853e Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Thu, 4 Jun 2015 12:16:06 +0200 Subject: [PATCH] do not create ES indexes for every ServerTester instances --- .../java/org/sonar/server/es/BaseIndexer.java | 7 +++- .../org/sonar/server/es/IndexDefinitions.java | 20 ++++++---- .../server/search/IndexSynchronizer.java | 39 ++++++++++--------- .../activity/ws/ActivitiesWsMediumTest.java | 2 +- .../ws/HistoryActionMediumTest.java | 2 +- .../java/org/sonar/server/es/EsTester.java | 2 + .../org/sonar/server/es/IndexCreatorTest.java | 7 ++-- .../IssueBulkChangeServiceMediumTest.java | 2 +- .../issue/IssueCommentServiceMediumTest.java | 2 +- .../server/issue/IssueServiceMediumTest.java | 2 +- .../ws/SearchActionComponentsMediumTest.java | 2 +- .../issue/ws/SearchActionMediumTest.java | 2 +- .../InternalPermissionServiceMediumTest.java | 2 +- .../QProfileServiceMediumTest.java | 2 +- .../org/sonar/server/tester/ServerTester.java | 9 +++++ 15 files changed, 62 insertions(+), 40 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java index b6c994f2f12..7317c20fabe 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java @@ -36,7 +36,7 @@ public abstract class BaseIndexer implements Startable { private final ThreadPoolExecutor executor; private final String indexName, typeName, dateFieldName; protected final EsClient esClient; - private volatile long lastUpdatedAt = 0L; + private volatile long lastUpdatedAt = -1L; /** * Indexers are disabled during server startup, to avoid too many consecutive refreshes of the same index @@ -65,6 +65,9 @@ public abstract class BaseIndexer implements Startable { Future submit = executor.submit(new Runnable() { @Override public void run() { + if (lastUpdatedAt == -1L) { + lastUpdatedAt = esClient.getMaxFieldValue(indexName, typeName, dateFieldName); + } if (requestedAt > lastUpdatedAt) { long l = task.index(lastUpdatedAt); // l can be 0 if no documents were indexed @@ -98,7 +101,7 @@ public abstract class BaseIndexer implements Startable { @Override public void start() { - lastUpdatedAt = esClient.getMaxFieldValue(indexName, typeName, dateFieldName); + // nothing to do at startup } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java index 393efeab4c4..6f0186cd54f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java @@ -21,12 +21,11 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import java.util.Map; import org.elasticsearch.common.settings.Settings; import org.picocontainer.Startable; import org.sonar.api.server.ServerSide; -import java.util.Map; - /** * This class collects definitions of all Elasticsearch indices during server startup */ @@ -88,9 +87,11 @@ public class IndexDefinitions implements Startable { private final Map byKey = Maps.newHashMap(); private final IndexDefinition[] defs; + private final org.sonar.api.config.Settings settings; - public IndexDefinitions(IndexDefinition[] defs) { + public IndexDefinitions(IndexDefinition[] defs, org.sonar.api.config.Settings settings) { this.defs = defs; + this.settings = settings; } public Map getIndices() { @@ -101,12 +102,15 @@ public class IndexDefinitions implements Startable { public void start() { // collect definitions IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - for (IndexDefinition definition : defs) { - definition.define(context); - } - for (Map.Entry entry : context.getIndices().entrySet()) { - byKey.put(entry.getKey(), new Index(entry.getValue())); + if (!settings.getBoolean("sonar.internal.es.disableIndexes")) { + for (IndexDefinition definition : defs) { + definition.define(context); + } + + for (Map.Entry entry : context.getIndices().entrySet()) { + byKey.put(entry.getKey(), new Index(entry.getValue())); + } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index 8d24946a043..26758b3994e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -19,7 +19,8 @@ */ package org.sonar.server.search; - +import java.util.Date; +import org.sonar.api.config.Settings; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.core.persistence.DbSession; @@ -35,8 +36,6 @@ import org.sonar.server.test.index.TestIndexer; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.view.index.ViewIndexer; -import java.util.Date; - public class IndexSynchronizer { private static final Logger LOG = Loggers.get(IndexSynchronizer.class); @@ -50,6 +49,7 @@ public class IndexSynchronizer { private final UserIndexer userIndexer; private final ViewIndexer viewIndexer; private final ActivityIndexer activityIndexer; + private final Settings settings; /** * Limitation - {@link org.sonar.server.es.BaseIndexer} are not injected through an array or a collection @@ -57,8 +57,8 @@ public class IndexSynchronizer { * {@link org.sonar.server.issue.index.IssueIndexer} */ public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer, - TestIndexer testIndexer, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, - UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer) { + TestIndexer testIndexer, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, + UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer, Settings settings) { this.db = db; this.index = index; this.sourceLineIndexer = sourceLineIndexer; @@ -68,6 +68,7 @@ public class IndexSynchronizer { this.userIndexer = userIndexer; this.viewIndexer = viewIndexer; this.activityIndexer = activityIndexer; + this.settings = settings; } public void executeDeprecated() { @@ -82,24 +83,26 @@ public class IndexSynchronizer { } public void execute() { - LOG.info("Index activities"); - activityIndexer.setEnabled(true).index(); + if (!settings.getBoolean("sonar.internal.es.disableIndexes")) { + LOG.info("Index activities"); + activityIndexer.setEnabled(true).index(); - LOG.info("Index issues"); - issueAuthorizationIndexer.setEnabled(true).index(); - issueIndexer.setEnabled(true).index(); + LOG.info("Index issues"); + issueAuthorizationIndexer.setEnabled(true).index(); + issueIndexer.setEnabled(true).index(); - LOG.info("Index source lines"); - sourceLineIndexer.setEnabled(true).index(); + LOG.info("Index source lines"); + sourceLineIndexer.setEnabled(true).index(); - LOG.info("Index tests"); - testIndexer.setEnabled(true).index(); + LOG.info("Index tests"); + testIndexer.setEnabled(true).index(); - LOG.info("Index users"); - userIndexer.setEnabled(true).index(); + LOG.info("Index users"); + userIndexer.setEnabled(true).index(); - LOG.info("Index views"); - viewIndexer.setEnabled(true).index(); + LOG.info("Index views"); + viewIndexer.setEnabled(true).index(); + } } void synchronize(DbSession session, Dao dao, Index index) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/activity/ws/ActivitiesWsMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/activity/ws/ActivitiesWsMediumTest.java index 25a4e53be94..91cf4c730ae 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/activity/ws/ActivitiesWsMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/activity/ws/ActivitiesWsMediumTest.java @@ -35,7 +35,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class ActivitiesWsMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/ws/HistoryActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/ws/HistoryActionMediumTest.java index 9c146fbba9a..42a93b83300 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/ws/HistoryActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/ws/HistoryActionMediumTest.java @@ -41,7 +41,7 @@ import java.util.Date; public class HistoryActionMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index e8df89bb003..2e1345d370a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -41,6 +41,7 @@ import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.search.SearchHit; import org.junit.rules.ExternalResource; +import org.sonar.api.config.Settings; import org.sonar.core.platform.ComponentContainer; import org.sonar.server.search.BaseDoc; import org.sonar.test.TestUtils; @@ -101,6 +102,7 @@ public class EsTester extends ExternalResource { if (!definitions.isEmpty()) { ComponentContainer container = new ComponentContainer(); + container.addSingleton(new Settings()); container.addSingletons(definitions); container.addSingleton(client); container.addSingleton(IndexDefinitions.class); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index ac55e4e4c53..a3a1920fecd 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -29,6 +29,7 @@ import javax.annotation.CheckForNull; import java.io.IOException; import java.util.Map; +import org.sonar.api.config.Settings; import static org.assertj.core.api.Assertions.assertThat; @@ -41,7 +42,7 @@ public class IndexCreatorTest { public void create_index() throws Exception { assertThat(mappings()).isEmpty(); - IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}); + IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}, new Settings()); registry.start(); IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); @@ -66,7 +67,7 @@ public class IndexCreatorTest { assertThat(mappings()).isEmpty(); // v1 - IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}); + IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}, new Settings()); registry.start(); IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); @@ -75,7 +76,7 @@ public class IndexCreatorTest { assertThat(hashV1).isNotEmpty(); // v2 - registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinitionV2()}); + registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinitionV2()}, new Settings()); registry.start(); creator = new IndexCreator(es.client(), registry); creator.start(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java index 6e01a4b3c9e..992df6249f9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueBulkChangeServiceMediumTest.java @@ -62,7 +62,7 @@ import static org.junit.Assert.fail; public class IssueBulkChangeServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java index 7942227af2d..a343a039412 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueCommentServiceMediumTest.java @@ -58,7 +58,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class IssueCommentServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index 24ae581f6fb..60092ddb04f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -79,7 +79,7 @@ import static org.junit.Assert.fail; public class IssueServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java index 18ac9cd2a4a..4c90e46023c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionComponentsMediumTest.java @@ -59,7 +59,7 @@ import static com.google.common.collect.Lists.newArrayList; public class SearchActionComponentsMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java index 4cf591bc187..791433a9e33 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/SearchActionMediumTest.java @@ -63,7 +63,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class SearchActionMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java index 6c369d534a9..2ba6021d607 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java @@ -52,7 +52,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class InternalPermissionServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks(); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes(); @Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileServiceMediumTest.java index 98e82dafb36..ff6118371a1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileServiceMediumTest.java @@ -64,7 +64,7 @@ import static org.sonar.server.qualityprofile.QProfileTesting.XOO_P2_KEY; public class QProfileServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester().withStartupTasks().addComponents(XooProfileImporter.class, XooExporter.class); + public static ServerTester tester = new ServerTester().withStartupTasks().withEsIndexes().addComponents(XooProfileImporter.class, XooExporter.class); @org.junit.Rule public UserSessionRule userSessionRule = UserSessionRule.forServerTester(tester); diff --git a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java index 6c5506ca867..a465a1b0151 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java @@ -70,12 +70,18 @@ public class ServerTester extends ExternalResource { private final ServletContext servletContext = new AttributeHolderServletContext(); private URL updateCenterUrl; private boolean startupTasks = false; + private boolean esIndexes = false; public ServerTester withStartupTasks() { this.startupTasks = true; return this; } + public ServerTester withEsIndexes() { + this.esIndexes = true; + return this; + } + /** * Called only when JUnit @Rule or @ClassRule is used. */ @@ -111,6 +117,9 @@ public class ServerTester extends ExternalResource { properties.put(StringUtils.substringAfter(key, PROP_PREFIX), entry.getValue()); } } + if (!esIndexes) { + properties.put("sonar.internal.es.disableIndexes", true); + } platform = new ServerTesterPlatform(); platform.init(properties, servletContext); platform.addComponents(components); -- 2.39.5