@Override
public void define(IndexDefinitionContext context) {
NewIndex index = context.create(INDEX);
- // refresh is always handled by ActivityIndexer
index.getSettings().put("index.refresh_interval", "-1");
index.getSettings().put("analysis.analyzer.default.type", "keyword");
*/
package org.sonar.server.activity.index;
-import org.elasticsearch.action.support.replication.ReplicationType;
import org.elasticsearch.action.update.UpdateRequest;
import org.sonar.core.persistence.DbSession;
import org.sonar.server.db.DbClient;
* Add to Elasticsearch index {@link org.sonar.server.activity.index.ActivityIndexDefinition} the rows of
* db table ACTIVITIES that are not indexed yet
* <p/>
- * TODO idea of improvement - index asynchronously with UpdateRequest#replicationType(ReplicationType.ASYNC)
*/
public class ActivityIndexer extends BaseIndexer {
@Override
protected long doIndex(long lastUpdatedAt) {
- final BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX);
+ BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX);
bulk.setLarge(lastUpdatedAt == 0L);
DbSession dbSession = dbClient.openSession(false);
}
public long index(Iterator<ActivityDoc> activities) {
- final BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX);
+ BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX);
return doIndex(bulk, activities);
}
private UpdateRequest newUpsertRequest(ActivityDoc doc) {
return new UpdateRequest(ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE, doc.getKey())
.doc(doc.getFields())
- .upsert(doc.getFields())
- .replicationType(ReplicationType.ASYNC);
+ .upsert(doc.getFields());
}
}
protected final EsClient esClient;
private volatile long lastUpdatedAt = 0L;
+ /**
+ * Indexers are disabled during server startup, to avoid too many consecutive refreshes of the same index
+ * An example is RegisterQualityProfiles. If {@link org.sonar.server.activity.index.ActivityIndexer} is enabled by
+ * default during startup, then each new activated rule generates a bulk request with a single document and then
+ * asks for index refresh -> big performance hit.
+ *
+ * Indices are populated and refreshed when all startup components have been executed. See
+ * {@link org.sonar.server.search.IndexSynchronizer}
+ */
+ private boolean enabled = false;
+
protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName) {
this.indexName = indexName;
this.typeName = typeName;
}
public void index() {
- final long requestedAt = System.currentTimeMillis();
- Future submit = executor.submit(new Runnable() {
- @Override
- public void run() {
- if (requestedAt > lastUpdatedAt) {
- long l = doIndex(lastUpdatedAt);
- // l can be 0 if no documents were indexed
- lastUpdatedAt = Math.max(l, lastUpdatedAt);
+ if (enabled) {
+ final long requestedAt = System.currentTimeMillis();
+ Future submit = executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ if (requestedAt > lastUpdatedAt) {
+ long l = doIndex(lastUpdatedAt);
+ // l can be 0 if no documents were indexed
+ lastUpdatedAt = Math.max(l, lastUpdatedAt);
+ }
}
+ });
+ try {
+ Uninterruptibles.getUninterruptibly(submit);
+ } catch (ExecutionException e) {
+ Throwables.propagate(e);
}
- });
- try {
- Uninterruptibles.getUninterruptibly(submit);
- } catch (ExecutionException e) {
- Throwables.propagate(e);
}
}
protected abstract long doIndex(long lastUpdatedAt);
+ public BaseIndexer setEnabled(boolean b) {
+ this.enabled = b;
+ return this;
+ }
+
@Override
public void start() {
lastUpdatedAt = esClient.getLastUpdatedAt(indexName, typeName);
private static final String SETTING_HASH = "sonar_hash";
private final EsClient client;
- private final IndexRegistry registry;
+ private final IndexDefinitions definitions;
- public IndexCreator(EsClient client, IndexRegistry registry) {
+ public IndexCreator(EsClient client, IndexDefinitions definitions) {
this.client = client;
- this.registry = registry;
+ this.definitions = definitions;
}
@Override
public void start() {
// create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...)
- for (IndexRegistry.Index index : registry.getIndices().values()) {
+ for (IndexDefinitions.Index index : definitions.getIndices().values()) {
boolean exists = client.prepareIndicesExist(index.getName()).get().isExists();
if (exists && needsToDeleteIndex(index)) {
LOGGER.info(String.format("Delete index %s (settings changed)", index.getName()));
// nothing to do
}
- private void createIndex(IndexRegistry.Index index) {
+ private void createIndex(IndexDefinitions.Index index) {
LOGGER.info(String.format("Create index %s", index.getName()));
ImmutableSettings.Builder settings = ImmutableSettings.builder();
settings.put(index.getSettings());
client.waitForStatus(ClusterHealthStatus.YELLOW);
// create types
- for (Map.Entry<String, IndexRegistry.IndexType> entry : index.getTypes().entrySet()) {
+ for (Map.Entry<String, IndexDefinitions.IndexType> entry : index.getTypes().entrySet()) {
LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey()));
PutMappingResponse mappingResponse = client.preparePutMapping(index.getName())
.setType(entry.getKey())
client.nativeClient().admin().indices().prepareDelete(indexName).get();
}
- private boolean needsToDeleteIndex(IndexRegistry.Index index) {
+ private boolean needsToDeleteIndex(IndexDefinitions.Index index) {
boolean toBeDeleted = false;
String hash = client.nativeClient().admin().indices().prepareGetSettings(index.getName()).get().getSetting(index.getName(), "index." + SETTING_HASH);
if (hash != null) {
private static final char DELIMITER = ',';
- String of(IndexRegistry.Index index) {
+ String of(IndexDefinitions.Index index) {
return of(index.getSettings().getAsMap(), index.getTypes());
}
}
private void appendObject(StringBuilder sb, Object value) {
- if (value instanceof IndexRegistry.IndexType) {
- appendIndexType(sb, (IndexRegistry.IndexType) value);
+ if (value instanceof IndexDefinitions.IndexType) {
+ appendIndexType(sb, (IndexDefinitions.IndexType) value);
} else if (value instanceof Map) {
appendMap(sb, (Map) value);
} else if (value instanceof Iterable) {
}
}
- private void appendIndexType(StringBuilder sb, IndexRegistry.IndexType type) {
+ private void appendIndexType(StringBuilder sb, IndexDefinitions.IndexType type) {
appendMap(sb, type.getAttributes());
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.elasticsearch.common.settings.Settings;
+import org.picocontainer.Startable;
+import org.sonar.api.ServerComponent;
+
+import java.util.Map;
+
+/**
+ * This class collects definitions of all Elasticsearch indices during server startup
+ */
+public class IndexDefinitions implements ServerComponent, Startable {
+
+ /**
+ * Immutable copy of {@link org.sonar.server.es.NewIndex}
+ */
+ public static class Index {
+ private final String name;
+ private final Settings settings;
+ private final Map<String, IndexType> types;
+
+ Index(NewIndex newIndex) {
+ this.name = newIndex.getName();
+ this.settings = newIndex.getSettings().build();
+ ImmutableMap.Builder<String, IndexType> builder = ImmutableMap.builder();
+ for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) {
+ IndexType type = new IndexType(newIndexType);
+ builder.put(type.getName(), type);
+ }
+ this.types = builder.build();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Settings getSettings() {
+ return settings;
+ }
+
+ public Map<String, IndexType> getTypes() {
+ return types;
+ }
+ }
+
+ /**
+ * Immutable copy of {@link org.sonar.server.es.NewIndex.NewIndexType}
+ */
+ public static class IndexType {
+ private final String name;
+ private final Map<String, Object> attributes;
+
+ private IndexType(NewIndex.NewIndexType newType) {
+ this.name = newType.getName();
+ this.attributes = ImmutableMap.copyOf(newType.getAttributes());
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+ }
+
+ private final Map<String, Index> byKey = Maps.newHashMap();
+ private final IndexDefinition[] defs;
+
+ public IndexDefinitions(IndexDefinition[] defs) {
+ this.defs = defs;
+ }
+
+ public Map<String, Index> getIndices() {
+ return byKey;
+ }
+
+ @Override
+ public void start() {
+ // collect definitions
+ IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
+ for (IndexDefinition definition : defs) {
+ definition.define(context);
+ }
+
+ for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) {
+ byKey.put(entry.getKey(), new Index(entry.getValue()));
+ }
+ }
+
+ @Override
+ public void stop() {
+ // nothing to do
+ }
+}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import org.elasticsearch.common.settings.Settings;
-import org.picocontainer.Startable;
-import org.sonar.api.ServerComponent;
-
-import java.util.Map;
-
-/**
- * This class collects definitions of all Elasticsearch indices during server startup
- */
-public class IndexRegistry implements ServerComponent, Startable {
-
- /**
- * Immutable copy of {@link org.sonar.server.es.NewIndex}
- */
- public static class Index {
- private final String name;
- private final Settings settings;
- private final Map<String, IndexType> types;
-
- Index(NewIndex newIndex) {
- this.name = newIndex.getName();
- this.settings = newIndex.getSettings().build();
- ImmutableMap.Builder<String, IndexType> builder = ImmutableMap.builder();
- for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) {
- IndexType type = new IndexType(newIndexType);
- builder.put(type.getName(), type);
- }
- this.types = builder.build();
- }
-
- public String getName() {
- return name;
- }
-
- public Settings getSettings() {
- return settings;
- }
-
- public Map<String, IndexType> getTypes() {
- return types;
- }
- }
-
- /**
- * Immutable copy of {@link org.sonar.server.es.NewIndex.NewIndexType}
- */
- public static class IndexType {
- private final String name;
- private final Map<String, Object> attributes;
-
- private IndexType(NewIndex.NewIndexType newType) {
- this.name = newType.getName();
- this.attributes = ImmutableMap.copyOf(newType.getAttributes());
- }
-
- public String getName() {
- return name;
- }
-
- public Map<String, Object> getAttributes() {
- return attributes;
- }
- }
-
- private final Map<String, Index> byKey = Maps.newHashMap();
- private final IndexDefinition[] defs;
-
- public IndexRegistry(IndexDefinition[] defs) {
- this.defs = defs;
- }
-
- public Map<String, Index> getIndices() {
- return byKey;
- }
-
- @Override
- public void start() {
- // collect definitions
- IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
- for (IndexDefinition definition : defs) {
- definition.define(context);
- }
-
- for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) {
- byKey.put(entry.getKey(), new Index(entry.getValue()));
- }
- }
-
- @Override
- public void stop() {
- // nothing to do
- }
-}
import org.sonar.server.duplication.ws.DuplicationsWs;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexCreator;
-import org.sonar.server.es.IndexRegistry;
+import org.sonar.server.es.IndexDefinitions;
import org.sonar.server.issue.ActionService;
import org.sonar.server.issue.AddTagsAction;
import org.sonar.server.issue.AssignAction;
pico.addSingleton(Periods.class);
pico.addSingleton(ServerWs.class);
pico.addSingleton(BackendCleanup.class);
- pico.addSingleton(IndexRegistry.class);
+ pico.addSingleton(IndexDefinitions.class);
pico.addSingleton(IndexCreator.class);
// Activity
DoPrivileged.execute(new DoPrivileged.Task() {
@Override
protected void doPrivileged() {
- startupContainer.getComponentsByType(IndexSynchronizer.class).get(0).execute();
startupContainer.startComponents();
+ startupContainer.getComponentByType(IndexSynchronizer.class).execute();
startupContainer.getComponentByType(ServerLifecycleNotifier.class).notifyStart();
}
});
private final ViewIndexer viewIndexer;
private final ActivityIndexer activityIndexer;
+ /**
+ * Limitation - {@link org.sonar.server.es.BaseIndexer} are not injected through an array or a collection
+ * because we need {@link org.sonar.server.issue.index.IssueAuthorizationIndexer} to be executed before
+ * {@link org.sonar.server.issue.index.IssueIndexer}
+ */
public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer,
IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer,
UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer) {
}
LOG.info("Index activities");
- activityIndexer.index();
+ activityIndexer.setEnabled(true).index();
LOG.info("Index issues");
- issueAuthorizationIndexer.index();
- issueIndexer.index();
+ issueAuthorizationIndexer.setEnabled(true).index();
+ issueIndexer.setEnabled(true).index();
- LOG.info("Index source files");
- sourceLineIndexer.index();
+ LOG.info("Index source lines");
+ sourceLineIndexer.setEnabled(true).index();
LOG.info("Index users");
- userIndexer.index();
+ userIndexer.setEnabled(true).index();
LOG.info("Index views");
- viewIndexer.index();
+ viewIndexer.setEnabled(true).index();
}
void synchronize(DbSession session, Dao dao, Index index) {
long count = index.getIndexStat().getDocumentCount();
Date lastSynch = index.getLastSynchronization();
+ LOG.info("Index {}s", index.getIndexType());
if (count <= 0) {
- LOG.info("Index {}s", index.getIndexType());
dao.synchronizeAfter(session);
} else {
- LOG.info("Index {}s for updates after {}", index.getIndexType(), lastSynch);
dao.synchronizeAfter(session, lastSynch);
}
}
ActivityDao activityDao = new ActivityDao(db.myBatis(), system);
IssueDao issueDao = new IssueDao(db.myBatis());
DbClient dbClient = new DbClient(db.database(), db.myBatis(), issueDao, activityDao);
- service = new ActivityService(dbClient, new ActivityIndexer(dbClient, es.client()));
+ ActivityIndexer indexer = new ActivityIndexer(dbClient, es.client());
+ // indexers are disabled by default
+ indexer.setEnabled(true);
+ service = new ActivityService(dbClient, indexer);
}
@Test
ComponentContainer container = new ComponentContainer();
container.addSingletons(definitions);
container.addSingleton(client);
- container.addSingleton(IndexRegistry.class);
+ container.addSingleton(IndexDefinitions.class);
container.addSingleton(IndexCreator.class);
container.startComponents();
}
public void create_index() throws Exception {
assertThat(mappings()).isEmpty();
- IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()});
+ IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()});
registry.start();
IndexCreator creator = new IndexCreator(es.client(), registry);
creator.start();
assertThat(mappings()).isEmpty();
// v1
- IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()});
+ IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()});
registry.start();
IndexCreator creator = new IndexCreator(es.client(), registry);
creator.start();
assertThat(hashV1).isNotEmpty();
// v2
- registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinitionV2()});
+ registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinitionV2()});
registry.start();
creator = new IndexCreator(es.client(), registry);
creator.start();
@Test
public void of() throws Exception {
- IndexRegistry.Index indexV1 = new IndexRegistry.Index(createIndex());
+ IndexDefinitions.Index indexV1 = new IndexDefinitions.Index(createIndex());
String hashV1 = new IndexDefinitionHash().of(indexV1);
assertThat(hashV1).isNotEmpty();
// always the same
NewIndex newIndexV2 = createIndex();
newIndexV2.getTypes().get("fake").createIntegerField("max");
- String hashV2 = new IndexDefinitionHash().of(new IndexRegistry.Index(newIndexV2));
+ String hashV2 = new IndexDefinitionHash().of(new IndexDefinitions.Index(newIndexV2));
assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1);
}
assertThat(esTester.countDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0L);
}
+ @Test
+ public void index_nothing_if_disabled() throws Exception {
+ dbTester.prepareDbUnit(getClass(), "index.xml");
+
+ createIndexer().setEnabled(false).index();
+
+ assertThat(esTester.countDocuments("issues", "issue")).isEqualTo(0);
+ }
+
@Test
public void index() throws Exception {
dbTester.prepareDbUnit(getClass(), "index.xml");
}
private IssueIndexer createIndexer() {
- return new IssueIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client());
+ IssueIndexer indexer = new IssueIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client());
+ indexer.setEnabled(true);
+ return indexer;
}
}
es.truncateIndices();
db.truncateTables();
indexer = new SourceLineIndexer(new DbClient(db.database(), db.myBatis()), es.client());
+ indexer.setEnabled(true);
}
@Test
assertThat(doc.updatedAt()).isEqualTo(1500000000000L);
}
+ @Test
+ public void do_nothing_if_disabled() throws Exception {
+ dbTester.prepareDbUnit(getClass(), "index.xml");
+
+ createIndexer().setEnabled(false).index();
+ assertThat(esTester.countDocuments("users", "user")).isEqualTo(0);
+ }
+
private UserIndexer createIndexer() {
- return new UserIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client());
+ UserIndexer indexer = new UserIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client());
+ indexer.setEnabled(true);
+ return indexer;
}
}
dbTester.truncateTables();
esTester.truncateIndices();
indexer = new ViewIndexer(new DbClient(dbTester.database(), dbTester.myBatis(), new ComponentDao()), esTester.client());
+ indexer.setEnabled(true);
}
@Test
require_parameters 'key'
@profile = Internal.qprofile_loader.getByKey(params[:key])
+ not_found('Quality profile does not exist') unless @profile
search = {'profileKey' => @profile.key().to_s, 'since' => params[:since], 'to' => params[:to], 'p' => params[:p]}
result = Internal.component(Java::OrgSonarServerActivity::RubyQProfileActivityService.java_class).search(search)
@changes = result.activities
%>
<tr class="<%= cycle('even', 'odd') -%>">
<%
- action = change.action()
+ action = change.getAction()
action_message = message('quality_profiles.changelog.' + action.downcase) if action
if change.authorName() && !change.authorName().empty?()
author = change.authorName()
- elsif change.login() && !change.login().empty?()
- author = change.login()
+ elsif change.login() && !change.getLogin().empty?()
+ author = change.getLogin()
else
author = 'System'
end
rule = change.ruleName() ? change.ruleName() : change.ruleKey()
%>
- <td valign="top" width="1%" nowrap><%= Internal.i18n.formatDateTime(change.time()) -%></td>
+ <td valign="top" width="1%" nowrap><%= Internal.i18n.formatDateTime(change.getCreatedAt()) -%></td>
<td valign="top" width="1%" nowrap><%= author %></td>
<td valign="top" width="1%" nowrap><%= action_message %></td>
<td valign="top"><%= rule %></td>
projects / sonarqube.git / commitdiff