diff options
33 files changed, 143 insertions, 104 deletions
@@ -74,7 +74,7 @@ <logback.version>1.1.2</logback.version> <slf4j.version>1.7.10</slf4j.version> <tomcat.version>8.0.18</tomcat.version> - <elasticsearch.version>1.4.2</elasticsearch.version> + <elasticsearch.version>1.4.4</elasticsearch.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.min.version>3.0.5</maven.min.version> <maven.api.version>3.0.5</maven.api.version> diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java index 9fd5756c51b..bac57c04233 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java @@ -50,7 +50,6 @@ public class ActivityIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - // refresh is always handled by ActivityIndexer index.getSettings().put("index.refresh_interval", "-1"); index.getSettings().put("analysis.analyzer.default.type", "keyword"); diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java index 1a668bd5a8f..babf3f671bd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java @@ -33,20 +33,19 @@ import java.util.Iterator; * Add to Elasticsearch index {@link org.sonar.server.activity.index.ActivityIndexDefinition} the rows of * db table ACTIVITIES that are not indexed yet * <p/> - * TODO idea of improvement - index asynchronously with UpdateRequest#replicationType(ReplicationType.ASYNC) */ public class ActivityIndexer extends BaseIndexer { private final DbClient dbClient; public ActivityIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE); + super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE, ActivityIndexDefinition.FIELD_CREATED_AT); this.dbClient = dbClient; } @Override protected long doIndex(long lastUpdatedAt) { - final BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX); + BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX); bulk.setLarge(lastUpdatedAt == 0L); DbSession dbSession = dbClient.openSession(false); @@ -63,7 +62,7 @@ public class ActivityIndexer extends BaseIndexer { } public long index(Iterator<ActivityDoc> activities) { - final BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX); + BulkIndexer bulk = new BulkIndexer(esClient, ActivityIndexDefinition.INDEX); return doIndex(bulk, activities); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java index 5f88f181b3c..39a0f443e89 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java @@ -33,42 +33,62 @@ import java.util.concurrent.TimeUnit; public abstract class BaseIndexer implements ServerComponent, Startable { private final ThreadPoolExecutor executor; - private final String indexName, typeName; + private final String indexName, typeName, dateFieldName; protected final EsClient esClient; private volatile long lastUpdatedAt = 0L; - protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName) { + /** + * Indexers are disabled during server startup, to avoid too many consecutive refreshes of the same index + * An example is RegisterQualityProfiles. If {@link org.sonar.server.activity.index.ActivityIndexer} is enabled by + * default during startup, then each new activated rule generates a bulk request with a single document and then + * asks for index refresh -> big performance hit. + * + * Indices are populated and refreshed when all startup components have been executed. See + * {@link org.sonar.server.search.IndexSynchronizer} + */ + private boolean enabled = false; + + protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName, + String dateFieldName) { this.indexName = indexName; this.typeName = typeName; + this.dateFieldName = dateFieldName; this.esClient = client; this.executor = new ThreadPoolExecutor(0, 1, threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); } public void index() { - final long requestedAt = System.currentTimeMillis(); - Future submit = executor.submit(new Runnable() { - @Override - public void run() { - if (requestedAt > lastUpdatedAt) { - long l = doIndex(lastUpdatedAt); - // l can be 0 if no documents were indexed - lastUpdatedAt = Math.max(l, lastUpdatedAt); + if (enabled) { + final long requestedAt = System.currentTimeMillis(); + Future submit = executor.submit(new Runnable() { + @Override + public void run() { + if (requestedAt > lastUpdatedAt) { + long l = doIndex(lastUpdatedAt); + // l can be 0 if no documents were indexed + lastUpdatedAt = Math.max(l, lastUpdatedAt); + } } + }); + try { + Uninterruptibles.getUninterruptibly(submit); + } catch (ExecutionException e) { + Throwables.propagate(e); } - }); - try { - Uninterruptibles.getUninterruptibly(submit); - } catch (ExecutionException e) { - Throwables.propagate(e); } } protected abstract long doIndex(long lastUpdatedAt); + public BaseIndexer setEnabled(boolean b) { + this.enabled = b; + return this; + } + @Override public void start() { - lastUpdatedAt = esClient.getLastUpdatedAt(indexName, typeName); + lastUpdatedAt = esClient.getMaxFieldValue(indexName, typeName, dateFieldName); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 28384f0f074..fd6f3fec50c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -57,7 +57,6 @@ public class BulkIndexer implements Startable { private final EsClient client; private final String indexName; private boolean large = false; - private boolean refresh = true; private long flushByteSize = FLUSH_BYTE_SIZE; private BulkRequestBuilder bulkRequest = null; private Map<String, Object> largeInitialSettings = null; @@ -83,12 +82,6 @@ public class BulkIndexer implements Startable { return this; } - public BulkIndexer setRefresh(boolean b) { - Preconditions.checkState(bulkRequest == null, ALREADY_STARTED_MESSAGE); - this.refresh = b; - return this; - } - /** * Default value is {@link org.sonar.server.es.BulkIndexer#FLUSH_BYTE_SIZE} * @see org.elasticsearch.common.unit.ByteSizeValue @@ -144,9 +137,7 @@ public class BulkIndexer implements Startable { progress.stop(); } - if (refresh) { - client.prepareRefresh(indexName).get(); - } + client.prepareRefresh(indexName).get(); if (large) { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 5c252935bc4..61e61608210 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -183,12 +183,12 @@ public class EsClient implements Startable { return new ProxyClearCacheRequestBuilder(client).setIndices(indices); } - public long getLastUpdatedAt(String indexName, String typeName) { + public long getMaxFieldValue(String indexName, String typeName, String fieldName) { SearchRequestBuilder request = prepareSearch(indexName) .setTypes(typeName) .setQuery(QueryBuilders.matchAllQuery()) .setSize(0) - .addAggregation(AggregationBuilders.max("latest").field("updatedAt")); + .addAggregation(AggregationBuilders.max("latest").field(fieldName)); Max max = request.get().getAggregations().get("latest"); return (long) max.getValue(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java index 909fd00bcba..1ef1d9ce3b8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java @@ -45,17 +45,17 @@ public class IndexCreator implements ServerComponent, Startable { private static final String SETTING_HASH = "sonar_hash"; private final EsClient client; - private final IndexRegistry registry; + private final IndexDefinitions definitions; - public IndexCreator(EsClient client, IndexRegistry registry) { + public IndexCreator(EsClient client, IndexDefinitions definitions) { this.client = client; - this.registry = registry; + this.definitions = definitions; } @Override public void start() { // create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...) - for (IndexRegistry.Index index : registry.getIndices().values()) { + for (IndexDefinitions.Index index : definitions.getIndices().values()) { boolean exists = client.prepareIndicesExist(index.getName()).get().isExists(); if (exists && needsToDeleteIndex(index)) { LOGGER.info(String.format("Delete index %s (settings changed)", index.getName())); @@ -73,7 +73,7 @@ public class IndexCreator implements ServerComponent, Startable { // nothing to do } - private void createIndex(IndexRegistry.Index index) { + private void createIndex(IndexDefinitions.Index index) { LOGGER.info(String.format("Create index %s", index.getName())); ImmutableSettings.Builder settings = ImmutableSettings.builder(); settings.put(index.getSettings()); @@ -88,7 +88,7 @@ public class IndexCreator implements ServerComponent, Startable { client.waitForStatus(ClusterHealthStatus.YELLOW); // create types - for (Map.Entry<String, IndexRegistry.IndexType> entry : index.getTypes().entrySet()) { + for (Map.Entry<String, IndexDefinitions.IndexType> entry : index.getTypes().entrySet()) { LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey())); PutMappingResponse mappingResponse = client.preparePutMapping(index.getName()) .setType(entry.getKey()) @@ -106,7 +106,7 @@ public class IndexCreator implements ServerComponent, Startable { client.nativeClient().admin().indices().prepareDelete(indexName).get(); } - private boolean needsToDeleteIndex(IndexRegistry.Index index) { + private boolean needsToDeleteIndex(IndexDefinitions.Index index) { boolean toBeDeleted = false; String hash = client.nativeClient().admin().indices().prepareGetSettings(index.getName()).get().getSetting(index.getName(), "index." + SETTING_HASH); if (hash != null) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitionHash.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitionHash.java index 6df309abaa2..6b2c208d022 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitionHash.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitionHash.java @@ -37,7 +37,7 @@ class IndexDefinitionHash { private static final char DELIMITER = ','; - String of(IndexRegistry.Index index) { + String of(IndexDefinitions.Index index) { return of(index.getSettings().getAsMap(), index.getTypes()); } @@ -50,8 +50,8 @@ class IndexDefinitionHash { } private void appendObject(StringBuilder sb, Object value) { - if (value instanceof IndexRegistry.IndexType) { - appendIndexType(sb, (IndexRegistry.IndexType) value); + if (value instanceof IndexDefinitions.IndexType) { + appendIndexType(sb, (IndexDefinitions.IndexType) value); } else if (value instanceof Map) { appendMap(sb, (Map) value); } else if (value instanceof Iterable) { @@ -61,7 +61,7 @@ class IndexDefinitionHash { } } - private void appendIndexType(StringBuilder sb, IndexRegistry.IndexType type) { + private void appendIndexType(StringBuilder sb, IndexDefinitions.IndexType type) { appendMap(sb, type.getAttributes()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java index 972830067be..771f9b8ed31 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexRegistry.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java @@ -30,7 +30,7 @@ import java.util.Map; /** * This class collects definitions of all Elasticsearch indices during server startup */ -public class IndexRegistry implements ServerComponent, Startable { +public class IndexDefinitions implements ServerComponent, Startable { /** * Immutable copy of {@link org.sonar.server.es.NewIndex} @@ -88,7 +88,7 @@ public class IndexRegistry implements ServerComponent, Startable { private final Map<String, Index> byKey = Maps.newHashMap(); private final IndexDefinition[] defs; - public IndexRegistry(IndexDefinition[] defs) { + public IndexDefinitions(IndexDefinition[] defs) { this.defs = defs; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java index 04bf850b395..0a8c3efd3f4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java @@ -47,7 +47,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer { private final DbClient dbClient; public IssueAuthorizationIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION); + super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index f705776bed4..68292b43a93 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -93,6 +93,9 @@ public class IssueIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); + // refresh is handled by IssueIndexer + index.getSettings().put("index.refresh_interval", "-1"); + // shards boolean clusterMode = settings.getBoolean(ProcessConstants.CLUSTER_ACTIVATE); if (clusterMode) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index e43df41dd98..011bcc1d7de 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -38,7 +38,7 @@ public class IssueIndexer extends BaseIndexer { private final DbClient dbClient; public IssueIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE); + super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index dd409d3e8ae..66c8d2ff0ec 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -136,7 +136,7 @@ import org.sonar.server.duplication.ws.DuplicationsParser; import org.sonar.server.duplication.ws.DuplicationsWs; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexCreator; -import org.sonar.server.es.IndexRegistry; +import org.sonar.server.es.IndexDefinitions; import org.sonar.server.issue.ActionService; import org.sonar.server.issue.AddTagsAction; import org.sonar.server.issue.AssignAction; @@ -485,7 +485,7 @@ class ServerComponents { pico.addSingleton(Periods.class); pico.addSingleton(ServerWs.class); pico.addSingleton(BackendCleanup.class); - pico.addSingleton(IndexRegistry.class); + pico.addSingleton(IndexDefinitions.class); pico.addSingleton(IndexCreator.class); // Activity @@ -847,8 +847,8 @@ class ServerComponents { DoPrivileged.execute(new DoPrivileged.Task() { @Override protected void doPrivileged() { - startupContainer.getComponentsByType(IndexSynchronizer.class).get(0).execute(); startupContainer.startComponents(); + startupContainer.getComponentByType(IndexSynchronizer.class).execute(); startupContainer.getComponentByType(ServerLifecycleNotifier.class).notifyStart(); } }); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java index b4499a4d20c..6e0a4dca98a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java @@ -49,6 +49,11 @@ public class IndexSynchronizer { private final ViewIndexer viewIndexer; private final ActivityIndexer activityIndexer; + /** + * Limitation - {@link org.sonar.server.es.BaseIndexer} are not injected through an array or a collection + * because we need {@link org.sonar.server.issue.index.IssueAuthorizationIndexer} to be executed before + * {@link org.sonar.server.issue.index.IssueIndexer} + */ public IndexSynchronizer(DbClient db, IndexClient index, SourceLineIndexer sourceLineIndexer, IssueAuthorizationIndexer issueAuthorizationIndexer, IssueIndexer issueIndexer, UserIndexer userIndexer, ViewIndexer viewIndexer, ActivityIndexer activityIndexer) { @@ -73,30 +78,29 @@ public class IndexSynchronizer { } LOG.info("Index activities"); - activityIndexer.index(); + activityIndexer.setEnabled(true).index(); LOG.info("Index issues"); - issueAuthorizationIndexer.index(); - issueIndexer.index(); + issueAuthorizationIndexer.setEnabled(true).index(); + issueIndexer.setEnabled(true).index(); - LOG.info("Index source files"); - sourceLineIndexer.index(); + LOG.info("Index source lines"); + sourceLineIndexer.setEnabled(true).index(); LOG.info("Index users"); - userIndexer.index(); + userIndexer.setEnabled(true).index(); LOG.info("Index views"); - viewIndexer.index(); + viewIndexer.setEnabled(true).index(); } void synchronize(DbSession session, Dao dao, Index index) { long count = index.getIndexStat().getDocumentCount(); Date lastSynch = index.getLastSynchronization(); + LOG.info("Index {}s", index.getIndexType()); if (count <= 0) { - LOG.info("Index {}s", index.getIndexType()); dao.synchronizeAfter(session); } else { - LOG.info("Index {}s for updates after {}", index.getIndexType(), lastSynch); dao.synchronizeAfter(session, lastSynch); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java index fd8e475a4d1..f00a5b8c457 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java @@ -21,12 +21,10 @@ package org.sonar.server.source.index; import com.google.common.collect.ImmutableList; import org.sonar.server.search.BaseDoc; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.search.IndexUtils; import javax.annotation.CheckForNull; import javax.annotation.Nullable; - import java.util.Collection; import java.util.Date; import java.util.HashMap; @@ -119,11 +117,11 @@ public class SourceLineDoc extends BaseDoc { } public Date updateDate() { - return getFieldAsDate(BaseNormalizer.UPDATED_AT_FIELD); + return getFieldAsDate(SourceLineIndexDefinition.FIELD_UPDATED_AT); } public SourceLineDoc setUpdateDate(@Nullable Date updatedAt) { - setField(BaseNormalizer.UPDATED_AT_FIELD, updatedAt); + setField(SourceLineIndexDefinition.FIELD_UPDATED_AT, updatedAt); return this; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index 70aa212006d..ad5615fdd9a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -24,7 +24,6 @@ import org.sonar.api.config.Settings; import org.sonar.process.ProcessConstants; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; -import org.sonar.server.search.BaseNormalizer; public class SourceLineIndexDefinition implements IndexDefinition { @@ -47,6 +46,7 @@ public class SourceLineIndexDefinition implements IndexDefinition { public static final String FIELD_OVERALL_COVERED_CONDITIONS = "overallCoveredConditions"; public static final String FIELD_SYMBOLS = "symbols"; public static final String FIELD_DUPLICATIONS = "duplications"; + public static final String FIELD_UPDATED_AT = "updatedAt"; public static final String INDEX = "sourcelines"; @@ -95,6 +95,6 @@ public class SourceLineIndexDefinition implements IndexDefinition { sourceLineMapping.createIntegerField(FIELD_OVERALL_COVERED_CONDITIONS); sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).disableSearch().build(); sourceLineMapping.createShortField(FIELD_DUPLICATIONS); - sourceLineMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD); + sourceLineMapping.createDateTimeField(FIELD_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 3e7ce485a15..9e862e6b389 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -43,7 +43,7 @@ public class SourceLineIndexer extends BaseIndexer { private final DbClient dbClient; public SourceLineIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE); + super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, SourceLineIndexDefinition.FIELD_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java b/server/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java index f7a3a25875f..0101265cad3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java +++ b/server/sonar-server/src/main/java/org/sonar/server/ui/JRubyFacade.java @@ -19,7 +19,6 @@ */ package org.sonar.server.ui; -import org.sonar.api.utils.log.Loggers; import org.sonar.api.config.License; import org.sonar.api.config.PropertyDefinitions; import org.sonar.api.config.Settings; @@ -30,6 +29,7 @@ import org.sonar.api.platform.PluginRepository; import org.sonar.api.resources.Language; import org.sonar.api.resources.ResourceType; import org.sonar.api.resources.ResourceTypes; +import org.sonar.api.utils.log.Loggers; import org.sonar.api.web.Footer; import org.sonar.api.web.Page; import org.sonar.api.web.RubyRailsWebservice; @@ -46,11 +46,7 @@ import org.sonar.server.platform.Platform; import org.sonar.server.platform.ServerIdGenerator; import org.sonar.server.platform.ServerSettings; import org.sonar.server.platform.SettingsChangeNotifier; -import org.sonar.server.plugins.InstalledPluginReferentialFactory; -import org.sonar.server.plugins.PluginDownloader; -import org.sonar.server.plugins.ServerPluginJarsInstaller; -import org.sonar.server.plugins.ServerPluginRepository; -import org.sonar.server.plugins.UpdateCenterMatrixFactory; +import org.sonar.server.plugins.*; import org.sonar.server.rule.RuleRepositories; import org.sonar.server.user.NewUserNotifier; import org.sonar.updatecenter.common.PluginReferential; @@ -309,6 +305,9 @@ public final class JRubyFacade { get(ResourceIndexerDao.class).indexResource(resourceId); } + /* + * /!\ Used by Views + */ public void deleteResourceTree(String projectKey) { try { get(ComponentCleanerService.class).delete(projectKey); diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 611b776a1ee..302b522b58c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -36,7 +36,7 @@ public class UserIndexer extends BaseIndexer { private final DbClient dbClient; public UserIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER); + super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, UserIndexDefinition.FIELD_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 3f00775cbf3..bc6f501be0b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -41,7 +41,7 @@ public class ViewIndexer extends BaseIndexer { private final EsClient esClient; public ViewIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW); + super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, "updatedAt"); this.dbClient = dbClient; this.esClient = esClient; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/activity/ActivityServiceTest.java b/server/sonar-server/src/test/java/org/sonar/server/activity/ActivityServiceTest.java index 14807cdde9c..d12d072f1f3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/activity/ActivityServiceTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/activity/ActivityServiceTest.java @@ -57,7 +57,10 @@ public class ActivityServiceTest { ActivityDao activityDao = new ActivityDao(db.myBatis(), system); IssueDao issueDao = new IssueDao(db.myBatis()); DbClient dbClient = new DbClient(db.database(), db.myBatis(), issueDao, activityDao); - service = new ActivityService(dbClient, new ActivityIndexer(dbClient, es.client())); + ActivityIndexer indexer = new ActivityIndexer(dbClient, es.client()); + // indexers are disabled by default + indexer.setEnabled(true); + service = new ActivityService(dbClient, indexer); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java index bd4be80943d..7f4e4e6a16f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -48,8 +48,7 @@ public class BulkIndexerTest { @Test public void index_documents() throws Exception { - BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX) - .setRefresh(true); + BulkIndexer indexer = new BulkIndexer(esTester.client(), FakeIndexDefinition.INDEX); indexer.start(); indexer.add(newIndexRequest(42)); indexer.add(newIndexRequest(78)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java index 09aaf8cd403..5c49de4aef1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java @@ -103,7 +103,7 @@ public class EsTester extends ExternalResource { ComponentContainer container = new ComponentContainer(); container.addSingletons(definitions); container.addSingleton(client); - container.addSingleton(IndexRegistry.class); + container.addSingleton(IndexDefinitions.class); container.addSingleton(IndexCreator.class); container.startComponents(); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 06ba9d18d4a..ac55e4e4c53 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -41,7 +41,7 @@ public class IndexCreatorTest { public void create_index() throws Exception { assertThat(mappings()).isEmpty(); - IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()}); + IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}); registry.start(); IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); @@ -66,7 +66,7 @@ public class IndexCreatorTest { assertThat(mappings()).isEmpty(); // v1 - IndexRegistry registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinition()}); + IndexDefinitions registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinition()}); registry.start(); IndexCreator creator = new IndexCreator(es.client(), registry); creator.start(); @@ -75,7 +75,7 @@ public class IndexCreatorTest { assertThat(hashV1).isNotEmpty(); // v2 - registry = new IndexRegistry(new IndexDefinition[] {new FakeIndexDefinitionV2()}); + registry = new IndexDefinitions(new IndexDefinition[] {new FakeIndexDefinitionV2()}); registry.start(); creator = new IndexCreator(es.client(), registry); creator.start(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java index 958247d87a5..20cff8f8548 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java @@ -29,7 +29,7 @@ public class IndexDefinitionHashTest { @Test public void of() throws Exception { - IndexRegistry.Index indexV1 = new IndexRegistry.Index(createIndex()); + IndexDefinitions.Index indexV1 = new IndexDefinitions.Index(createIndex()); String hashV1 = new IndexDefinitionHash().of(indexV1); assertThat(hashV1).isNotEmpty(); // always the same @@ -37,7 +37,7 @@ public class IndexDefinitionHashTest { NewIndex newIndexV2 = createIndex(); newIndexV2.getTypes().get("fake").createIntegerField("max"); - String hashV2 = new IndexDefinitionHash().of(new IndexRegistry.Index(newIndexV2)); + String hashV2 = new IndexDefinitionHash().of(new IndexDefinitions.Index(newIndexV2)); assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index 3023ae60034..2caaa0bd6b5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -19,7 +19,11 @@ */ package org.sonar.server.issue; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; +import com.google.common.collect.Multiset; +import com.google.common.collect.Sets; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -56,7 +60,6 @@ import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.source.index.SourceLineDoc; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.source.index.SourceLineResultSetIterator; @@ -74,10 +77,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.junit.Assert.fail; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_FILE_UUID; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_LINE; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_PROJECT_UUID; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SCM_AUTHOR; +import static org.sonar.server.source.index.SourceLineIndexDefinition.*; public class IssueServiceMediumTest { @@ -652,7 +652,7 @@ public class IssueServiceMediumTest { .put(FIELD_PROJECT_UUID, file.projectUuid()) .put(FIELD_FILE_UUID, file.uuid()) .put(FIELD_LINE, line) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .put(FIELD_SCM_AUTHOR, scmAuthor) .build()); SourceLineResultSetIterator.SourceFile sourceFile = new SourceLineResultSetIterator.SourceFile(file.uuid(), System.currentTimeMillis()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index cc87af2c9a8..f32df80aa98 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -58,6 +58,15 @@ public class IssueIndexerTest { } @Test + public void index_nothing_if_disabled() throws Exception { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + createIndexer().setEnabled(false).index(); + + assertThat(esTester.countDocuments("issues", "issue")).isEqualTo(0); + } + + @Test public void index() throws Exception { dbTester.prepareDbUnit(getClass(), "index.xml"); @@ -82,6 +91,8 @@ public class IssueIndexerTest { } private IssueIndexer createIndexer() { - return new IssueIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client()); + IssueIndexer indexer = new IssueIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client()); + indexer.setEnabled(true); + return indexer; } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java index 5ba75c5f6ec..8560c032717 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -37,7 +37,6 @@ import org.sonar.api.utils.DateUtils; import org.sonar.core.persistence.DbTester; import org.sonar.server.db.DbClient; import org.sonar.server.es.EsTester; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.source.db.FileSourceTesting; import org.sonar.test.DbTests; import org.sonar.test.TestUtils; @@ -68,6 +67,7 @@ public class SourceLineIndexerTest { es.truncateIndices(); db.truncateTables(); indexer = new SourceLineIndexer(new DbClient(db.database(), db.myBatis()), es.client()); + indexer.setEnabled(true); } @Test @@ -102,7 +102,7 @@ public class SourceLineIndexerTest { .put(FIELD_SCM_AUTHOR, "polop") .put(FIELD_SOURCE, "package org.sonar.server.source;") .put(FIELD_DUPLICATIONS, duplications) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .build()); SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis()); file.addLine(line1); @@ -175,7 +175,7 @@ public class SourceLineIndexerTest { .put(FIELD_OVERALL_LINE_HITS, bigValue) .put(FIELD_OVERALL_CONDITIONS, bigValue) .put(FIELD_OVERALL_COVERED_CONDITIONS, bigValue) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .build()); SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis()); file.addLine(line1); diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java index fdbcefb81ac..812b703252a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java @@ -74,7 +74,17 @@ public class UserIndexerTest { assertThat(doc.updatedAt()).isEqualTo(1500000000000L); } + @Test + public void do_nothing_if_disabled() throws Exception { + dbTester.prepareDbUnit(getClass(), "index.xml"); + + createIndexer().setEnabled(false).index(); + assertThat(esTester.countDocuments("users", "user")).isEqualTo(0); + } + private UserIndexer createIndexer() { - return new UserIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client()); + UserIndexer indexer = new UserIndexer(new DbClient(dbTester.database(), dbTester.myBatis()), esTester.client()); + indexer.setEnabled(true); + return indexer; } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java index b809a46922e..9656c199948 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java @@ -55,6 +55,7 @@ public class ViewIndexerTest { dbTester.truncateTables(); esTester.truncateIndices(); indexer = new ViewIndexer(new DbClient(dbTester.database(), dbTester.myBatis(), new ComponentDao()), esTester.client()); + indexer.setEnabled(true); } @Test diff --git a/server/sonar-web/src/main/webapp/WEB-INF/app/controllers/profiles_controller.rb b/server/sonar-web/src/main/webapp/WEB-INF/app/controllers/profiles_controller.rb index 2b107edfefb..79b88b22bb0 100644 --- a/server/sonar-web/src/main/webapp/WEB-INF/app/controllers/profiles_controller.rb +++ b/server/sonar-web/src/main/webapp/WEB-INF/app/controllers/profiles_controller.rb @@ -268,6 +268,7 @@ class ProfilesController < ApplicationController require_parameters 'key' @profile = Internal.qprofile_loader.getByKey(params[:key]) + not_found('Quality profile does not exist') unless @profile search = {'profileKey' => @profile.key().to_s, 'since' => params[:since], 'to' => params[:to], 'p' => params[:p]} result = Internal.component(Java::OrgSonarServerActivity::RubyQProfileActivityService.java_class).search(search) @changes = result.activities diff --git a/server/sonar-web/src/main/webapp/WEB-INF/app/views/profiles/changelog.html.erb b/server/sonar-web/src/main/webapp/WEB-INF/app/views/profiles/changelog.html.erb index 035bf52d2d5..f807ae791c2 100644 --- a/server/sonar-web/src/main/webapp/WEB-INF/app/views/profiles/changelog.html.erb +++ b/server/sonar-web/src/main/webapp/WEB-INF/app/views/profiles/changelog.html.erb @@ -37,19 +37,19 @@ %> <tr class="<%= cycle('even', 'odd') -%>"> <% - action = change.action() + action = change.getAction() action_message = message('quality_profiles.changelog.' + action.downcase) if action if change.authorName() && !change.authorName().empty?() author = change.authorName() - elsif change.login() && !change.login().empty?() - author = change.login() + elsif change.login() && !change.getLogin().empty?() + author = change.getLogin() else author = 'System' end rule = change.ruleName() ? change.ruleName() : change.ruleKey() %> - <td valign="top" width="1%" nowrap><%= Internal.i18n.formatDateTime(change.time()) -%></td> + <td valign="top" width="1%" nowrap><%= Internal.i18n.formatDateTime(change.getCreatedAt()) -%></td> <td valign="top" width="1%" nowrap><%= author %></td> <td valign="top" width="1%" nowrap><%= action_message %></td> <td valign="top"><%= rule %></td> diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java index 0e2c128b185..393e8968ad9 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java @@ -242,7 +242,7 @@ public class DefaultIndex extends SonarIndex { if (metric == null) { throw new SonarException("Unknown metric: " + measure.getMetricKey()); } - if (!isTechnicalProjectCopy(resource) && !measure.isFromCore() && INTERNAL_METRICS.contains(metric)) { + if (!isViewResource(resource) && !measure.isFromCore() && INTERNAL_METRICS.contains(metric)) { LOG.debug("Metric " + metric.key() + " is an internal metric computed by SonarQube. Provided value is ignored."); return measure; } @@ -257,8 +257,9 @@ public class DefaultIndex extends SonarIndex { /** * Views plugin creates copy of technical projects and should be allowed to copy all measures even internal ones */ - private boolean isTechnicalProjectCopy(Resource resource) { - return Scopes.FILE.equals(resource.getScope()) && Qualifiers.PROJECT.equals(resource.getQualifier()); + private boolean isViewResource(Resource resource) { + boolean isTechnicalProject = Scopes.FILE.equals(resource.getScope()) && Qualifiers.PROJECT.equals(resource.getQualifier()); + return isTechnicalProject || ResourceUtils.isView(resource) || ResourceUtils.isSubview(resource); } // |