From c2ab5d088d449d67ee30fc3187ea67d8b94187b3 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Wed, 4 Mar 2015 16:44:24 +0100 Subject: [PATCH] SONAR-6229 incremental indexing --- .../server/activity/index/ActivityIndexer.java | 2 +- .../main/java/org/sonar/server/es/BaseIndexer.java | 8 +++++--- .../main/java/org/sonar/server/es/EsClient.java | 4 ++-- .../issue/index/IssueAuthorizationIndexer.java | 2 +- .../org/sonar/server/issue/index/IssueIndexer.java | 2 +- .../sonar/server/source/index/SourceLineDoc.java | 6 ++---- .../source/index/SourceLineIndexDefinition.java | 4 ++-- .../server/source/index/SourceLineIndexer.java | 2 +- .../org/sonar/server/user/index/UserIndexer.java | 2 +- .../org/sonar/server/view/index/ViewIndexer.java | 2 +- .../sonar/server/issue/IssueServiceMediumTest.java | 14 +++++++------- .../server/source/index/SourceLineIndexerTest.java | 5 ++--- 12 files changed, 26 insertions(+), 27 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java index 8a179f99fd5..babf3f671bd 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java @@ -39,7 +39,7 @@ public class ActivityIndexer extends BaseIndexer { private final DbClient dbClient; public ActivityIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE); + super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE, ActivityIndexDefinition.FIELD_CREATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java index 5f5f564058d..39a0f443e89 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java @@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit; public abstract class BaseIndexer implements ServerComponent, Startable { private final ThreadPoolExecutor executor; - private final String indexName, typeName; + private final String indexName, typeName, dateFieldName; protected final EsClient esClient; private volatile long lastUpdatedAt = 0L; @@ -48,9 +48,11 @@ public abstract class BaseIndexer implements ServerComponent, Startable { */ private boolean enabled = false; - protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName) { + protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName, + String dateFieldName) { this.indexName = indexName; this.typeName = typeName; + this.dateFieldName = dateFieldName; this.esClient = client; this.executor = new ThreadPoolExecutor(0, 1, threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue()); @@ -86,7 +88,7 @@ public abstract class BaseIndexer implements ServerComponent, Startable { @Override public void start() { - lastUpdatedAt = esClient.getLastUpdatedAt(indexName, typeName); + lastUpdatedAt = esClient.getMaxFieldValue(indexName, typeName, dateFieldName); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 5c252935bc4..61e61608210 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -183,12 +183,12 @@ public class EsClient implements Startable { return new ProxyClearCacheRequestBuilder(client).setIndices(indices); } - public long getLastUpdatedAt(String indexName, String typeName) { + public long getMaxFieldValue(String indexName, String typeName, String fieldName) { SearchRequestBuilder request = prepareSearch(indexName) .setTypes(typeName) .setQuery(QueryBuilders.matchAllQuery()) .setSize(0) - .addAggregation(AggregationBuilders.max("latest").field("updatedAt")); + .addAggregation(AggregationBuilders.max("latest").field(fieldName)); Max max = request.get().getAggregations().get("latest"); return (long) max.getValue(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java index 04bf850b395..0a8c3efd3f4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java @@ -47,7 +47,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer { private final DbClient dbClient; public IssueAuthorizationIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION); + super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index e43df41dd98..011bcc1d7de 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -38,7 +38,7 @@ public class IssueIndexer extends BaseIndexer { private final DbClient dbClient; public IssueIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE); + super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java index fd8e475a4d1..f00a5b8c457 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java @@ -21,12 +21,10 @@ package org.sonar.server.source.index; import com.google.common.collect.ImmutableList; import org.sonar.server.search.BaseDoc; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.search.IndexUtils; import javax.annotation.CheckForNull; import javax.annotation.Nullable; - import java.util.Collection; import java.util.Date; import java.util.HashMap; @@ -119,11 +117,11 @@ public class SourceLineDoc extends BaseDoc { } public Date updateDate() { - return getFieldAsDate(BaseNormalizer.UPDATED_AT_FIELD); + return getFieldAsDate(SourceLineIndexDefinition.FIELD_UPDATED_AT); } public SourceLineDoc setUpdateDate(@Nullable Date updatedAt) { - setField(BaseNormalizer.UPDATED_AT_FIELD, updatedAt); + setField(SourceLineIndexDefinition.FIELD_UPDATED_AT, updatedAt); return this; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index 70aa212006d..ad5615fdd9a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -24,7 +24,6 @@ import org.sonar.api.config.Settings; import org.sonar.process.ProcessConstants; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; -import org.sonar.server.search.BaseNormalizer; public class SourceLineIndexDefinition implements IndexDefinition { @@ -47,6 +46,7 @@ public class SourceLineIndexDefinition implements IndexDefinition { public static final String FIELD_OVERALL_COVERED_CONDITIONS = "overallCoveredConditions"; public static final String FIELD_SYMBOLS = "symbols"; public static final String FIELD_DUPLICATIONS = "duplications"; + public static final String FIELD_UPDATED_AT = "updatedAt"; public static final String INDEX = "sourcelines"; @@ -95,6 +95,6 @@ public class SourceLineIndexDefinition implements IndexDefinition { sourceLineMapping.createIntegerField(FIELD_OVERALL_COVERED_CONDITIONS); sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).disableSearch().build(); sourceLineMapping.createShortField(FIELD_DUPLICATIONS); - sourceLineMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD); + sourceLineMapping.createDateTimeField(FIELD_UPDATED_AT); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java index 3e7ce485a15..9e862e6b389 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java @@ -43,7 +43,7 @@ public class SourceLineIndexer extends BaseIndexer { private final DbClient dbClient; public SourceLineIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE); + super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, SourceLineIndexDefinition.FIELD_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 611b776a1ee..302b522b58c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -36,7 +36,7 @@ public class UserIndexer extends BaseIndexer { private final DbClient dbClient; public UserIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER); + super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, UserIndexDefinition.FIELD_UPDATED_AT); this.dbClient = dbClient; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 3f00775cbf3..bc6f501be0b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -41,7 +41,7 @@ public class ViewIndexer extends BaseIndexer { private final EsClient esClient; public ViewIndexer(DbClient dbClient, EsClient esClient) { - super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW); + super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, "updatedAt"); this.dbClient = dbClient; this.esClient = esClient; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index 3023ae60034..2caaa0bd6b5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -19,7 +19,11 @@ */ package org.sonar.server.issue; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; +import com.google.common.collect.Multiset; +import com.google.common.collect.Sets; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; @@ -56,7 +60,6 @@ import org.sonar.server.permission.InternalPermissionService; import org.sonar.server.permission.PermissionChange; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.source.index.SourceLineDoc; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.source.index.SourceLineResultSetIterator; @@ -74,10 +77,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.junit.Assert.fail; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_FILE_UUID; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_LINE; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_PROJECT_UUID; -import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SCM_AUTHOR; +import static org.sonar.server.source.index.SourceLineIndexDefinition.*; public class IssueServiceMediumTest { @@ -652,7 +652,7 @@ public class IssueServiceMediumTest { .put(FIELD_PROJECT_UUID, file.projectUuid()) .put(FIELD_FILE_UUID, file.uuid()) .put(FIELD_LINE, line) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .put(FIELD_SCM_AUTHOR, scmAuthor) .build()); SourceLineResultSetIterator.SourceFile sourceFile = new SourceLineResultSetIterator.SourceFile(file.uuid(), System.currentTimeMillis()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java index a9b87db14d5..8560c032717 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java @@ -37,7 +37,6 @@ import org.sonar.api.utils.DateUtils; import org.sonar.core.persistence.DbTester; import org.sonar.server.db.DbClient; import org.sonar.server.es.EsTester; -import org.sonar.server.search.BaseNormalizer; import org.sonar.server.source.db.FileSourceTesting; import org.sonar.test.DbTests; import org.sonar.test.TestUtils; @@ -103,7 +102,7 @@ public class SourceLineIndexerTest { .put(FIELD_SCM_AUTHOR, "polop") .put(FIELD_SOURCE, "package org.sonar.server.source;") .put(FIELD_DUPLICATIONS, duplications) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .build()); SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis()); file.addLine(line1); @@ -176,7 +175,7 @@ public class SourceLineIndexerTest { .put(FIELD_OVERALL_LINE_HITS, bigValue) .put(FIELD_OVERALL_CONDITIONS, bigValue) .put(FIELD_OVERALL_COVERED_CONDITIONS, bigValue) - .put(BaseNormalizer.UPDATED_AT_FIELD, new Date()) + .put(FIELD_UPDATED_AT, new Date()) .build()); SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis()); file.addLine(line1); -- 2.39.5