SONAR-6229 incremental indexing

diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java
index 8a179f99fd5028f1f6f730e1f8c944affe8c48c5..babf3f671bd5f0b9d4796305d2f1c0d239e753cb 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexer.java
@@ -39,7 +39,7 @@ public class ActivityIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public ActivityIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE);
+    super(esClient, 0L, ActivityIndexDefinition.INDEX, ActivityIndexDefinition.TYPE, ActivityIndexDefinition.FIELD_CREATED_AT);
     this.dbClient = dbClient;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
index 5f5f564058d4b532eb4d294cb2c67547a080fd2b..39a0f443e895f88e5a1d417e3bd5bd17a7c55cf8 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
@@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class BaseIndexer implements ServerComponent, Startable {
 
   private final ThreadPoolExecutor executor;
-  private final String indexName, typeName;
+  private final String indexName, typeName, dateFieldName;
   protected final EsClient esClient;
   private volatile long lastUpdatedAt = 0L;
 
@@ -48,9 +48,11 @@ public abstract class BaseIndexer implements ServerComponent, Startable {
    */
   private boolean enabled = false;
 
-  protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName) {
+  protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName,
+                        String dateFieldName) {
     this.indexName = indexName;
     this.typeName = typeName;
+    this.dateFieldName = dateFieldName;
     this.esClient = client;
     this.executor = new ThreadPoolExecutor(0, 1,
       threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
@@ -86,7 +88,7 @@ public abstract class BaseIndexer implements ServerComponent, Startable {
 
   @Override
   public void start() {
-    lastUpdatedAt = esClient.getLastUpdatedAt(indexName, typeName);
+    lastUpdatedAt = esClient.getMaxFieldValue(indexName, typeName, dateFieldName);
   }
 
   @Override

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
index 5c252935bc42e8d99ded5e68cf2358308d717d41..61e61608210f5ab3fc2f1041b0eeea11ede8999d 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
@@ -183,12 +183,12 @@ public class EsClient implements Startable {
     return new ProxyClearCacheRequestBuilder(client).setIndices(indices);
   }
 
-  public long getLastUpdatedAt(String indexName, String typeName) {
+  public long getMaxFieldValue(String indexName, String typeName, String fieldName) {
     SearchRequestBuilder request = prepareSearch(indexName)
       .setTypes(typeName)
       .setQuery(QueryBuilders.matchAllQuery())
       .setSize(0)
-      .addAggregation(AggregationBuilders.max("latest").field("updatedAt"));
+      .addAggregation(AggregationBuilders.max("latest").field(fieldName));
 
     Max max = request.get().getAggregations().get("latest");
     return (long) max.getValue();

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java
index 04bf850b395d4b31c6c98108ab1ed56107743b9e..0a8c3efd3f4a2d904f7edd3cddb3626763e82de9 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueAuthorizationIndexer.java
@@ -47,7 +47,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public IssueAuthorizationIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION);
+    super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT);
     this.dbClient = dbClient;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
index e43df41dd98c2ac290144de53ff391bb1ba54d6f..011bcc1d7de576c16ba106950a58297619f7b64a 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
@@ -38,7 +38,7 @@ public class IssueIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public IssueIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE);
+    super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, IssueIndexDefinition.FIELD_ISSUE_TECHNICAL_UPDATED_AT);
     this.dbClient = dbClient;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java
index fd8e475a4d17ae87eae9d857d0eb2beabc4064f1..f00a5b8c4573f714b4d1e7c7858e3325716f16c5 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineDoc.java
@@ -21,12 +21,10 @@ package org.sonar.server.source.index;
 
 import com.google.common.collect.ImmutableList;
 import org.sonar.server.search.BaseDoc;
-import org.sonar.server.search.BaseNormalizer;
 import org.sonar.server.search.IndexUtils;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
-
 import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
@@ -119,11 +117,11 @@ public class SourceLineDoc extends BaseDoc {
   }
 
   public Date updateDate() {
-    return getFieldAsDate(BaseNormalizer.UPDATED_AT_FIELD);
+    return getFieldAsDate(SourceLineIndexDefinition.FIELD_UPDATED_AT);
   }
 
   public SourceLineDoc setUpdateDate(@Nullable Date updatedAt) {
-    setField(BaseNormalizer.UPDATED_AT_FIELD, updatedAt);
+    setField(SourceLineIndexDefinition.FIELD_UPDATED_AT, updatedAt);
     return this;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
index 70aa212006d58f8042bdfb9e21ab546ec9831f46..ad5615fdd9a5f819220232efb486b272236913c2 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
@@ -24,7 +24,6 @@ import org.sonar.api.config.Settings;
 import org.sonar.process.ProcessConstants;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
-import org.sonar.server.search.BaseNormalizer;
 
 public class SourceLineIndexDefinition implements IndexDefinition {
 
@@ -47,6 +46,7 @@ public class SourceLineIndexDefinition implements IndexDefinition {
   public static final String FIELD_OVERALL_COVERED_CONDITIONS = "overallCoveredConditions";
   public static final String FIELD_SYMBOLS = "symbols";
   public static final String FIELD_DUPLICATIONS = "duplications";
+  public static final String FIELD_UPDATED_AT = "updatedAt";
 
   public static final String INDEX = "sourcelines";
 
@@ -95,6 +95,6 @@ public class SourceLineIndexDefinition implements IndexDefinition {
     sourceLineMapping.createIntegerField(FIELD_OVERALL_COVERED_CONDITIONS);
     sourceLineMapping.stringFieldBuilder(FIELD_SYMBOLS).disableSearch().build();
     sourceLineMapping.createShortField(FIELD_DUPLICATIONS);
-    sourceLineMapping.createDateTimeField(BaseNormalizer.UPDATED_AT_FIELD);
+    sourceLineMapping.createDateTimeField(FIELD_UPDATED_AT);
   }
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
index 3e7ce485a159cf26cbd453613d8536364eb0a142..9e862e6b38963d8c21a16d90702f4c704d088f3b 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
@@ -43,7 +43,7 @@ public class SourceLineIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public SourceLineIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE);
+    super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, SourceLineIndexDefinition.FIELD_UPDATED_AT);
     this.dbClient = dbClient;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
index 611b776a1ee776959f3840739eb676a0a9bd0561..302b522b58cc9e3e5f006e845709ecd405ffd3e5 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java
@@ -36,7 +36,7 @@ public class UserIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public UserIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER);
+    super(esClient, 300, UserIndexDefinition.INDEX, UserIndexDefinition.TYPE_USER, UserIndexDefinition.FIELD_UPDATED_AT);
     this.dbClient = dbClient;
   }
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java
index 3f00775cbf3e15d096766ee1820b15230b6e2dde..bc6f501be0b82983be84fdb943cbabe1a2e4431f 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java
@@ -41,7 +41,7 @@ public class ViewIndexer extends BaseIndexer {
   private final EsClient esClient;
 
   public ViewIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW);
+    super(esClient, 300, ViewIndexDefinition.INDEX, ViewIndexDefinition.TYPE_VIEW, "updatedAt");
     this.dbClient = dbClient;
     this.esClient = esClient;
   }

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java
index 3023ae60034e22aad233145668f6a100c457b789..2caaa0bd6b51218790ddb2e9526afd1e61f2705c 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java
@@ -19,7 +19,11 @@
  */
 package org.sonar.server.issue;
 
-import com.google.common.collect.*;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Multiset;
+import com.google.common.collect.Sets;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
@@ -56,7 +60,6 @@ import org.sonar.server.permission.InternalPermissionService;
 import org.sonar.server.permission.PermissionChange;
 import org.sonar.server.rule.RuleTesting;
 import org.sonar.server.rule.db.RuleDao;
-import org.sonar.server.search.BaseNormalizer;
 import org.sonar.server.source.index.SourceLineDoc;
 import org.sonar.server.source.index.SourceLineIndexer;
 import org.sonar.server.source.index.SourceLineResultSetIterator;
@@ -74,10 +77,7 @@ import java.util.Map;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.entry;
 import static org.junit.Assert.fail;
-import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_FILE_UUID;
-import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_LINE;
-import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_PROJECT_UUID;
-import static org.sonar.server.source.index.SourceLineIndexDefinition.FIELD_SCM_AUTHOR;
+import static org.sonar.server.source.index.SourceLineIndexDefinition.*;
 
 public class IssueServiceMediumTest {
 
@@ -652,7 +652,7 @@ public class IssueServiceMediumTest {
       .put(FIELD_PROJECT_UUID, file.projectUuid())
       .put(FIELD_FILE_UUID, file.uuid())
       .put(FIELD_LINE, line)
-      .put(BaseNormalizer.UPDATED_AT_FIELD, new Date())
+      .put(FIELD_UPDATED_AT, new Date())
       .put(FIELD_SCM_AUTHOR, scmAuthor)
       .build());
     SourceLineResultSetIterator.SourceFile sourceFile = new SourceLineResultSetIterator.SourceFile(file.uuid(), System.currentTimeMillis());

diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
index a9b87db14d5dde7430b375fdb08c482b4796363b..8560c032717267262648b640cc0d8b8424b794c8 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
@@ -37,7 +37,6 @@ import org.sonar.api.utils.DateUtils;
 import org.sonar.core.persistence.DbTester;
 import org.sonar.server.db.DbClient;
 import org.sonar.server.es.EsTester;
-import org.sonar.server.search.BaseNormalizer;
 import org.sonar.server.source.db.FileSourceTesting;
 import org.sonar.test.DbTests;
 import org.sonar.test.TestUtils;
@@ -103,7 +102,7 @@ public class SourceLineIndexerTest {
       .put(FIELD_SCM_AUTHOR, "polop")
       .put(FIELD_SOURCE, "package org.sonar.server.source;")
       .put(FIELD_DUPLICATIONS, duplications)
-      .put(BaseNormalizer.UPDATED_AT_FIELD, new Date())
+      .put(FIELD_UPDATED_AT, new Date())
       .build());
     SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis());
     file.addLine(line1);
@@ -176,7 +175,7 @@ public class SourceLineIndexerTest {
       .put(FIELD_OVERALL_LINE_HITS, bigValue)
       .put(FIELD_OVERALL_CONDITIONS, bigValue)
       .put(FIELD_OVERALL_COVERED_CONDITIONS, bigValue)
-      .put(BaseNormalizer.UPDATED_AT_FIELD, new Date())
+      .put(FIELD_UPDATED_AT, new Date())
       .build());
     SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis());
     file.addLine(line1);