SONAR-8221 Take into account date when indexing project measures at startup

diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java
index 7efb8eab91447af390554d7bf9e54314d05f8ae8..8aa7252c0f4ece18fb1191cea6c23f3b3083bf95 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java
@@ -20,6 +20,7 @@
 
 package org.sonar.server.project.es;
 
+import java.util.Date;
 import java.util.Iterator;
 import javax.annotation.Nullable;
 import org.elasticsearch.action.index.IndexRequest;
@@ -44,40 +45,42 @@ public class ProjectMeasuresIndexer extends BaseIndexer {
 
   @Override
   protected long doIndex(long lastUpdatedAt) {
-    return doIndex(createBulkIndexer(false), (String) null);
+    return doIndex(createBulkIndexer(false), lastUpdatedAt, null);
   }
 
   public void index(String projectUuid) {
-    index(lastUpdatedAt -> doIndex(createBulkIndexer(false), projectUuid));
+    doIndex(createBulkIndexer(false), 0L, projectUuid);
   }
 
   public void deleteProject(String uuid) {
     esClient
       .prepareDelete(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, uuid)
       .setRefresh(true)
-      .setRouting(uuid)
       .get();
   }
 
-  private long doIndex(BulkIndexer bulk, @Nullable String projectUuid) {
-    DbSession dbSession = dbClient.openSession(false);
-    try {
-      ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession, projectUuid);
-      doIndex(bulk, rowIt);
+  private long doIndex(BulkIndexer bulk, long lastUpdatedAt, @Nullable String projectUuid) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession, lastUpdatedAt, projectUuid);
+      long maxDate = doIndex(bulk, rowIt);
       rowIt.close();
-      return 0L;
-    } finally {
-      dbClient.closeSession(dbSession);
+      return maxDate;
     }
   }
 
-  private static void doIndex(BulkIndexer bulk, Iterator<ProjectMeasuresDoc> docs) {
+  private static long doIndex(BulkIndexer bulk, Iterator<ProjectMeasuresDoc> docs) {
     bulk.start();
+    long maxDate = 0L;
     while (docs.hasNext()) {
       ProjectMeasuresDoc doc = docs.next();
       bulk.add(newIndexRequest(doc));
+
+      Date analysisDate = doc.getAnalysedAt();
+      // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance)
+      maxDate = Math.max(maxDate, analysisDate == null ? 0L : analysisDate.getTime());
     }
     bulk.stop();
+    return maxDate;
   }
 
   private BulkIndexer createBulkIndexer(boolean large) {

diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java
index 47ad6f59f758ee93b51ab22354aef68ca289580d..1cf84b6ba547a82ad59a763efabfe4b88667da80 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java
@@ -45,24 +45,31 @@ public class ProjectMeasuresResultSetIterator extends ResultSetIterator<ProjectM
     "LEFT OUTER JOIN snapshots s ON s.component_uuid=p.uuid AND s.islast=? " +
     "WHERE p.enabled=? AND p.scope=? AND p.qualifier=?";
 
+  private static final String DATE_FILTER = " AND s.created_at>?";
+
   private static final String PROJECT_FILTER = " AND p.uuid=?";
 
   private ProjectMeasuresResultSetIterator(PreparedStatement stmt) throws SQLException {
     super(stmt);
   }
 
-  static ProjectMeasuresResultSetIterator create(DbClient dbClient, DbSession session, @Nullable String projectUuid) {
+  static ProjectMeasuresResultSetIterator create(DbClient dbClient, DbSession session, long afterDate, @Nullable String projectUuid) {
     try {
-
       String sql = SQL_ALL;
+      sql += afterDate <= 0L ? "" : DATE_FILTER;
       sql += projectUuid == null ? "" : PROJECT_FILTER;
       PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
       stmt.setBoolean(1, true);
       stmt.setBoolean(2, true);
       stmt.setString(3, Scopes.PROJECT);
       stmt.setString(4, Qualifiers.PROJECT);
+      int index = 5;
+      if (afterDate > 0L) {
+        stmt.setLong(index, afterDate);
+        index++;
+      }
       if (projectUuid != null) {
-        stmt.setString(5, projectUuid);
+        stmt.setString(index, projectUuid);
       }
       return new ProjectMeasuresResultSetIterator(stmt);
     } catch (SQLException e) {
@@ -77,7 +84,7 @@ public class ProjectMeasuresResultSetIterator extends ResultSetIterator<ProjectM
       .setKey(rs.getString(2))
       .setName(rs.getString(3));
     long analysisDate = rs.getLong(4);
-    doc.setAnalysedAt(analysisDate != 0 ? new Date(analysisDate) : null);
+    doc.setAnalysedAt(rs.wasNull() ? null : new Date(analysisDate));
     return doc;
   }
 }

diff --git a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java
index b018d1b72f00f6cb6b617385fbc4136126f5a004..add468981a691189c2b5714cc9a7901eb97938af 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/project/es/ProjectMeasuresResultSetIteratorTest.java
@@ -114,7 +114,7 @@ public class ProjectMeasuresResultSetIteratorTest {
     componentDbTester.insertProjectAndSnapshot(newProjectDto());
     componentDbTester.insertProjectAndSnapshot(newProjectDto());
 
-    Map<String, ProjectMeasuresDoc> docsById = createResultSetAndReturnDocsById(project.uuid());
+    Map<String, ProjectMeasuresDoc> docsById = createResultSetAndReturnDocsById(0L, project.uuid());
 
     assertThat(docsById).hasSize(1);
     ProjectMeasuresDoc doc = docsById.get(project.uuid());
@@ -125,21 +125,37 @@ public class ProjectMeasuresResultSetIteratorTest {
     assertThat(doc.getAnalysedAt()).isNotNull().isEqualTo(new Date(analysis.getCreatedAt()));
   }
 
+  @Test
+  public void return_only_docs_after_date() throws Exception {
+    ComponentDto project1 = newProjectDto();
+    dbClient.componentDao().insert(dbSession, project1);
+    dbClient.snapshotDao().insert(dbSession, newAnalysis(project1).setCreatedAt(1_000_000L));
+    ComponentDto project2 = newProjectDto();
+    dbClient.componentDao().insert(dbSession, project2);
+    dbClient.snapshotDao().insert(dbSession, newAnalysis(project2).setCreatedAt(2_000_000L));
+    dbSession.commit();
+
+    Map<String, ProjectMeasuresDoc> docsById = createResultSetAndReturnDocsById(1_500_000L, null);
+
+    assertThat(docsById).hasSize(1);
+    assertThat(docsById.get(project2.uuid())).isNotNull();
+  }
+
   @Test
   public void return_nothing_on_unknown_project() throws Exception {
     componentDbTester.insertProjectAndSnapshot(newProjectDto());
 
-    Map<String, ProjectMeasuresDoc> docsById = createResultSetAndReturnDocsById("UNKNOWN");
+    Map<String, ProjectMeasuresDoc> docsById = createResultSetAndReturnDocsById(0L, "UNKNOWN");
 
     assertThat(docsById).isEmpty();
   }
 
   private Map<String, ProjectMeasuresDoc> createResultSetAndReturnDocsById() {
-    return createResultSetAndReturnDocsById(null);
+    return createResultSetAndReturnDocsById(0L, null);
   }
 
-  private Map<String, ProjectMeasuresDoc> createResultSetAndReturnDocsById(@Nullable String projectUuid) {
-    ProjectMeasuresResultSetIterator it = ProjectMeasuresResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), projectUuid);
+  private Map<String, ProjectMeasuresDoc> createResultSetAndReturnDocsById(long date, @Nullable String projectUuid) {
+    ProjectMeasuresResultSetIterator it = ProjectMeasuresResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), date, projectUuid);
     Map<String, ProjectMeasuresDoc> docsById = Maps.uniqueIndex(it, ProjectMeasuresDoc::getId);
     it.close();
     return docsById;