From 2e2d8c71b0f3f8169d22c17bc4e23c13c6046c6d Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 13 Oct 2016 11:06:46 +0200 Subject: [PATCH] SONAR-8221 Take into account date when indexing project measures at startup --- .../project/es/ProjectMeasuresIndexer.java | 27 ++++++++++--------- .../es/ProjectMeasuresResultSetIterator.java | 15 ++++++++--- .../ProjectMeasuresResultSetIteratorTest.java | 26 ++++++++++++++---- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java index 7efb8eab914..8aa7252c0f4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresIndexer.java @@ -20,6 +20,7 @@ package org.sonar.server.project.es; +import java.util.Date; import java.util.Iterator; import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; @@ -44,40 +45,42 @@ public class ProjectMeasuresIndexer extends BaseIndexer { @Override protected long doIndex(long lastUpdatedAt) { - return doIndex(createBulkIndexer(false), (String) null); + return doIndex(createBulkIndexer(false), lastUpdatedAt, null); } public void index(String projectUuid) { - index(lastUpdatedAt -> doIndex(createBulkIndexer(false), projectUuid)); + doIndex(createBulkIndexer(false), 0L, projectUuid); } public void deleteProject(String uuid) { esClient .prepareDelete(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES, uuid) .setRefresh(true) - .setRouting(uuid) .get(); } - private long doIndex(BulkIndexer bulk, @Nullable String projectUuid) { - DbSession dbSession = dbClient.openSession(false); - try { - ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession, projectUuid); - doIndex(bulk, rowIt); + private long doIndex(BulkIndexer bulk, long lastUpdatedAt, @Nullable String projectUuid) { + try (DbSession dbSession = dbClient.openSession(false)) { + ProjectMeasuresResultSetIterator rowIt = ProjectMeasuresResultSetIterator.create(dbClient, dbSession, lastUpdatedAt, projectUuid); + long maxDate = doIndex(bulk, rowIt); rowIt.close(); - return 0L; - } finally { - dbClient.closeSession(dbSession); + return maxDate; } } - private static void doIndex(BulkIndexer bulk, Iterator docs) { + private static long doIndex(BulkIndexer bulk, Iterator docs) { bulk.start(); + long maxDate = 0L; while (docs.hasNext()) { ProjectMeasuresDoc doc = docs.next(); bulk.add(newIndexRequest(doc)); + + Date analysisDate = doc.getAnalysedAt(); + // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance) + maxDate = Math.max(maxDate, analysisDate == null ? 0L : analysisDate.getTime()); } bulk.stop(); + return maxDate; } private BulkIndexer createBulkIndexer(boolean large) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java index 47ad6f59f75..1cf84b6ba54 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/project/es/ProjectMeasuresResultSetIterator.java @@ -45,24 +45,31 @@ public class ProjectMeasuresResultSetIterator extends ResultSetIterator 0L) { + stmt.setLong(index, afterDate); + index++; + } if (projectUuid != null) { - stmt.setString(5, projectUuid); + stmt.setString(index, projectUuid); } return new ProjectMeasuresResultSetIterator(stmt); } catch (SQLException e) { @@ -77,7 +84,7 @@ public class ProjectMeasuresResultSetIterator extends ResultSetIterator docsById = createResultSetAndReturnDocsById(project.uuid()); + Map docsById = createResultSetAndReturnDocsById(0L, project.uuid()); assertThat(docsById).hasSize(1); ProjectMeasuresDoc doc = docsById.get(project.uuid()); @@ -125,21 +125,37 @@ public class ProjectMeasuresResultSetIteratorTest { assertThat(doc.getAnalysedAt()).isNotNull().isEqualTo(new Date(analysis.getCreatedAt())); } + @Test + public void return_only_docs_after_date() throws Exception { + ComponentDto project1 = newProjectDto(); + dbClient.componentDao().insert(dbSession, project1); + dbClient.snapshotDao().insert(dbSession, newAnalysis(project1).setCreatedAt(1_000_000L)); + ComponentDto project2 = newProjectDto(); + dbClient.componentDao().insert(dbSession, project2); + dbClient.snapshotDao().insert(dbSession, newAnalysis(project2).setCreatedAt(2_000_000L)); + dbSession.commit(); + + Map docsById = createResultSetAndReturnDocsById(1_500_000L, null); + + assertThat(docsById).hasSize(1); + assertThat(docsById.get(project2.uuid())).isNotNull(); + } + @Test public void return_nothing_on_unknown_project() throws Exception { componentDbTester.insertProjectAndSnapshot(newProjectDto()); - Map docsById = createResultSetAndReturnDocsById("UNKNOWN"); + Map docsById = createResultSetAndReturnDocsById(0L, "UNKNOWN"); assertThat(docsById).isEmpty(); } private Map createResultSetAndReturnDocsById() { - return createResultSetAndReturnDocsById(null); + return createResultSetAndReturnDocsById(0L, null); } - private Map createResultSetAndReturnDocsById(@Nullable String projectUuid) { - ProjectMeasuresResultSetIterator it = ProjectMeasuresResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), projectUuid); + private Map createResultSetAndReturnDocsById(long date, @Nullable String projectUuid) { + ProjectMeasuresResultSetIterator it = ProjectMeasuresResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), date, projectUuid); Map docsById = Maps.uniqueIndex(it, ProjectMeasuresDoc::getId); it.close(); return docsById; -- 2.39.5