From a60450467b834efdc4378b0452b150cab3d3c22f Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Tue, 1 Aug 2017 13:54:05 +0200 Subject: SONAR-9616 do not index branches in ES indices components and projectmeasures --- .../db/measure/ProjectMeasuresIndexerIterator.java | 2 +- .../org/sonar/db/component/ComponentMapper.xml | 1 + .../org/sonar/db/component/ComponentTesting.java | 1 + .../measure/ProjectMeasuresIndexerIteratorTest.java | 17 +++++++++++++++++ .../server/component/index/ComponentIndexer.java | 14 +++++++------- .../task/projectanalysis/step/IndexAnalysisStep.java | 4 ++-- .../java/org/sonar/server/es/ProjectIndexer.java | 10 +++++----- .../org/sonar/server/issue/index/IssueIndexer.java | 4 ++-- .../index/ProjectMeasuresIndexDefinition.java | 4 ++++ .../server/measure/index/ProjectMeasuresIndexer.java | 8 ++++---- .../server/permission/index/PermissionIndexer.java | 2 +- .../org/sonar/server/test/index/TestIndexer.java | 8 ++++---- .../server/component/index/ComponentIndexerTest.java | 19 +++++++++++++++++++ .../org/sonar/server/es/ProjectIndexersTest.java | 2 +- .../measure/index/ProjectMeasuresIndexerTest.java | 20 ++++++++++++++++++++ .../sonar/server/permission/index/FooIndexer.java | 6 +++--- 16 files changed, 92 insertions(+), 30 deletions(-) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java index 31672e601d9..d6cd39815a9 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java @@ -72,7 +72,7 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator and p.project_uuid = #{projectUuid,jdbcType=VARCHAR} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java index f3814d67d83..08d0854f7b4 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java @@ -196,6 +196,7 @@ public class ComponentTesting { public static ComponentDto newProjectBranch(ComponentDto project, String branchName) { checkArgument(project.qualifier().equals(Qualifiers.PROJECT)); + checkArgument(project.getMainBranchProjectUuid() == null); String uuid = Uuids.createFast(); return new ComponentDto() .setUuid(uuid) diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java index be37829a7e0..321bbcbe8aa 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java @@ -257,6 +257,23 @@ public class ProjectMeasuresIndexerIteratorTest { assertThat(docsById).isEmpty(); } + @Test + public void non_main_branches_are_not_indexed() { + MetricDto metric = insertIntMetric("ncloc"); + ComponentDto project = ComponentTesting.newPrivateProjectDto(dbTester.getDefaultOrganization()); + SnapshotDto projectAnalysis = dbTester.components().insertProjectAndSnapshot(project); + insertMeasure(project, projectAnalysis, metric, 10d); + + ComponentDto branch = ComponentTesting.newProjectBranch(project, "feature/foo"); + SnapshotDto branchAnalysis = dbTester.components().insertProjectAndSnapshot(branch); + insertMeasure(branch, branchAnalysis, metric, 20d); + + Map docsById = createResultSetAndReturnDocsById(); + + assertThat(docsById).hasSize(1).containsOnlyKeys(project.uuid()); + assertThat(docsById.get(project.uuid()).getMeasures().getNumericMeasures().get(metric.getKey())).isEqualTo(10d); + } + private Map createResultSetAndReturnDocsById() { return createResultSetAndReturnDocsById(null); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index 2311bedc694..e9fd1eb5bb9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -72,8 +72,8 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe } @Override - public void indexOnAnalysis(String projectUuid) { - doIndexByProjectUuid(projectUuid, Size.REGULAR); + public void indexOnAnalysis(String branchUuid) { + doIndexByProjectUuid(branchUuid, Size.REGULAR); } @Override @@ -93,7 +93,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe case PROJECT_DELETION: case PROJECT_KEY_UPDATE: List items = projectUuids.stream() - .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), projectUuid, null, projectUuid)) + .map(branchUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), branchUuid, null, branchUuid)) .collect(MoreCollectors.toArrayList(projectUuids.size())); return dbClient.esQueueDao().insert(dbSession, items); @@ -112,12 +112,12 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener); bulkIndexer.start(); - Set projectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); - Set remaining = new HashSet<>(projectUuids); + Set branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); + Set remaining = new HashSet<>(branchUuids); - for (String projectUuid : projectUuids) { + for (String branchUuid : branchUuids) { // TODO allow scrolling multiple projects at the same time - dbClient.componentDao().scrollForIndexing(dbSession, projectUuid, context -> { + dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> { ComponentDto dto = context.getResultObject(); bulkIndexer.add(newIndexRequest(toDocument(dto))); remaining.remove(dto.projectUuid()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IndexAnalysisStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IndexAnalysisStep.java index bc2ec1621cd..6526858ebab 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IndexAnalysisStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IndexAnalysisStep.java @@ -39,10 +39,10 @@ public class IndexAnalysisStep implements ComputationStep { @Override public void execute() { - String projectUuid = treeRootHolder.getRoot().getUuid(); + String branchUuid = treeRootHolder.getRoot().getUuid(); for (ProjectIndexer indexer : indexers) { LOGGER.debug("Call {}", indexer); - indexer.indexOnAnalysis(projectUuid); + indexer.indexOnAnalysis(branchUuid); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/ProjectIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/ProjectIndexer.java index c36c7a5076e..05dd783bf5a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/ProjectIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/ProjectIndexer.java @@ -44,12 +44,12 @@ public interface ProjectIndexer extends ResilientIndexer { } /** - * This method is called when a project must be (re-)indexed, - * for example when project is created or when a new analysis - * is being processed. - * @param projectUuid non-null UUID of project + * This method is called when an analysis must be indexed. + * + * @param branchUuid non-null UUID of branch in table "projects". It can reference + * a non-main branch */ - void indexOnAnalysis(String projectUuid); + void indexOnAnalysis(String branchUuid); Collection prepareForRecovery(DbSession dbSession, Collection projectUuids, ProjectIndexer.Cause cause); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 20b9f08a161..63c08c3fb59 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -98,8 +98,8 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } @Override - public void indexOnAnalysis(String projectUuid) { - try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) { + public void indexOnAnalysis(String branchUuid) { + try (IssueIterator issues = issueIteratorFactory.createForProject(branchUuid)) { doIndex(issues, Size.REGULAR, IndexingListener.NOOP); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java index 44108cc49a5..0def55e7fee 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java @@ -34,6 +34,10 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition { public static final IndexType INDEX_TYPE_PROJECT_MEASURES = new IndexType("projectmeasures", "projectmeasure"); public static final String FIELD_UUID = "uuid"; public static final String FIELD_ORGANIZATION_UUID = "organizationUuid"; + + /** + * Project key. Only projects (qualifier=TRK) + */ public static final String FIELD_KEY = "key"; public static final String FIELD_NAME = "name"; public static final String FIELD_ANALYSED_AT = "analysedAt"; diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index bc0336a204f..daae14186da 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -78,8 +78,8 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization } @Override - public void indexOnAnalysis(String projectUuid) { - doIndex(Size.REGULAR, projectUuid); + public void indexOnAnalysis(String branchUuid) { + doIndex(Size.REGULAR, branchUuid); } @Override @@ -127,7 +127,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization } } - // the remaining uuids reference issues that don't exist in db. They must + // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid)); @@ -136,7 +136,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization private void doIndex(Size size, @Nullable String projectUuid) { try (DbSession dbSession = dbClient.openSession(false); - ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { + ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.NOOP); bulkIndexer.start(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java index 06ab10499af..ef63ee9ac6d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java @@ -93,7 +93,7 @@ public class PermissionIndexer implements ProjectIndexer { } @Override - public void indexOnAnalysis(String projectUuid) { + public void indexOnAnalysis(String branchUuid) { // nothing to do, permissions don't change during an analysis } diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java index f09036bd748..d7fb0c23996 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java @@ -67,7 +67,7 @@ public class TestIndexer implements ProjectIndexer { @Override public void indexOnStartup(Set uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false); - TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, null)) { + TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, null)) { BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.LARGE); bulkIndexer.start(); @@ -77,12 +77,12 @@ public class TestIndexer implements ProjectIndexer { } @Override - public void indexOnAnalysis(String projectUuid) { + public void indexOnAnalysis(String branchUuid) { BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR); bulkIndexer.start(); - addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid); + addProjectDeletionToBulkIndexer(bulkIndexer, branchUuid); try (DbSession dbSession = dbClient.openSession(false); - TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, projectUuid)) { + TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, branchUuid)) { addTestsToBulkIndexer(rowIt, bulkIndexer); } bulkIndexer.stop(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java index 0af3a20efc8..47549914406 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java @@ -83,6 +83,15 @@ public class ComponentIndexerTest { assertThatIndexContainsOnly(project1, project2); } + @Test + public void indexOnStartup_does_not_index_non_main_branches() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo"); + + underTest.indexOnStartup(emptySet()); + + assertThatIndexContainsOnly(project); + } @Test public void indexOnAnalysis_indexes_project() { @@ -121,6 +130,16 @@ public class ComponentIndexerTest { assertThatComponentHasName(project, "NewName"); } + @Test + public void indexOnAnalysis_does_not_index_non_main_branches() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo"); + + underTest.indexOnAnalysis(branch.uuid()); + + assertThatIndexHasSize(0); + } + @Test public void do_not_update_index_on_project_tag_update() { ComponentDto project = db.components().insertPrivateProject(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/ProjectIndexersTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/ProjectIndexersTest.java index 9ec69689c1b..136e32079b3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/ProjectIndexersTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/ProjectIndexersTest.java @@ -85,7 +85,7 @@ public class ProjectIndexersTest { } @Override - public void indexOnAnalysis(String projectUuid) { + public void indexOnAnalysis(String branchUuid) { throw new UnsupportedOperationException(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java index 396c3517cc3..e8f63c4a87c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java @@ -93,6 +93,16 @@ public class ProjectMeasuresIndexerTest { assertThatIndexContainsOnly(project); } + @Test + public void indexOnStartup_ignores_non_main_branches() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo"); + + underTest.indexOnStartup(emptySet()); + + assertThatIndexContainsOnly(project); + } + @Test public void indexOnAnalysis_indexes_provisioned_project() { ComponentDto project1 = db.components().insertPrivateProject(); @@ -189,6 +199,16 @@ public class ProjectMeasuresIndexerTest { assertThatIndexContainsOnly(project); } + @Test + public void non_main_branches_are_not_indexed_during_analysis() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo"); + + underTest.indexOnAnalysis(branch.uuid()); + + assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); + } + private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) { DbSession dbSession = db.getSession(); Collection items = underTest.prepareForRecovery(dbSession, singletonList(project.uuid()), cause); diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java index cb6a69925b7..e6f74bffca4 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java @@ -51,9 +51,9 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } @Override - public void indexOnAnalysis(String projectUuid) { - addToIndex(projectUuid, "bar"); - addToIndex(projectUuid, "baz"); + public void indexOnAnalysis(String branchUuid) { + addToIndex(branchUuid, "bar"); + addToIndex(branchUuid, "baz"); } @Override -- cgit v1.2.3