]> source.dussan.org Git - sonarqube.git/commitdiff
SONAR-9616 do not index branches in ES indices components and projectmeasures
authorSimon Brandhof <simon.brandhof@sonarsource.com>
Tue, 1 Aug 2017 11:54:05 +0000 (13:54 +0200)
committerJanos Gyerik <janos.gyerik@sonarsource.com>
Tue, 12 Sep 2017 08:55:09 +0000 (10:55 +0200)
16 files changed:
server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java
server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml
server/sonar-db-dao/src/test/java/org/sonar/db/component/ComponentTesting.java
server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java
server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java
server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IndexAnalysisStep.java
server/sonar-server/src/main/java/org/sonar/server/es/ProjectIndexer.java
server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java
server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java
server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java
server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java
server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java
server/sonar-server/src/test/java/org/sonar/server/es/ProjectIndexersTest.java
server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java
server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java

index 31672e601d9ab354a407622df8fff03e3c8ab982..d6cd39815a93f99c0470a57eed4d7f7c9286d89e 100644 (file)
@@ -72,7 +72,7 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator<ProjectMea
   private static final String SQL_PROJECTS = "SELECT p.organization_uuid, p.uuid, p.kee, p.name, s.uuid, s.created_at, p.tags " +
     "FROM projects p " +
     "LEFT OUTER JOIN snapshots s ON s.component_uuid=p.uuid AND s.islast=? " +
-    "WHERE p.enabled=? AND p.scope=? AND p.qualifier=?";
+    "WHERE p.enabled=? AND p.scope=? AND p.qualifier=? and p.main_branch_project_uuid is null ";
 
   private static final String PROJECT_FILTER = " AND p.uuid=?";
 
index eb3249b44f8f744922d16c1b4eaa64c41a0f22ca..47b464dd2dd79910faf2af81348dd72c5fbc847a 100644 (file)
     where
       p.enabled=${_true}
       and p.copy_component_uuid is null
+      and p.main_branch_project_uuid is null
       <if test="projectUuid != null">
         and p.project_uuid = #{projectUuid,jdbcType=VARCHAR}
       </if>
index f3814d67d838ed63bfaf0c633f00a32377246c65..08d0854f7b4aec57e25ce28ec73a30b8f6b58627 100644 (file)
@@ -196,6 +196,7 @@ public class ComponentTesting {
 
   public static ComponentDto newProjectBranch(ComponentDto project, String branchName) {
     checkArgument(project.qualifier().equals(Qualifiers.PROJECT));
+    checkArgument(project.getMainBranchProjectUuid() == null);
     String uuid = Uuids.createFast();
     return new ComponentDto()
       .setUuid(uuid)
index be37829a7e06652eb2e7e45756f97b3503c6aeea..321bbcbe8aa46f8ca2dd9ac6a061f358fd3551ad 100644 (file)
@@ -257,6 +257,23 @@ public class ProjectMeasuresIndexerIteratorTest {
     assertThat(docsById).isEmpty();
   }
 
+  @Test
+  public void non_main_branches_are_not_indexed() {
+    MetricDto metric = insertIntMetric("ncloc");
+    ComponentDto project = ComponentTesting.newPrivateProjectDto(dbTester.getDefaultOrganization());
+    SnapshotDto projectAnalysis = dbTester.components().insertProjectAndSnapshot(project);
+    insertMeasure(project, projectAnalysis, metric, 10d);
+
+    ComponentDto branch = ComponentTesting.newProjectBranch(project, "feature/foo");
+    SnapshotDto branchAnalysis = dbTester.components().insertProjectAndSnapshot(branch);
+    insertMeasure(branch, branchAnalysis, metric, 20d);
+
+    Map<String, ProjectMeasures> docsById = createResultSetAndReturnDocsById();
+
+    assertThat(docsById).hasSize(1).containsOnlyKeys(project.uuid());
+    assertThat(docsById.get(project.uuid()).getMeasures().getNumericMeasures().get(metric.getKey())).isEqualTo(10d);
+  }
+
   private Map<String, ProjectMeasures> createResultSetAndReturnDocsById() {
     return createResultSetAndReturnDocsById(null);
   }
index 2311bedc694f74bfbb3ecf93e62db4fa8502573b..e9fd1eb5bb9dc7dbd8f160b77ba6efbe2d1732d8 100644 (file)
@@ -72,8 +72,8 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
-    doIndexByProjectUuid(projectUuid, Size.REGULAR);
+  public void indexOnAnalysis(String branchUuid) {
+    doIndexByProjectUuid(branchUuid, Size.REGULAR);
   }
 
   @Override
@@ -93,7 +93,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
       case PROJECT_DELETION:
       case PROJECT_KEY_UPDATE:
         List<EsQueueDto> items = projectUuids.stream()
-          .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), projectUuid, null, projectUuid))
+          .map(branchUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), branchUuid, null, branchUuid))
           .collect(MoreCollectors.toArrayList(projectUuids.size()));
         return dbClient.esQueueDao().insert(dbSession, items);
 
@@ -112,12 +112,12 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe
     OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items);
     BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener);
     bulkIndexer.start();
-    Set<String> projectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size()));
-    Set<String> remaining = new HashSet<>(projectUuids);
+    Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size()));
+    Set<String> remaining = new HashSet<>(branchUuids);
 
-    for (String projectUuid : projectUuids) {
+    for (String branchUuid : branchUuids) {
       // TODO allow scrolling multiple projects at the same time
-      dbClient.componentDao().scrollForIndexing(dbSession, projectUuid, context -> {
+      dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> {
         ComponentDto dto = context.getResultObject();
         bulkIndexer.add(newIndexRequest(toDocument(dto)));
         remaining.remove(dto.projectUuid());
index bc2ec1621cd0fd05ac4babb452462b7d0686c308..6526858ebabc5b69925290e236e622b2d5c74dc8 100644 (file)
@@ -39,10 +39,10 @@ public class IndexAnalysisStep implements ComputationStep {
 
   @Override
   public void execute() {
-    String projectUuid = treeRootHolder.getRoot().getUuid();
+    String branchUuid = treeRootHolder.getRoot().getUuid();
     for (ProjectIndexer indexer : indexers) {
       LOGGER.debug("Call {}", indexer);
-      indexer.indexOnAnalysis(projectUuid);
+      indexer.indexOnAnalysis(branchUuid);
     }
   }
 
index c36c7a5076e1500e4639f08b311c84af3ae141b2..05dd783bf5a08b661abb5b037743775b2b2974fd 100644 (file)
@@ -44,12 +44,12 @@ public interface ProjectIndexer extends ResilientIndexer {
   }
 
   /**
-   * This method is called when a project must be (re-)indexed,
-   * for example when project is created or when a new analysis
-   * is being processed.
-   * @param projectUuid non-null UUID of project
+   * This method is called when an analysis must be indexed.
+   *
+   * @param branchUuid non-null UUID of branch in table "projects". It can reference
+   *                   a non-main branch
    */
-  void indexOnAnalysis(String projectUuid);
+  void indexOnAnalysis(String branchUuid);
 
   Collection<EsQueueDto> prepareForRecovery(DbSession dbSession, Collection<String> projectUuids, ProjectIndexer.Cause cause);
 }
index 20b9f08a1617f9b09cf4382916818e65ce1db0bf..63c08c3fb59971f346b80464aacfe88c5cae0233 100644 (file)
@@ -98,8 +98,8 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
-    try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) {
+  public void indexOnAnalysis(String branchUuid) {
+    try (IssueIterator issues = issueIteratorFactory.createForProject(branchUuid)) {
       doIndex(issues, Size.REGULAR, IndexingListener.NOOP);
     }
   }
index 44108cc49a552c3cc46109fc8e09318223a2c1f0..0def55e7fee793156beff43b0c2211f1e913f861 100644 (file)
@@ -34,6 +34,10 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition {
   public static final IndexType INDEX_TYPE_PROJECT_MEASURES = new IndexType("projectmeasures", "projectmeasure");
   public static final String FIELD_UUID = "uuid";
   public static final String FIELD_ORGANIZATION_UUID = "organizationUuid";
+
+  /**
+   * Project key. Only projects (qualifier=TRK)
+   */
   public static final String FIELD_KEY = "key";
   public static final String FIELD_NAME = "name";
   public static final String FIELD_ANALYSED_AT = "analysedAt";
index bc0336a204f2d16f7f700399485e2b06ca8a9fe6..daae14186da7975fe4f9ce92596891bd0438a77a 100644 (file)
@@ -78,8 +78,8 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
-    doIndex(Size.REGULAR, projectUuid);
+  public void indexOnAnalysis(String branchUuid) {
+    doIndex(Size.REGULAR, branchUuid);
   }
 
   @Override
@@ -127,7 +127,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization
       }
     }
 
-    // the remaining uuids reference issues that don't exist in db. They must
+    // the remaining uuids reference projects that don't exist in db. They must
     // be deleted from index.
     projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid));
 
@@ -136,7 +136,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization
 
   private void doIndex(Size size, @Nullable String projectUuid) {
     try (DbSession dbSession = dbClient.openSession(false);
-         ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) {
+      ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) {
 
       BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.NOOP);
       bulkIndexer.start();
index 06ab10499afc635b4f47aee123c2b3d580f9ff2b..ef63ee9ac6df759a68df37a2d6d27102715eaa02 100644 (file)
@@ -93,7 +93,7 @@ public class PermissionIndexer implements ProjectIndexer {
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
+  public void indexOnAnalysis(String branchUuid) {
     // nothing to do, permissions don't change during an analysis
   }
 
index f09036bd74856c5680e17612a1961f0372b0c55f..d7fb0c239960269c7f1ed7f6015cc4b3d9c42151 100644 (file)
@@ -67,7 +67,7 @@ public class TestIndexer implements ProjectIndexer {
   @Override
   public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     try (DbSession dbSession = dbClient.openSession(false);
-         TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, null)) {
+      TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, null)) {
 
       BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.LARGE);
       bulkIndexer.start();
@@ -77,12 +77,12 @@ public class TestIndexer implements ProjectIndexer {
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
+  public void indexOnAnalysis(String branchUuid) {
     BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR);
     bulkIndexer.start();
-    addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid);
+    addProjectDeletionToBulkIndexer(bulkIndexer, branchUuid);
     try (DbSession dbSession = dbClient.openSession(false);
-         TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, projectUuid)) {
+      TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, branchUuid)) {
       addTestsToBulkIndexer(rowIt, bulkIndexer);
     }
     bulkIndexer.stop();
index 0af3a20efc86f95e20e012ea441798cef9a834b6..475499144067129ec0afbb5acc23afb7dfad3069 100644 (file)
@@ -83,6 +83,15 @@ public class ComponentIndexerTest {
     assertThatIndexContainsOnly(project1, project2);
   }
 
+  @Test
+  public void indexOnStartup_does_not_index_non_main_branches() {
+    ComponentDto project = db.components().insertPrivateProject();
+    ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo");
+
+    underTest.indexOnStartup(emptySet());
+
+    assertThatIndexContainsOnly(project);
+  }
 
   @Test
   public void indexOnAnalysis_indexes_project() {
@@ -121,6 +130,16 @@ public class ComponentIndexerTest {
     assertThatComponentHasName(project, "NewName");
   }
 
+  @Test
+  public void indexOnAnalysis_does_not_index_non_main_branches() {
+    ComponentDto project = db.components().insertPrivateProject();
+    ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo");
+
+    underTest.indexOnAnalysis(branch.uuid());
+
+    assertThatIndexHasSize(0);
+  }
+
   @Test
   public void do_not_update_index_on_project_tag_update() {
     ComponentDto project = db.components().insertPrivateProject();
index 9ec69689c1bd08fbb339bba5ec9c182bc9fea509..136e32079b3c1a9aa1c40be450f3cb684d09093b 100644 (file)
@@ -85,7 +85,7 @@ public class ProjectIndexersTest {
     }
 
     @Override
-    public void indexOnAnalysis(String projectUuid) {
+    public void indexOnAnalysis(String branchUuid) {
       throw new UnsupportedOperationException();
     }
   }
index 396c3517cc36ad0969689305135b6d6a846756be..e8f63c4a87c762757b04a97fcb8a732f6f9ded83 100644 (file)
@@ -93,6 +93,16 @@ public class ProjectMeasuresIndexerTest {
     assertThatIndexContainsOnly(project);
   }
 
+  @Test
+  public void indexOnStartup_ignores_non_main_branches() {
+    ComponentDto project = db.components().insertPrivateProject();
+    ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo");
+
+    underTest.indexOnStartup(emptySet());
+
+    assertThatIndexContainsOnly(project);
+  }
+
   @Test
   public void indexOnAnalysis_indexes_provisioned_project() {
     ComponentDto project1 = db.components().insertPrivateProject();
@@ -189,6 +199,16 @@ public class ProjectMeasuresIndexerTest {
     assertThatIndexContainsOnly(project);
   }
 
+  @Test
+  public void non_main_branches_are_not_indexed_during_analysis() {
+    ComponentDto project = db.components().insertPrivateProject();
+    ComponentDto branch = db.components().insertProjectBranch(project, "feature/foo");
+
+    underTest.indexOnAnalysis(branch.uuid());
+
+    assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0);
+  }
+
   private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) {
     DbSession dbSession = db.getSession();
     Collection<EsQueueDto> items = underTest.prepareForRecovery(dbSession, singletonList(project.uuid()), cause);
index cb6a69925b7bc1afe5a89a278d6ba300b71cceb3..e6f74bffca40f278e1fa985c01934fa4c13d5f60 100644 (file)
@@ -51,9 +51,9 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
   }
 
   @Override
-  public void indexOnAnalysis(String projectUuid) {
-    addToIndex(projectUuid, "bar");
-    addToIndex(projectUuid, "baz");
+  public void indexOnAnalysis(String branchUuid) {
+    addToIndex(branchUuid, "bar");
+    addToIndex(branchUuid, "baz");
   }
 
   @Override