SONAR-9621 Read cpd text blocks from DB in incremental mode

10 files changed, 166 insertions, 53 deletions

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationDao.java
index 8883f303e03..dc4f8e1133a 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationDao.java
@@ -45,5 +45,12 @@ public class DuplicationDao implements Dao {
   public void insert(DbSession session, DuplicationUnitDto dto) {
     session.getMapper(DuplicationMapper.class).batchInsert(dto);
   }
+  
+  /**
+   * @param componentUUid uuid of the component
+   */
+  public List<DuplicationUnitDto> selectComponent(DbSession session, String componentUuid, String analysisUuid) {
+    return session.getMapper(DuplicationMapper.class).selectComponent(componentUuid, analysisUuid);
+  }
 
 }
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationMapper.java
index 8edec655644..109caebb30a 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/duplication/DuplicationMapper.java
@@ -33,4 +33,5 @@ public interface DuplicationMapper {
 
   void batchInsert(DuplicationUnitDto unit);
 
+  List<DuplicationUnitDto> selectComponent(@Param("componentUuid") String componentUuid, @Param("analysisUuid") String analysisUuid);
 }
diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml
index 21135fa3248..d24950c4c99 100644
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml
@@ -25,6 +25,21 @@
       </if>
     </where>
   </select>
+  
+  <select id="selectComponent" parameterType="map" resultType="DuplicationUnit">
+    SELECT DISTINCT
+    dup.id as id,
+    dup.analysis_uuid as analysisUuid,
+    dup.component_uuid as componentUuid,
+    dup.hash as hash,
+    dup.index_in_file as indexInFile,
+    dup.start_line as startLine,
+    dup.end_line as endLine
+    FROM duplications_index dup
+    WHERE
+      dup.component_uuid = #{componentUuid,jdbcType=VARCHAR} AND
+      dup.analysis_uuid = #{analysisUuid,jdbcType=VARCHAR} 
+  </select>
 
   <insert id="batchInsert" parameterType="DuplicationUnit" useGeneratedKeys="false">
     INSERT INTO duplications_index (
diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java
index 9c027764c2f..0c685ed1f97 100644
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java
@@ -29,7 +29,6 @@ import org.sonar.db.DbTester;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 
-
 public class DuplicationDaoTest {
 
   @Rule
@@ -61,6 +60,25 @@ public class DuplicationDaoTest {
   }
 
   @Test
+  public void select_component() {
+    db.prepareDbUnit(getClass(), "select_component.xml");
+    dbSession.commit();
+
+    List<DuplicationUnitDto> blocks = dao.selectComponent(dbSession, "uuid_3", "u5");
+    assertThat(blocks).hasSize(1);
+
+    DuplicationUnitDto block = blocks.get(0);
+    assertThat(block.getComponentKey()).isNull();
+    assertThat(block.getComponentUuid()).isEqualTo("uuid_3");
+    assertThat(block.getHash()).isEqualTo("bb");
+    assertThat(block.getAnalysisUuid()).isEqualTo("u5");
+    assertThat(block.getIndexInFile()).isEqualTo(0);
+    assertThat(block.getStartLine()).isEqualTo(0);
+    assertThat(block.getEndLine()).isEqualTo(0);
+
+  }
+
+  @Test
   public void insert() {
     db.prepareDbUnit(getClass(), "insert.xml");
     dbSession.commit();
diff --git a/server/sonar-db-dao/src/test/resources/org/sonar/db/duplication/DuplicationDaoTest/select_component.xml b/server/sonar-db-dao/src/test/resources/org/sonar/db/duplication/DuplicationDaoTest/select_component.xml
new file mode 100644
index 00000000000..b31be796646
--- /dev/null
+++ b/server/sonar-db-dao/src/test/resources/org/sonar/db/duplication/DuplicationDaoTest/select_component.xml
@@ -0,0 +1,33 @@
+<dataset>
+  <duplications_index id="1"
+                      analysis_uuid="u1"
+                      component_uuid="uuid_1"
+                      hash="bb"
+                      index_in_file="0"
+                      start_line="0"
+                      end_line="0"/>
+
+  <duplications_index id="2"
+                      analysis_uuid="u3"
+                      component_uuid="uuid_2"
+                      hash="aa"
+                      index_in_file="0"
+                      start_line="1"
+                      end_line="2"/>
+
+  <duplications_index id="3"
+                      analysis_uuid="u5"
+                      component_uuid="uuid_3"
+                      hash="bb"
+                      index_in_file="0"
+                      start_line="0"
+                      end_line="0"/>
+
+  <duplications_index id="4"
+                      analysis_uuid="u5"
+                      component_uuid="uuid_4"
+                      hash="aa"
+                      index_in_file="0"
+                      start_line="0"
+                      end_line="0"/>
+</dataset>
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IncrementalDuplicationsStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IncrementalDuplicationsStep.java
deleted file mode 100644
index bff28106be8..00000000000
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/IncrementalDuplicationsStep.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.computation.task.projectanalysis.step;
-
-import org.sonar.server.computation.task.step.ComputationStep;
-
-public class IncrementalDuplicationsStep implements ComputationStep {
-
-  @Override
-  public void execute() {
-    // TODO Auto-generated method stub
-    
-  }
-
-  @Override
-  public String getDescription() {
-    // TODO Auto-generated method stub
-    return null;
-  }
-
-}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStep.java
index 7dc928e7c1f..2a44cf87bce 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStep.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStep.java
@@ -24,6 +24,7 @@ import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.duplication.DuplicationUnitDto;
 import org.sonar.scanner.protocol.output.ScannerReport;
+import org.sonar.server.computation.task.projectanalysis.analysis.Analysis;
 import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolder;
 import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReader;
 import org.sonar.server.computation.task.projectanalysis.component.Component;
@@ -31,11 +32,16 @@ import org.sonar.server.computation.task.projectanalysis.component.CrawlerDepthL
 import org.sonar.server.computation.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
 import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolder;
 import org.sonar.server.computation.task.projectanalysis.component.TypeAwareVisitorAdapter;
+import org.sonar.server.computation.task.projectanalysis.component.Component.Status;
 import org.sonar.server.computation.task.projectanalysis.duplication.CrossProjectDuplicationStatusHolder;
 import org.sonar.server.computation.task.step.ComputationStep;
 
 import static org.sonar.server.computation.task.projectanalysis.component.ComponentVisitor.Order.PRE_ORDER;
 
+import java.util.List;
+
+import javax.annotation.Nullable;
+
 /**
  * Persist cross project duplications text blocks into DUPLICATIONS_INDEX table
  */
@@ -65,7 +71,9 @@ public class PersistCrossProjectDuplicationIndexStep implements ComputationStep
 
     try (DbSession dbSession = dbClient.openSession(true)) {
       Component project = treeRootHolder.getRoot();
-      new DepthTraversalTypeAwareCrawler(new DuplicationVisitor(dbSession, analysisMetadataHolder.getUuid())).visit(project);
+      Analysis baseAnalysis = analysisMetadataHolder.getBaseAnalysis();
+      String lastAnalysysUuid = (baseAnalysis != null) ? baseAnalysis.getUuid() : null;
+      new DepthTraversalTypeAwareCrawler(new DuplicationVisitor(dbSession, analysisMetadataHolder.getUuid(), lastAnalysysUuid)).visit(project);
       dbSession.commit();
     }
   }
@@ -74,11 +82,13 @@ public class PersistCrossProjectDuplicationIndexStep implements ComputationStep
 
     private final DbSession session;
     private final String analysisUuid;
+    private final String lastAnalysisUuid;
 
-    private DuplicationVisitor(DbSession session, String analysisUuid) {
+    private DuplicationVisitor(DbSession session, String analysisUuid, @Nullable String lastAnalysisUuid) {
       super(CrawlerDepthLimit.FILE, PRE_ORDER);
       this.session = session;
       this.analysisUuid = analysisUuid;
+      this.lastAnalysisUuid = lastAnalysisUuid;
     }
 
     @Override
@@ -87,6 +97,14 @@ public class PersistCrossProjectDuplicationIndexStep implements ComputationStep
     }
 
     private void visitComponent(Component component) {
+      if (component.getStatus() == Status.SAME) {
+        readFromDb(component);
+      } else {
+        readFromReport(component);
+      }
+    }
+
+    private void readFromReport(Component component) {
       int indexInFile = 0;
       CloseableIterator<ScannerReport.CpdTextBlock> blocks = reportReader.readCpdTextBlocks(component.getReportAttributes().getRef());
       try {
@@ -107,6 +125,17 @@ public class PersistCrossProjectDuplicationIndexStep implements ComputationStep
         blocks.close();
       }
     }
+
+    private void readFromDb(Component component) {
+      int indexInFile = 0;
+      List<DuplicationUnitDto> units = dbClient.duplicationDao().selectComponent(session, component.getUuid(), lastAnalysisUuid);
+      for (DuplicationUnitDto unit : units) {
+        unit.setAnalysisUuid(analysisUuid);
+        unit.setIndexInFile(indexInFile);
+        dbClient.duplicationDao().insert(session, unit);
+        indexInFile++;
+      }
+    }
   }
 
   @Override
diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ReportDuplicationMeasuresStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/duplication/ReportDuplicationMeasuresTest.java
index 7c5e0a66307..7c5e0a66307 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ReportDuplicationMeasuresStepTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/duplication/ReportDuplicationMeasuresTest.java
diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ViewsDuplicationMeasuresStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/duplication/ViewsDuplicationMeasuresTest.java
index 9c4213b30b4..9c4213b30b4 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/ViewsDuplicationMeasuresStepTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/duplication/ViewsDuplicationMeasuresTest.java
diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStepTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStepTest.java
index 8a63ed33a97..a10bc797231 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStepTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/step/PersistCrossProjectDuplicationIndexStepTest.java
@@ -26,13 +26,19 @@ import java.util.Map;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
+import org.sonar.db.duplication.DuplicationUnitDto;
 import org.sonar.scanner.protocol.output.ScannerReport;
+import org.sonar.server.computation.task.projectanalysis.analysis.Analysis;
 import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
 import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReaderRule;
 import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolderRule;
+import org.sonar.server.computation.task.projectanalysis.component.Component.Status;
 import org.sonar.server.computation.task.projectanalysis.component.Component;
 import org.sonar.server.computation.task.projectanalysis.component.ReportComponent;
 import org.sonar.server.computation.task.projectanalysis.duplication.CrossProjectDuplicationStatusHolder;
@@ -40,16 +46,21 @@ import org.sonar.server.computation.task.step.ComputationStep;
 
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class PersistCrossProjectDuplicationIndexStepTest {
 
-  private static final int FILE_REF = 2;
-  private static final Component FILE = ReportComponent.builder(Component.Type.FILE, FILE_REF).build();
+  private static final int FILE_1_REF = 2;
+  private static final int FILE_2_REF = 3;
+  private static final String FILE_2_UUID = "file2";
+
+  private static final Component FILE_1 = ReportComponent.builder(Component.Type.FILE, FILE_1_REF).build();
+  private static final Component FILE_2 = ReportComponent.builder(Component.Type.FILE, FILE_2_REF)
+    .setStatus(Status.SAME).setUuid(FILE_2_UUID).build();
 
   private static final Component PROJECT = ReportComponent.builder(Component.Type.PROJECT, 1)
-    .addChildren(FILE)
+    .addChildren(FILE_1)
+    .addChildren(FILE_2)
     .build();
 
   private static final ScannerReport.CpdTextBlock CPD_TEXT_BLOCK = ScannerReport.CpdTextBlock.newBuilder()
@@ -58,6 +69,7 @@ public class PersistCrossProjectDuplicationIndexStepTest {
     .setEndLine(45)
     .build();
   private static final String ANALYSIS_UUID = "analysis uuid";
+  private static final String BASE_ANALYSIS_UUID = "base analysis uuid";
 
   @Rule
   public DbTester dbTester = DbTester.create(System2.INSTANCE);
@@ -68,21 +80,57 @@ public class PersistCrossProjectDuplicationIndexStepTest {
   @Rule
   public AnalysisMetadataHolderRule analysisMetadataHolder = new AnalysisMetadataHolderRule();
 
-  CrossProjectDuplicationStatusHolder crossProjectDuplicationStatusHolder = mock(CrossProjectDuplicationStatusHolder.class);
+  @Mock
+  CrossProjectDuplicationStatusHolder crossProjectDuplicationStatusHolder;
+  @Mock
+  Analysis baseAnalysis;
 
   DbClient dbClient = dbTester.getDbClient();
 
-  ComputationStep underTest = new PersistCrossProjectDuplicationIndexStep(crossProjectDuplicationStatusHolder, dbClient, treeRootHolder, analysisMetadataHolder, reportReader);
+  ComputationStep underTest;
 
   @Before
   public void setUp() throws Exception {
+    MockitoAnnotations.initMocks(this);
+    when(baseAnalysis.getUuid()).thenReturn(BASE_ANALYSIS_UUID);
     analysisMetadataHolder.setUuid(ANALYSIS_UUID);
+    analysisMetadataHolder.setBaseAnalysis(baseAnalysis);
+    underTest = new PersistCrossProjectDuplicationIndexStep(crossProjectDuplicationStatusHolder, dbClient, treeRootHolder, analysisMetadataHolder, reportReader);
+  }
+
+  @Test
+  public void copy_base_analysis_in_incremental_mode() {
+    when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(true);
+    DuplicationUnitDto dup = new DuplicationUnitDto();
+    dup.setAnalysisUuid(BASE_ANALYSIS_UUID);
+    dup.setComponentUuid(FILE_2_UUID);
+    dup.setEndLine(0);
+    dup.setHash("asd");
+    dup.setStartLine(0);
+    dup.setId(1);
+    dup.setIndexInFile(1);
+    try (DbSession session = dbTester.getSession()) {
+      dbClient.duplicationDao().insert(session, dup);
+      session.commit();
+    }
+    assertThat(dbTester.countRowsOfTable("duplications_index")).isEqualTo(1);
+    underTest.execute();
+
+    Map<String, Object> dto = dbTester.selectFirst("select HASH, START_LINE, END_LINE, INDEX_IN_FILE, COMPONENT_UUID, ANALYSIS_UUID "
+      + "from duplications_index where analysis_uuid = '" + ANALYSIS_UUID + "'");
+    assertThat(dto.get("HASH")).isEqualTo("asd");
+    assertThat(dto.get("START_LINE")).isEqualTo(0L);
+    assertThat(dto.get("END_LINE")).isEqualTo(0L);
+    assertThat(dto.get("INDEX_IN_FILE")).isEqualTo(0L);
+    assertThat(dto.get("COMPONENT_UUID")).isEqualTo(FILE_2.getUuid());
+    assertThat(dto.get("ANALYSIS_UUID")).isEqualTo(ANALYSIS_UUID);
+
   }
 
   @Test
   public void persist_cpd_text_block() throws Exception {
     when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(true);
-    reportReader.putDuplicationBlocks(FILE_REF, singletonList(CPD_TEXT_BLOCK));
+    reportReader.putDuplicationBlocks(FILE_1_REF, singletonList(CPD_TEXT_BLOCK));
 
     underTest.execute();
 
@@ -91,14 +139,14 @@ public class PersistCrossProjectDuplicationIndexStepTest {
     assertThat(dto.get("START_LINE")).isEqualTo(30L);
     assertThat(dto.get("END_LINE")).isEqualTo(45L);
     assertThat(dto.get("INDEX_IN_FILE")).isEqualTo(0L);
-    assertThat(dto.get("COMPONENT_UUID")).isEqualTo(FILE.getUuid());
+    assertThat(dto.get("COMPONENT_UUID")).isEqualTo(FILE_1.getUuid());
     assertThat(dto.get("ANALYSIS_UUID")).isEqualTo(ANALYSIS_UUID);
   }
 
   @Test
   public void persist_many_cpd_text_blocks() throws Exception {
     when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(true);
-    reportReader.putDuplicationBlocks(FILE_REF, Arrays.asList(
+    reportReader.putDuplicationBlocks(FILE_1_REF, Arrays.asList(
       CPD_TEXT_BLOCK,
       ScannerReport.CpdTextBlock.newBuilder()
         .setHash("b1234353e96320ff")
@@ -113,14 +161,14 @@ public class PersistCrossProjectDuplicationIndexStepTest {
     assertThat(dtos).extracting("START_LINE").containsOnly(30L, 20L);
     assertThat(dtos).extracting("END_LINE").containsOnly(45L, 15L);
     assertThat(dtos).extracting("INDEX_IN_FILE").containsOnly(0L, 1L);
-    assertThat(dtos).extracting("COMPONENT_UUID").containsOnly(FILE.getUuid());
+    assertThat(dtos).extracting("COMPONENT_UUID").containsOnly(FILE_1.getUuid());
     assertThat(dtos).extracting("ANALYSIS_UUID").containsOnly(ANALYSIS_UUID);
   }
 
   @Test
   public void nothing_to_persist_when_no_cpd_text_blocks_in_report() throws Exception {
     when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(true);
-    reportReader.putDuplicationBlocks(FILE_REF, Collections.<ScannerReport.CpdTextBlock>emptyList());
+    reportReader.putDuplicationBlocks(FILE_1_REF, Collections.<ScannerReport.CpdTextBlock>emptyList());
 
     underTest.execute();
 
@@ -130,7 +178,7 @@ public class PersistCrossProjectDuplicationIndexStepTest {
   @Test
   public void nothing_to_do_when_cross_project_duplication_is_disabled() throws Exception {
     when(crossProjectDuplicationStatusHolder.isEnabled()).thenReturn(false);
-    reportReader.putDuplicationBlocks(FILE_REF, singletonList(CPD_TEXT_BLOCK));
+    reportReader.putDuplicationBlocks(FILE_1_REF, singletonList(CPD_TEXT_BLOCK));
 
     underTest.execute();