SONAR-11139 Store changed lines information received in scanner report

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java

@@ -111,10 +111,12 @@ import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolderImpl;
 import org.sonar.ce.task.projectanalysis.scm.ScmInfoDbLoader;
 import org.sonar.ce.task.projectanalysis.scm.ScmInfoRepositoryImpl;
 import org.sonar.ce.task.projectanalysis.source.DbLineHashVersion;
+import org.sonar.ce.task.projectanalysis.source.FileSourceDataComputer;
 import org.sonar.ce.task.projectanalysis.source.LastCommitVisitor;
 import org.sonar.ce.task.projectanalysis.source.NewLinesRepository;
 import org.sonar.ce.task.projectanalysis.source.SignificantCodeRepository;
 import org.sonar.ce.task.projectanalysis.source.SourceHashRepositoryImpl;
+import org.sonar.ce.task.projectanalysis.source.SourceLineReadersFactory;
 import org.sonar.ce.task.projectanalysis.source.SourceLinesDiffImpl;
 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashCache;
 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepositoryImpl;
@@ -207,6 +209,8 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop
       SignificantCodeRepository.class,
       SourceLinesHashCache.class,
       NewLinesRepository.class,
+      FileSourceDataComputer.class,
+      SourceLineReadersFactory.class,
 
       // issues
       RuleRepositoryImpl.class,

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/FileSourceDataComputer.java

@@ -34,7 +34,8 @@ public class FileSourceDataComputer {
   private final SourceLinesHashRepository sourceLinesHash;
   private final SourceHashComputer sourceHashComputer;
 
-  public FileSourceDataComputer(SourceLinesRepository sourceLinesRepository, SourceLineReadersFactory sourceLineReadersFactory, SourceLinesHashRepository sourceLinesHash) {
+  public FileSourceDataComputer(SourceLinesRepository sourceLinesRepository, SourceLineReadersFactory sourceLineReadersFactory,
+    SourceLinesHashRepository sourceLinesHash) {
     this.sourceLinesRepository = sourceLinesRepository;
     this.sourceLineReadersFactory = sourceLineReadersFactory;
     this.sourceLinesHash = sourceLinesHash;
@@ -44,7 +45,6 @@ public class FileSourceDataComputer {
   public Data compute(Component file) {
     try (CloseableIterator<String> linesIterator = sourceLinesRepository.readLines(file);
       SourceLineReadersFactory.LineReaders lineReaders = sourceLineReadersFactory.getLineReaders(file)) {
-
       SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = sourceLinesHash.getLineHashesComputerToPersist(file);
       DbFileSources.Data.Builder fileSourceBuilder = DbFileSources.Data.newBuilder();
       int currentLine = 0;
@@ -54,7 +54,8 @@ public class FileSourceDataComputer {
         read(fileSourceBuilder, lineHashesComputer, lineReaders, currentLine, linesIterator.next(), linesIterator.hasNext());
       }
 
-      return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash(), lineReaders.getLatestChangeWithRevision());
+      Changeset latestChangeWithRevision = lineReaders.getLatestChangeWithRevision();
+      return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash(), latestChangeWithRevision);
     }
   }
 

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceHashRepository.java

@@ -19,7 +19,6 @@
  */
 package org.sonar.ce.task.projectanalysis.source;
 
-import org.sonar.ce.task.projectanalysis.component.Component;
 import org.sonar.ce.task.projectanalysis.component.Component;
 
 public interface SourceHashRepository {

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactory.java

@@ -33,6 +33,7 @@ import org.sonar.ce.task.projectanalysis.scm.ScmInfoRepository;
 import org.sonar.ce.task.projectanalysis.source.linereader.CoverageLineReader;
 import org.sonar.ce.task.projectanalysis.source.linereader.DuplicationLineReader;
 import org.sonar.ce.task.projectanalysis.source.linereader.HighlightingLineReader;
+import org.sonar.ce.task.projectanalysis.source.linereader.IsNewLineReader;
 import org.sonar.ce.task.projectanalysis.source.linereader.LineReader;
 import org.sonar.ce.task.projectanalysis.source.linereader.RangeOffsetConverter;
 import org.sonar.ce.task.projectanalysis.source.linereader.ScmLineReader;
@@ -45,11 +46,14 @@ public class SourceLineReadersFactory {
   private final BatchReportReader reportReader;
   private final ScmInfoRepository scmInfoRepository;
   private final DuplicationRepository duplicationRepository;
+  private final NewLinesRepository newLinesRepository;
 
-  SourceLineReadersFactory(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository) {
+  public SourceLineReadersFactory(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository,
+    NewLinesRepository newLinesRepository) {
     this.reportReader = reportReader;
     this.scmInfoRepository = scmInfoRepository;
     this.duplicationRepository = duplicationRepository;
+    this.newLinesRepository = newLinesRepository;
   }
 
   public LineReaders getLineReaders(Component component) {
@@ -77,6 +81,7 @@ public class SourceLineReadersFactory {
     closeables.add(symbolsIt);
     readers.add(new SymbolsLineReader(component, symbolsIt, rangeOffsetConverter));
     readers.add(new DuplicationLineReader(duplicationRepository.getDuplications(component)));
+    readers.add(new IsNewLineReader(newLinesRepository, component));
 
     return new LineReaders(readers, scmLineReader, closeables);
   }

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLinesDiffImpl.java

@@ -25,7 +25,6 @@ import org.sonar.ce.task.projectanalysis.component.Component;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.source.FileSourceDao;
-import org.sonar.ce.task.projectanalysis.component.Component;
 
 public class SourceLinesDiffImpl implements SourceLinesDiff {
 

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/DuplicationLineReader.java

@@ -33,9 +33,6 @@ import org.sonar.ce.task.projectanalysis.duplication.Duplication;
 import org.sonar.ce.task.projectanalysis.duplication.InnerDuplicate;
 import org.sonar.ce.task.projectanalysis.duplication.TextBlock;
 import org.sonar.db.protobuf.DbFileSources;
-import org.sonar.ce.task.projectanalysis.duplication.Duplication;
-import org.sonar.ce.task.projectanalysis.duplication.InnerDuplicate;
-import org.sonar.ce.task.projectanalysis.duplication.TextBlock;
 
 import static com.google.common.collect.FluentIterable.from;
 import static com.google.common.collect.Iterables.size;

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReader.java

@@ -0,0 +1,38 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.source.linereader;
+
+import java.util.Collections;
+import java.util.Set;
+import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.source.NewLinesRepository;
+import org.sonar.db.protobuf.DbFileSources;
+
+public class IsNewLineReader implements LineReader {
+  private final Set<Integer> newLines;
+
+  public IsNewLineReader(NewLinesRepository newLinesRepository, Component file) {
+    this.newLines = newLinesRepository.getNewLines(file).orElse(Collections.emptySet());
+  }
+
+  @Override public void read(DbFileSources.Line.Builder lineBuilder) {
+    lineBuilder.setIsNewLine(newLines.contains(lineBuilder.getLine()));
+  }
+}

server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistFileSourcesStep.java

@@ -95,20 +95,21 @@ public class PersistFileSourcesStep implements ComputationStep {
     public void visitFile(Component file) {
       try {
         FileSourceDataComputer.Data fileSourceData = fileSourceDataComputer.compute(file);
-        persistSource(fileSourceData, file, fileSourceData.getLatestChangeWithRevision());
+        persistSource(fileSourceData, file);
       } catch (Exception e) {
         throw new IllegalStateException(String.format("Cannot persist sources of %s", file.getKey()), e);
       }
     }
 
-    private void persistSource(FileSourceDataComputer.Data fileSourceData, Component file, @Nullable Changeset latestChangeWithRevision) {
+    private void persistSource(FileSourceDataComputer.Data fileSourceData, Component file) {
       DbFileSources.Data lineData = fileSourceData.getLineData();
 
       byte[] binaryData = FileSourceDto.encodeSourceData(lineData);
       String dataHash = DigestUtils.md5Hex(binaryData);
       String srcHash = fileSourceData.getSrcHash();
       List<String> lineHashes = fileSourceData.getLineHashes();
-      Integer lineHashesVersion = sourceLinesHash.getLineHashesVersion(file);
+      Changeset latestChangeWithRevision = fileSourceData.getLatestChangeWithRevision();
+      int lineHashesVersion = sourceLinesHash.getLineHashesVersion(file);
       FileSourceDto previousDto = previousFileSourcesByUuid.get(file.getUuid());
 
       if (previousDto == null) {

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactoryTest.java

@@ -60,12 +60,13 @@ public class SourceLineReadersFactoryTest {
   public ScmInfoRepositoryRule scmInfoRepository = new ScmInfoRepositoryRule();
   @Rule
   public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder);
+  private NewLinesRepository newLinesRepository = mock(NewLinesRepository.class);
 
   private SourceLineReadersFactory underTest;
 
   @Before
   public void setUp() {
-    underTest = new SourceLineReadersFactory(reportReader, scmInfoRepository, duplicationRepository);
+    underTest = new SourceLineReadersFactory(reportReader, scmInfoRepository, duplicationRepository, newLinesRepository);
   }
 
   @Test
@@ -75,7 +76,7 @@ public class SourceLineReadersFactoryTest {
 
     assertThat(lineReaders).isNotNull();
     assertThat(lineReaders.closeables).hasSize(3);
-    assertThat(lineReaders.readers).hasSize(4);
+    assertThat(lineReaders.readers).hasSize(5);
   }
 
   @Test

server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReaderTest.java

@@ -0,0 +1,74 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.source.linereader;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import org.junit.Test;
+import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.source.NewLinesRepository;
+import org.sonar.db.protobuf.DbFileSources;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IsNewLineReaderTest {
+  private NewLinesRepository repository = mock(NewLinesRepository.class);
+  private Component component = mock(Component.class);
+
+  @Test
+  public void should_set_isNewLines_in_builder() {
+    Set<Integer> newLines = new HashSet<>(Arrays.asList(1, 3));
+    when(repository.getNewLines(component)).thenReturn(Optional.of(newLines));
+
+    IsNewLineReader reader = new IsNewLineReader(repository, component);
+    DbFileSources.Line.Builder[] builders = runReader(reader);
+
+    assertThat(builders[0].getIsNewLine()).isTrue();
+    assertThat(builders[1].getIsNewLine()).isFalse();
+    assertThat(builders[2].getIsNewLine()).isTrue();
+  }
+
+  @Test
+  public void should_set_isNewLines_false_if_no_new_lines_available() {
+    when(repository.getNewLines(component)).thenReturn(Optional.empty());
+
+    IsNewLineReader reader = new IsNewLineReader(repository, component);
+    DbFileSources.Line.Builder[] builders = runReader(reader);
+
+    assertThat(builders[0].getIsNewLine()).isFalse();
+    assertThat(builders[1].getIsNewLine()).isFalse();
+    assertThat(builders[2].getIsNewLine()).isFalse();
+  }
+
+  private DbFileSources.Line.Builder[] runReader(LineReader reader) {
+    DbFileSources.Line.Builder[] builders = new DbFileSources.Line.Builder[3];
+    for (int i = 1; i <= 3; i++) {
+      builders[i - 1] = DbFileSources.Line.newBuilder()
+        .setLine(i);
+      reader.read(builders[i - 1]);
+    }
+
+    return builders;
+  }
+}

server/sonar-db-dao/src/main/protobuf/db-file-sources.proto

@@ -60,6 +60,8 @@ message Line {
   optional int32 line_hits = 18;
   optional int32 conditions = 19;
   optional int32 covered_conditions = 20;
+
+  optional bool is_new_line = 21;
 }
 
 message Range {