From 63b270ad490104a724d1938334877eb3c9ba261d Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Wed, 15 Aug 2018 14:17:06 +0200 Subject: [PATCH] SONAR-11139 Store changed lines information received in scanner report --- ...ProjectAnalysisTaskContainerPopulator.java | 4 + .../source/FileSourceDataComputer.java | 7 +- .../source/SourceHashRepository.java | 1 - .../source/SourceLineReadersFactory.java | 7 +- .../source/SourceLinesDiffImpl.java | 1 - .../linereader/DuplicationLineReader.java | 3 - .../source/linereader/IsNewLineReader.java | 38 ++++++++++ .../step/PersistFileSourcesStep.java | 7 +- .../source/SourceLineReadersFactoryTest.java | 5 +- .../linereader/IsNewLineReaderTest.java | 74 +++++++++++++++++++ .../src/main/protobuf/db-file-sources.proto | 2 + 11 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReader.java create mode 100644 server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReaderTest.java diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index 39911cc9047..4a0eb5c372f 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -111,10 +111,12 @@ import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolderImpl; import org.sonar.ce.task.projectanalysis.scm.ScmInfoDbLoader; import org.sonar.ce.task.projectanalysis.scm.ScmInfoRepositoryImpl; import org.sonar.ce.task.projectanalysis.source.DbLineHashVersion; +import org.sonar.ce.task.projectanalysis.source.FileSourceDataComputer; import org.sonar.ce.task.projectanalysis.source.LastCommitVisitor; import org.sonar.ce.task.projectanalysis.source.NewLinesRepository; import org.sonar.ce.task.projectanalysis.source.SignificantCodeRepository; import org.sonar.ce.task.projectanalysis.source.SourceHashRepositoryImpl; +import org.sonar.ce.task.projectanalysis.source.SourceLineReadersFactory; import org.sonar.ce.task.projectanalysis.source.SourceLinesDiffImpl; import org.sonar.ce.task.projectanalysis.source.SourceLinesHashCache; import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepositoryImpl; @@ -207,6 +209,8 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop SignificantCodeRepository.class, SourceLinesHashCache.class, NewLinesRepository.class, + FileSourceDataComputer.class, + SourceLineReadersFactory.class, // issues RuleRepositoryImpl.class, diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/FileSourceDataComputer.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/FileSourceDataComputer.java index de2ebfe4b5f..b560a23f239 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/FileSourceDataComputer.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/FileSourceDataComputer.java @@ -34,7 +34,8 @@ public class FileSourceDataComputer { private final SourceLinesHashRepository sourceLinesHash; private final SourceHashComputer sourceHashComputer; - public FileSourceDataComputer(SourceLinesRepository sourceLinesRepository, SourceLineReadersFactory sourceLineReadersFactory, SourceLinesHashRepository sourceLinesHash) { + public FileSourceDataComputer(SourceLinesRepository sourceLinesRepository, SourceLineReadersFactory sourceLineReadersFactory, + SourceLinesHashRepository sourceLinesHash) { this.sourceLinesRepository = sourceLinesRepository; this.sourceLineReadersFactory = sourceLineReadersFactory; this.sourceLinesHash = sourceLinesHash; @@ -44,7 +45,6 @@ public class FileSourceDataComputer { public Data compute(Component file) { try (CloseableIterator linesIterator = sourceLinesRepository.readLines(file); SourceLineReadersFactory.LineReaders lineReaders = sourceLineReadersFactory.getLineReaders(file)) { - SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = sourceLinesHash.getLineHashesComputerToPersist(file); DbFileSources.Data.Builder fileSourceBuilder = DbFileSources.Data.newBuilder(); int currentLine = 0; @@ -54,7 +54,8 @@ public class FileSourceDataComputer { read(fileSourceBuilder, lineHashesComputer, lineReaders, currentLine, linesIterator.next(), linesIterator.hasNext()); } - return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash(), lineReaders.getLatestChangeWithRevision()); + Changeset latestChangeWithRevision = lineReaders.getLatestChangeWithRevision(); + return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash(), latestChangeWithRevision); } } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceHashRepository.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceHashRepository.java index 1c19ae71bc8..c0497bb15ff 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceHashRepository.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceHashRepository.java @@ -19,7 +19,6 @@ */ package org.sonar.ce.task.projectanalysis.source; -import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.ce.task.projectanalysis.component.Component; public interface SourceHashRepository { diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactory.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactory.java index 82f3858e4fd..eb22f1ca4cb 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactory.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactory.java @@ -33,6 +33,7 @@ import org.sonar.ce.task.projectanalysis.scm.ScmInfoRepository; import org.sonar.ce.task.projectanalysis.source.linereader.CoverageLineReader; import org.sonar.ce.task.projectanalysis.source.linereader.DuplicationLineReader; import org.sonar.ce.task.projectanalysis.source.linereader.HighlightingLineReader; +import org.sonar.ce.task.projectanalysis.source.linereader.IsNewLineReader; import org.sonar.ce.task.projectanalysis.source.linereader.LineReader; import org.sonar.ce.task.projectanalysis.source.linereader.RangeOffsetConverter; import org.sonar.ce.task.projectanalysis.source.linereader.ScmLineReader; @@ -45,11 +46,14 @@ public class SourceLineReadersFactory { private final BatchReportReader reportReader; private final ScmInfoRepository scmInfoRepository; private final DuplicationRepository duplicationRepository; + private final NewLinesRepository newLinesRepository; - SourceLineReadersFactory(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository) { + public SourceLineReadersFactory(BatchReportReader reportReader, ScmInfoRepository scmInfoRepository, DuplicationRepository duplicationRepository, + NewLinesRepository newLinesRepository) { this.reportReader = reportReader; this.scmInfoRepository = scmInfoRepository; this.duplicationRepository = duplicationRepository; + this.newLinesRepository = newLinesRepository; } public LineReaders getLineReaders(Component component) { @@ -77,6 +81,7 @@ public class SourceLineReadersFactory { closeables.add(symbolsIt); readers.add(new SymbolsLineReader(component, symbolsIt, rangeOffsetConverter)); readers.add(new DuplicationLineReader(duplicationRepository.getDuplications(component))); + readers.add(new IsNewLineReader(newLinesRepository, component)); return new LineReaders(readers, scmLineReader, closeables); } diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLinesDiffImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLinesDiffImpl.java index 80cf01baaa4..4c421508784 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLinesDiffImpl.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/SourceLinesDiffImpl.java @@ -25,7 +25,6 @@ import org.sonar.ce.task.projectanalysis.component.Component; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.source.FileSourceDao; -import org.sonar.ce.task.projectanalysis.component.Component; public class SourceLinesDiffImpl implements SourceLinesDiff { diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/DuplicationLineReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/DuplicationLineReader.java index 985dbcdb2b1..6b885074ef0 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/DuplicationLineReader.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/DuplicationLineReader.java @@ -33,9 +33,6 @@ import org.sonar.ce.task.projectanalysis.duplication.Duplication; import org.sonar.ce.task.projectanalysis.duplication.InnerDuplicate; import org.sonar.ce.task.projectanalysis.duplication.TextBlock; import org.sonar.db.protobuf.DbFileSources; -import org.sonar.ce.task.projectanalysis.duplication.Duplication; -import org.sonar.ce.task.projectanalysis.duplication.InnerDuplicate; -import org.sonar.ce.task.projectanalysis.duplication.TextBlock; import static com.google.common.collect.FluentIterable.from; import static com.google.common.collect.Iterables.size; diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReader.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReader.java new file mode 100644 index 00000000000..f4d158c84ae --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReader.java @@ -0,0 +1,38 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.source.linereader; + +import java.util.Collections; +import java.util.Set; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.source.NewLinesRepository; +import org.sonar.db.protobuf.DbFileSources; + +public class IsNewLineReader implements LineReader { + private final Set newLines; + + public IsNewLineReader(NewLinesRepository newLinesRepository, Component file) { + this.newLines = newLinesRepository.getNewLines(file).orElse(Collections.emptySet()); + } + + @Override public void read(DbFileSources.Line.Builder lineBuilder) { + lineBuilder.setIsNewLine(newLines.contains(lineBuilder.getLine())); + } +} diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistFileSourcesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistFileSourcesStep.java index 50c61d5f941..2ad2e2a72f8 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistFileSourcesStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistFileSourcesStep.java @@ -95,20 +95,21 @@ public class PersistFileSourcesStep implements ComputationStep { public void visitFile(Component file) { try { FileSourceDataComputer.Data fileSourceData = fileSourceDataComputer.compute(file); - persistSource(fileSourceData, file, fileSourceData.getLatestChangeWithRevision()); + persistSource(fileSourceData, file); } catch (Exception e) { throw new IllegalStateException(String.format("Cannot persist sources of %s", file.getKey()), e); } } - private void persistSource(FileSourceDataComputer.Data fileSourceData, Component file, @Nullable Changeset latestChangeWithRevision) { + private void persistSource(FileSourceDataComputer.Data fileSourceData, Component file) { DbFileSources.Data lineData = fileSourceData.getLineData(); byte[] binaryData = FileSourceDto.encodeSourceData(lineData); String dataHash = DigestUtils.md5Hex(binaryData); String srcHash = fileSourceData.getSrcHash(); List lineHashes = fileSourceData.getLineHashes(); - Integer lineHashesVersion = sourceLinesHash.getLineHashesVersion(file); + Changeset latestChangeWithRevision = fileSourceData.getLatestChangeWithRevision(); + int lineHashesVersion = sourceLinesHash.getLineHashesVersion(file); FileSourceDto previousDto = previousFileSourcesByUuid.get(file.getUuid()); if (previousDto == null) { diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactoryTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactoryTest.java index 2942a4ba617..052baadae3d 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactoryTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/SourceLineReadersFactoryTest.java @@ -60,12 +60,13 @@ public class SourceLineReadersFactoryTest { public ScmInfoRepositoryRule scmInfoRepository = new ScmInfoRepositoryRule(); @Rule public DuplicationRepositoryRule duplicationRepository = DuplicationRepositoryRule.create(treeRootHolder); + private NewLinesRepository newLinesRepository = mock(NewLinesRepository.class); private SourceLineReadersFactory underTest; @Before public void setUp() { - underTest = new SourceLineReadersFactory(reportReader, scmInfoRepository, duplicationRepository); + underTest = new SourceLineReadersFactory(reportReader, scmInfoRepository, duplicationRepository, newLinesRepository); } @Test @@ -75,7 +76,7 @@ public class SourceLineReadersFactoryTest { assertThat(lineReaders).isNotNull(); assertThat(lineReaders.closeables).hasSize(3); - assertThat(lineReaders.readers).hasSize(4); + assertThat(lineReaders.readers).hasSize(5); } @Test diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReaderTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReaderTest.java new file mode 100644 index 00000000000..e21bfb846d5 --- /dev/null +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/source/linereader/IsNewLineReaderTest.java @@ -0,0 +1,74 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.ce.task.projectanalysis.source.linereader; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.junit.Test; +import org.sonar.ce.task.projectanalysis.component.Component; +import org.sonar.ce.task.projectanalysis.source.NewLinesRepository; +import org.sonar.db.protobuf.DbFileSources; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IsNewLineReaderTest { + private NewLinesRepository repository = mock(NewLinesRepository.class); + private Component component = mock(Component.class); + + @Test + public void should_set_isNewLines_in_builder() { + Set newLines = new HashSet<>(Arrays.asList(1, 3)); + when(repository.getNewLines(component)).thenReturn(Optional.of(newLines)); + + IsNewLineReader reader = new IsNewLineReader(repository, component); + DbFileSources.Line.Builder[] builders = runReader(reader); + + assertThat(builders[0].getIsNewLine()).isTrue(); + assertThat(builders[1].getIsNewLine()).isFalse(); + assertThat(builders[2].getIsNewLine()).isTrue(); + } + + @Test + public void should_set_isNewLines_false_if_no_new_lines_available() { + when(repository.getNewLines(component)).thenReturn(Optional.empty()); + + IsNewLineReader reader = new IsNewLineReader(repository, component); + DbFileSources.Line.Builder[] builders = runReader(reader); + + assertThat(builders[0].getIsNewLine()).isFalse(); + assertThat(builders[1].getIsNewLine()).isFalse(); + assertThat(builders[2].getIsNewLine()).isFalse(); + } + + private DbFileSources.Line.Builder[] runReader(LineReader reader) { + DbFileSources.Line.Builder[] builders = new DbFileSources.Line.Builder[3]; + for (int i = 1; i <= 3; i++) { + builders[i - 1] = DbFileSources.Line.newBuilder() + .setLine(i); + reader.read(builders[i - 1]); + } + + return builders; + } +} diff --git a/server/sonar-db-dao/src/main/protobuf/db-file-sources.proto b/server/sonar-db-dao/src/main/protobuf/db-file-sources.proto index e330e536763..bad663a92d0 100644 --- a/server/sonar-db-dao/src/main/protobuf/db-file-sources.proto +++ b/server/sonar-db-dao/src/main/protobuf/db-file-sources.proto @@ -60,6 +60,8 @@ message Line { optional int32 line_hits = 18; optional int32 conditions = 19; optional int32 covered_conditions = 20; + + optional bool is_new_line = 21; } message Range { -- 2.39.5