From ad83f03eefb1fcce357bfefc666db8e86174fcb1 Mon Sep 17 00:00:00 2001 From: Duarte Meneses Date: Fri, 12 Jan 2018 09:43:04 +0100 Subject: [PATCH] SONAR-10257 Generate SCM Info for changed lines --- ...ProjectAnalysisTaskContainerPopulator.java | 2 + .../task/projectanalysis/scm/DbScmInfo.java | 12 +- .../projectanalysis/scm/GeneratedScmInfo.java | 82 +++++ .../projectanalysis/scm/ScmInfoDbLoader.java | 25 +- .../task/projectanalysis/scm/ScmInfoImpl.java | 4 +- .../scm/ScmInfoRepositoryImpl.java | 126 ++++---- .../source/SourceLinesDiff.java | 27 ++ .../source/SourceLinesDiffFinder.java | 71 +++++ .../source/SourceLinesDiffImpl.java | 67 +++++ .../projectanalysis/scm/DbScmInfoTest.java | 20 +- .../scm/ScmInfoDbLoaderTest.java | 65 ++-- .../projectanalysis/scm/ScmInfoImplTest.java | 12 +- .../scm/ScmInfoRepositoryImplTest.java | 206 ++++++++++--- .../source/SourceLinesDiffFinderTest.java | 282 ++++++++++++++++++ .../source/SourceLinesDiffImplTest.java | 115 +++++++ 15 files changed, 927 insertions(+), 189 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/GeneratedScmInfo.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiff.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinder.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImpl.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinderTest.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImplTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java index 9b9f86676ed..f40d4dece13 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java @@ -114,6 +114,7 @@ import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoDbLoader; import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepositoryImpl; import org.sonar.server.computation.task.projectanalysis.source.LastCommitVisitor; import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepositoryImpl; +import org.sonar.server.computation.task.projectanalysis.source.SourceLinesDiffImpl; import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryImpl; import org.sonar.server.computation.task.projectanalysis.step.ReportComputationSteps; import org.sonar.server.computation.task.projectanalysis.step.SmallChangesetQualityGateSpecialCase; @@ -189,6 +190,7 @@ public final class ProjectAnalysisTaskContainerPopulator implements ContainerPop EvaluationResultTextConverterImpl.class, SourceLinesRepositoryImpl.class, SourceHashRepositoryImpl.class, + SourceLinesDiffImpl.class, ScmInfoRepositoryImpl.class, ScmInfoDbLoader.class, DuplicationRepositoryImpl.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfo.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfo.java index c8220e929cd..ff670adc8cf 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfo.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfo.java @@ -36,12 +36,14 @@ import org.sonar.db.protobuf.DbFileSources; class DbScmInfo implements ScmInfo { private final ScmInfo delegate; + private final String fileHash; - private DbScmInfo(ScmInfo delegate) { + private DbScmInfo(ScmInfo delegate, String fileHash) { this.delegate = delegate; + this.fileHash = fileHash; } - static Optional create(Iterable lines) { + public static Optional create(Iterable lines, String fileHash) { LineToChangeset lineToChangeset = new LineToChangeset(); Map lineChanges = new LinkedHashMap<>(); @@ -55,7 +57,11 @@ class DbScmInfo implements ScmInfo { if (lineChanges.isEmpty()) { return Optional.empty(); } - return Optional.of(new DbScmInfo(new ScmInfoImpl(lineChanges))); + return Optional.of(new DbScmInfo(new ScmInfoImpl(lineChanges), fileHash)); + } + + public String fileHash() { + return fileHash; } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/GeneratedScmInfo.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/GeneratedScmInfo.java new file mode 100644 index 00000000000..408895252a4 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/GeneratedScmInfo.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.scm; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkState; + +public class GeneratedScmInfo implements ScmInfo { + private final ScmInfoImpl delegate; + + public GeneratedScmInfo(Map changesets) { + delegate = new ScmInfoImpl(changesets); + } + + public static ScmInfo create(long analysisDate, Set lines) { + checkState(!lines.isEmpty(), "No changesets"); + + Changeset changeset = Changeset.newChangesetBuilder() + .setDate(analysisDate) + .build(); + Map changesets = lines.stream() + .collect(Collectors.toMap(x -> x, i -> changeset)); + return new GeneratedScmInfo(changesets); + } + + public static ScmInfo create(long analysisDate, Set lines, ScmInfo dbScmInfo) { + checkState(!lines.isEmpty(), "No changesets"); + + Changeset changeset = Changeset.newChangesetBuilder() + .setDate(analysisDate) + .build(); + Map changesets = lines.stream() + .collect(Collectors.toMap(x -> x, i -> changeset)); + + dbScmInfo.getAllChangesets().entrySet().stream() + .filter(e -> !lines.contains(e.getKey())) + .forEach(e -> changesets.put(e.getKey(), e.getValue())); + + return new GeneratedScmInfo(changesets); + } + + @Override + public Changeset getLatestChangeset() { + return delegate.getLatestChangeset(); + } + + @Override + public Changeset getChangesetForLine(int lineNumber) { + return delegate.getChangesetForLine(lineNumber); + } + + @Override + public boolean hasChangesetForLine(int lineNumber) { + return delegate.hasChangesetForLine(lineNumber); + } + + @Override + public Map getAllChangesets() { + return delegate.getAllChangesets(); + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoader.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoader.java index de56432ae5b..7ee76a4f646 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoader.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoader.java @@ -28,41 +28,34 @@ import org.sonar.db.source.FileSourceDto; import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.server.computation.task.projectanalysis.analysis.Branch; import org.sonar.server.computation.task.projectanalysis.component.Component; -import org.sonar.server.computation.task.projectanalysis.component.Component.Status; import org.sonar.server.computation.task.projectanalysis.component.MergeBranchComponentUuids; -import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepositoryImpl.NoScmInfo; -import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepository; public class ScmInfoDbLoader { private static final Logger LOGGER = Loggers.get(ScmInfoDbLoader.class); private final AnalysisMetadataHolder analysisMetadataHolder; private final DbClient dbClient; - private final SourceHashRepository sourceHashRepository; private final MergeBranchComponentUuids mergeBranchComponentUuid; - public ScmInfoDbLoader(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient, - SourceHashRepository sourceHashRepository, MergeBranchComponentUuids mergeBranchComponentUuid) { + public ScmInfoDbLoader(AnalysisMetadataHolder analysisMetadataHolder, DbClient dbClient, MergeBranchComponentUuids mergeBranchComponentUuid) { this.analysisMetadataHolder = analysisMetadataHolder; this.dbClient = dbClient; - this.sourceHashRepository = sourceHashRepository; this.mergeBranchComponentUuid = mergeBranchComponentUuid; } - public ScmInfo getScmInfoFromDb(Component file) { + public Optional getScmInfo(Component file) { Optional uuid = getFileUUid(file); - if (!uuid.isPresent()) { - return NoScmInfo.INSTANCE; + return Optional.empty(); } LOGGER.trace("Reading SCM info from db for file '{}'", uuid.get()); try (DbSession dbSession = dbClient.openSession(false)) { FileSourceDto dto = dbClient.fileSourceDao().selectSourceByFileUuid(dbSession, uuid.get()); - if (dto == null || !isDtoValid(file, dto)) { - return NoScmInfo.INSTANCE; + if (dto == null) { + return Optional.empty(); } - return DbScmInfo.create(dto.getSourceData().getLinesList()).orElse(NoScmInfo.INSTANCE); + return DbScmInfo.create(dto.getSourceData().getLinesList(), dto.getSrcHash()); } } @@ -80,10 +73,4 @@ public class ScmInfoDbLoader { return Optional.empty(); } - private boolean isDtoValid(Component file, FileSourceDto dto) { - if (file.getStatus() == Status.SAME) { - return true; - } - return sourceHashRepository.getRawSourceHash(file).equals(dto.getSrcHash()); - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImpl.java index df006eaf2fe..97b5233f18e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImpl.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.concurrent.Immutable; +import static com.google.common.base.Preconditions.checkState; @Immutable public class ScmInfoImpl implements ScmInfo { @@ -33,6 +34,7 @@ public class ScmInfoImpl implements ScmInfo { private final Map lineChangesets; public ScmInfoImpl(Map lineChangesets) { + checkState(!lineChangesets.isEmpty(), "A ScmInfo must have at least one Changeset and does not support any null one"); this.lineChangesets = lineChangesets; this.latestChangeset = computeLatestChangeset(lineChangesets); } @@ -54,7 +56,7 @@ public class ScmInfoImpl implements ScmInfo { if (changeset != null) { return changeset; } - throw new IllegalArgumentException("Line " + lineNumber + " doesn't have a changeset"); + throw new IllegalArgumentException("There's no changeset on line " + lineNumber); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java index 63203f0598a..4d592e8a0eb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImpl.java @@ -19,14 +19,21 @@ */ package org.sonar.server.computation.task.projectanalysis.scm; -import com.google.common.base.Optional; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.scanner.protocol.output.ScannerReport; +import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolder; import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReader; import org.sonar.server.computation.task.projectanalysis.component.Component; +import org.sonar.server.computation.task.projectanalysis.component.Component.Status; +import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepository; +import org.sonar.server.computation.task.projectanalysis.source.SourceLinesDiff; import static java.util.Objects.requireNonNull; @@ -34,87 +41,90 @@ public class ScmInfoRepositoryImpl implements ScmInfoRepository { private static final Logger LOGGER = Loggers.get(ScmInfoRepositoryImpl.class); - private final BatchReportReader batchReportReader; - private final Map scmInfoCache = new HashMap<>(); + private final BatchReportReader scannerReportReader; + private final Map> scmInfoCache = new HashMap<>(); private final ScmInfoDbLoader scmInfoDbLoader; - - public ScmInfoRepositoryImpl(BatchReportReader batchReportReader, ScmInfoDbLoader scmInfoDbLoader) { - this.batchReportReader = batchReportReader; + private final AnalysisMetadataHolder analysisMetadata; + private final SourceLinesDiff sourceLinesDiff; + private final SourceHashRepository sourceHashRepository; + + public ScmInfoRepositoryImpl(BatchReportReader scannerReportReader, AnalysisMetadataHolder analysisMetadata, ScmInfoDbLoader scmInfoDbLoader, + SourceLinesDiff sourceLinesDiff, SourceHashRepository sourceHashRepository) { + this.scannerReportReader = scannerReportReader; + this.analysisMetadata = analysisMetadata; this.scmInfoDbLoader = scmInfoDbLoader; + this.sourceLinesDiff = sourceLinesDiff; + this.sourceHashRepository = sourceHashRepository; } @Override - public Optional getScmInfo(Component component) { + public com.google.common.base.Optional getScmInfo(Component component) { requireNonNull(component, "Component cannot be null"); - return initializeScmInfoForComponent(component); - } - private Optional initializeScmInfoForComponent(Component component) { if (component.getType() != Component.Type.FILE) { - return Optional.absent(); - } - ScmInfo scmInfo = scmInfoCache.get(component); - if (scmInfo != null) { - return optionalOf(scmInfo); + return com.google.common.base.Optional.absent(); } - scmInfo = getScmInfoForComponent(component); - scmInfoCache.put(component, scmInfo); - return optionalOf(scmInfo); + return toGuavaOptional(scmInfoCache.computeIfAbsent(component, this::getScmInfoForComponent)); } - private static Optional optionalOf(ScmInfo scmInfo) { - if (scmInfo == NoScmInfo.INSTANCE) { - return Optional.absent(); - } - return Optional.of(scmInfo); + private static com.google.common.base.Optional toGuavaOptional(Optional scmInfo) { + return com.google.common.base.Optional.fromNullable(scmInfo.orElse(null)); } - private ScmInfo getScmInfoForComponent(Component component) { - ScannerReport.Changesets changesets = batchReportReader.readChangesets(component.getReportAttributes().getRef()); - if (changesets == null) { - LOGGER.trace("No SCM info for file '{}'", component.getKey()); - return NoScmInfo.INSTANCE; - } - if (changesets.getCopyFromPrevious()) { - return scmInfoDbLoader.getScmInfoFromDb(component); + private Optional getScmInfoForComponent(Component component) { + ScannerReport.Changesets changesets = scannerReportReader.readChangesets(component.getReportAttributes().getRef()); + + if (changesets != null) { + if (changesets.getChangesetCount() == 0) { + return generateAndMergeDb(component, changesets.getCopyFromPrevious()); + } + return getScmInfoFromReport(component, changesets); } - return getScmInfoFromReport(component, changesets); + + LOGGER.trace("No SCM info for file '{}'", component.getKey()); + return generateAndMergeDb(component, false); } - private static ScmInfo getScmInfoFromReport(Component file, ScannerReport.Changesets changesets) { + private static Optional getScmInfoFromReport(Component file, ScannerReport.Changesets changesets) { LOGGER.trace("Reading SCM info from report for file '{}'", file.getKey()); - return new ReportScmInfo(changesets); + return Optional.of(new ReportScmInfo(changesets)); } - /** - * Internally used to populate cache when no ScmInfo exist. - */ - enum NoScmInfo implements ScmInfo { - INSTANCE { - @Override - public Changeset getLatestChangeset() { - return notImplemented(); - } + private Optional generateScmInfoForAllFile(Component file) { + Set newOrChangedLines = IntStream.rangeClosed(1, file.getFileAttributes().getLines()).boxed().collect(Collectors.toSet()); + return Optional.of(GeneratedScmInfo.create(analysisMetadata.getAnalysisDate(), newOrChangedLines)); + } - @Override - public Changeset getChangesetForLine(int lineNumber) { - return notImplemented(); - } + private ScmInfo removeAuthorAndRevision(ScmInfo info) { + Map cleanedScmInfo = info.getAllChangesets().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> removeAuthorAndRevision(e.getValue()))); + return new ScmInfoImpl(cleanedScmInfo); + } - @Override - public boolean hasChangesetForLine(int lineNumber) { - return notImplemented(); - } + private static Changeset removeAuthorAndRevision(Changeset changeset) { + return Changeset.newChangesetBuilder().setDate(changeset.getDate()).build(); + } - @Override - public Iterable getAllChangesets() { - return notImplemented(); - } + private Optional generateAndMergeDb(Component file, boolean copyFromPrevious) { + Optional dbInfoOpt = scmInfoDbLoader.getScmInfo(file); + if (!dbInfoOpt.isPresent()) { + return generateScmInfoForAllFile(file); + } - private T notImplemented() { - throw new UnsupportedOperationException("NoScmInfo does not implement any method"); - } + ScmInfo scmInfo = copyFromPrevious ? dbInfoOpt.get() : removeAuthorAndRevision(dbInfoOpt.get()); + boolean fileUnchanged = file.getStatus() == Status.SAME && sourceHashRepository.getRawSourceHash(file).equals(dbInfoOpt.get().fileHash()); + + if (fileUnchanged) { + return Optional.of(scmInfo); } + + // generate date for new/changed lines + Set newOrChangedLines = sourceLinesDiff.getNewOrChangedLines(file); + if (newOrChangedLines.isEmpty()) { + return Optional.of(scmInfo); + } + return Optional.of(GeneratedScmInfo.create(analysisMetadata.getAnalysisDate(), newOrChangedLines, scmInfo)); } + } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiff.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiff.java new file mode 100644 index 00000000000..b6f555b3353 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiff.java @@ -0,0 +1,27 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.source; + +import java.util.Set; +import org.sonar.server.computation.task.projectanalysis.component.Component; + +public interface SourceLinesDiff { + Set getNewOrChangedLines(Component component); +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinder.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinder.java new file mode 100644 index 00000000000..2255b74823a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinder.java @@ -0,0 +1,71 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.source; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SourceLinesDiffFinder { + + private final List database; + private final List report; + + public SourceLinesDiffFinder(List database, List report) { + this.database = database; + this.report = report; + } + + public Set findNewOrChangedLines() { + return walk(0, 0, new HashSet<>()); + } + + private Set walk(int r, int db, HashSet acc) { + + if (r >= report.size()) { + return acc; + } + + if (db < database.size()) { + + if (report.get(r).equals(database.get(db))) { + walk(stepIndex(r), stepIndex(db), acc); + return acc; + } + + List remainingDatabase = database.subList(db, database.size()); + if (remainingDatabase.contains(report.get(r))) { + int nextDb = db + remainingDatabase.indexOf(report.get(r)); + walk(r, nextDb, acc); + return acc; + } + + } + + acc.add(r+1); + walk(stepIndex(r), db, acc); + return acc; + } + + private static int stepIndex(int r) { + return ++r; + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImpl.java new file mode 100644 index 00000000000..3404a2668ea --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImpl.java @@ -0,0 +1,67 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.source; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.sonar.core.hash.SourceLinesHashesComputer; +import org.sonar.core.util.CloseableIterator; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.source.FileSourceDao; +import org.sonar.server.computation.task.projectanalysis.component.Component; + +public class SourceLinesDiffImpl implements SourceLinesDiff { + + private final SourceLinesRepository sourceLinesRepository; + + private final DbClient dbClient; + private final FileSourceDao fileSourceDao; + + public SourceLinesDiffImpl(DbClient dbClient, FileSourceDao fileSourceDao, SourceLinesRepository sourceLinesRepository) { + this.dbClient = dbClient; + this.fileSourceDao = fileSourceDao; + this.sourceLinesRepository = sourceLinesRepository; + } + + @Override + public Set getNewOrChangedLines(Component component) { + + List database = new ArrayList<>(); + try (DbSession dbSession = dbClient.openSession(false)) { + database.addAll(fileSourceDao.selectLineHashes(dbSession, component.getUuid())); + } + + List report = new ArrayList<>(); + SourceLinesHashesComputer linesHashesComputer = new SourceLinesHashesComputer(); + try (CloseableIterator lineIterator = sourceLinesRepository.readLines(component)) { + while (lineIterator.hasNext()) { + String line = lineIterator.next(); + linesHashesComputer.addLine(line); + } + } + report.addAll(linesHashesComputer.getLineHashes()); + + return new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfoTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfoTest.java index a3b98652874..91c71ceab4a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfoTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/DbScmInfoTest.java @@ -34,7 +34,7 @@ public class DbScmInfoTest { @Test public void create_scm_info_with_some_changesets() throws Exception { - ScmInfo scmInfo = DbScmInfo.create(newFakeData(10).build().getLinesList()).get(); + ScmInfo scmInfo = DbScmInfo.create(newFakeData(10).build().getLinesList(), "hash").get(); assertThat(scmInfo.getAllChangesets()).hasSize(10); } @@ -48,7 +48,7 @@ public class DbScmInfoTest { addLine(fileDataBuilder, 4, "john", 123456789L, "rev-1"); fileDataBuilder.build(); - ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList()).get(); + ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get(); assertThat(scmInfo.getAllChangesets()).hasSize(4); @@ -66,7 +66,7 @@ public class DbScmInfoTest { fileDataBuilder.addLinesBuilder().setScmRevision("rev1").setScmDate(6541L).setLine(3); fileDataBuilder.addLinesBuilder().setScmRevision("rev").setScmDate(65L).setLine(4); - ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList()).get(); + ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get(); assertThat(scmInfo.getAllChangesets()).hasSize(4); @@ -82,7 +82,7 @@ public class DbScmInfoTest { addLine(fileDataBuilder, 3, "john", 123456789L, "rev-1"); fileDataBuilder.build(); - ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList()).get(); + ScmInfo scmInfo = DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get(); Changeset latestChangeset = scmInfo.getLatestChangeset(); assertThat(latestChangeset.getAuthor()).isEqualTo("henry"); @@ -95,7 +95,7 @@ public class DbScmInfoTest { DbFileSources.Data.Builder fileDataBuilder = DbFileSources.Data.newBuilder(); fileDataBuilder.addLinesBuilder().setLine(1); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList())).isNotPresent(); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash")).isNotPresent(); } @Test @@ -105,7 +105,7 @@ public class DbScmInfoTest { fileDataBuilder.addLinesBuilder().setLine(2); fileDataBuilder.build(); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList()).get().getAllChangesets()).hasSize(1); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get().getAllChangesets()).hasSize(1); } @Test @@ -115,8 +115,8 @@ public class DbScmInfoTest { fileDataBuilder.addLinesBuilder().setScmRevision("rev-1").setLine(2); fileDataBuilder.build(); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList()).get().getAllChangesets()).hasSize(1); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList()).get().getChangesetForLine(1).getRevision()).isEqualTo("rev"); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get().getAllChangesets()).hasSize(1); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get().getChangesetForLine(1).getRevision()).isEqualTo("rev"); } @Test @@ -127,8 +127,8 @@ public class DbScmInfoTest { fileDataBuilder.addLinesBuilder().setScmRevision("rev").setScmDate(555L).setLine(2); fileDataBuilder.build(); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList()).get().getAllChangesets()).hasSize(1); - assertThat(DbScmInfo.create(fileDataBuilder.getLinesList()).get().getChangesetForLine(2).getAuthor()).isNull(); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get().getAllChangesets()).hasSize(1); + assertThat(DbScmInfo.create(fileDataBuilder.getLinesList(), "hash").get().getChangesetForLine(2).getAuthor()).isNull(); } private static void addLine(DbFileSources.Data.Builder dataBuilder, Integer line, String author, Long date, String revision) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoaderTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoaderTest.java index 71f0f53a0df..bd1845e4832 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoaderTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoDbLoaderTest.java @@ -32,16 +32,12 @@ import org.sonar.core.hash.SourceHashComputer; import org.sonar.db.DbTester; import org.sonar.db.protobuf.DbFileSources; import org.sonar.db.source.FileSourceDto; -import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.server.computation.task.projectanalysis.analysis.Analysis; import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolderRule; import org.sonar.server.computation.task.projectanalysis.analysis.Branch; import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReaderRule; import org.sonar.server.computation.task.projectanalysis.component.Component; import org.sonar.server.computation.task.projectanalysis.component.MergeBranchComponentUuids; -import org.sonar.server.computation.task.projectanalysis.scm.ScmInfoRepositoryImpl.NoScmInfo; -import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepositoryImpl; -import org.sonar.server.computation.task.projectanalysis.source.SourceLinesRepositoryImpl; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -71,21 +67,21 @@ public class ScmInfoDbLoaderTest { public BatchReportReaderRule reportReader = new BatchReportReaderRule(); private Branch branch = mock(Branch.class); - private SourceHashRepositoryImpl sourceHashRepository = new SourceHashRepositoryImpl(new SourceLinesRepositoryImpl(reportReader)); private MergeBranchComponentUuids mergeBranchComponentUuids = mock(MergeBranchComponentUuids.class); - private ScmInfoDbLoader underTest = new ScmInfoDbLoader(analysisMetadataHolder, dbTester.getDbClient(), sourceHashRepository, mergeBranchComponentUuids); + private ScmInfoDbLoader underTest = new ScmInfoDbLoader(analysisMetadataHolder, dbTester.getDbClient(), mergeBranchComponentUuids); @Test - public void returns_ScmInfo_from_DB_if_hashes_are_the_same() { + public void returns_ScmInfo_from_DB() { analysisMetadataHolder.setBaseAnalysis(baseProjectAnalysis); analysisMetadataHolder.setBranch(null); - addFileSourceInDb("henry", DATE_1, "rev-1", computeSourceHash(1)); - addFileSourceInReport(1); + String hash = computeSourceHash(1); + addFileSourceInDb("henry", DATE_1, "rev-1", hash); - ScmInfo scmInfo = underTest.getScmInfoFromDb(FILE); + DbScmInfo scmInfo = underTest.getScmInfo(FILE).get(); assertThat(scmInfo.getAllChangesets()).hasSize(1); + assertThat(scmInfo.fileHash()).isEqualTo(hash); assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from db for file 'FILE_UUID'"); } @@ -94,55 +90,39 @@ public class ScmInfoDbLoaderTest { public void read_from_merge_branch_if_no_base() { analysisMetadataHolder.setBaseAnalysis(null); analysisMetadataHolder.setBranch(branch); - String mergeFileUuid = "mergeFileUuid"; - when(branch.getMergeBranchUuid()).thenReturn(Optional.of("mergeBranchUuid")); - when(mergeBranchComponentUuids.getUuid(FILE.getKey())).thenReturn(mergeFileUuid); - addFileSourceInDb("henry", DATE_1, "rev-1", computeSourceHash(1), mergeFileUuid); - addFileSourceInReport(1); - - ScmInfo scmInfo = underTest.getScmInfoFromDb(FILE); - assertThat(scmInfo.getAllChangesets()).hasSize(1); - assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from db for file 'mergeFileUuid'"); - } - - @Test - public void returns_absent_when_branch_and_source_is_different() { - analysisMetadataHolder.setBaseAnalysis(null); - analysisMetadataHolder.setBranch(branch); String mergeFileUuid = "mergeFileUuid"; + String hash = computeSourceHash(1); when(branch.getMergeBranchUuid()).thenReturn(Optional.of("mergeBranchUuid")); when(mergeBranchComponentUuids.getUuid(FILE.getKey())).thenReturn(mergeFileUuid); - addFileSourceInDb("henry", DATE_1, "rev-1", computeSourceHash(1) + "dif", mergeFileUuid); - addFileSourceInReport(1); + addFileSourceInDb("henry", DATE_1, "rev-1", hash, mergeFileUuid); - assertThat(underTest.getScmInfoFromDb(FILE)).isEqualTo(NoScmInfo.INSTANCE); + DbScmInfo scmInfo = underTest.getScmInfo(FILE).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(1); + assertThat(scmInfo.fileHash()).isEqualTo(hash); assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from db for file 'mergeFileUuid'"); } - + @Test - public void returns_absent_when__hashes_are_not_the_same() { + public void return_empty_if_no_dto_available() { analysisMetadataHolder.setBaseAnalysis(baseProjectAnalysis); analysisMetadataHolder.setBranch(null); - - addFileSourceInReport(1); - addFileSourceInDb("henry", DATE_1, "rev-1", computeSourceHash(1) + "_different"); - - assertThat(underTest.getScmInfoFromDb(FILE)).isEqualTo(NoScmInfo.INSTANCE); + + Optional scmInfo = underTest.getScmInfo(FILE); + assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from db for file 'FILE_UUID'"); + assertThat(scmInfo).isEmpty(); } @Test - public void not_read_in_db_on_first_analysis_and_no_merge_branch() { + public void do_not_read_from_db_on_first_analysis_and_no_merge_branch() { Branch branch = mock(Branch.class); when(branch.getMergeBranchUuid()).thenReturn(Optional.empty()); analysisMetadataHolder.setBaseAnalysis(null); analysisMetadataHolder.setBranch(branch); - addFileSourceInReport(1); - - assertThat(underTest.getScmInfoFromDb(FILE)).isEqualTo(NoScmInfo.INSTANCE); + assertThat(underTest.getScmInfo(FILE)).isEmpty(); assertThat(logTester.logs(TRACE)).isEmpty(); } @@ -188,11 +168,4 @@ public class ScmInfoDbLoaderTest { dbTester.commit(); } - private void addFileSourceInReport(int lineCount) { - reportReader.putFileSourceLines(FILE_REF, generateLines(lineCount)); - reportReader.putComponent(ScannerReport.Component.newBuilder() - .setRef(FILE_REF) - .setLines(lineCount) - .build()); - } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImplTest.java index d5efe051080..2f2264a95b1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoImplTest.java @@ -109,12 +109,12 @@ public class ScmInfoImplTest { assertThat(scmInfo.toString()).isEqualTo("ScmInfoImpl{" + "latestChangeset=Changeset{revision='rev-2', author='henry', date=1234567810}, " + - "lineChangesets=[" + - "Changeset{revision='rev-1', author='john', date=123456789}, " + - "Changeset{revision='rev-2', author='henry', date=1234567810}, " + - "Changeset{revision='rev-1', author='john', date=123456789}, " + - "Changeset{revision='rev-1', author='john', date=123456789}" + - "]}"); + "lineChangesets={" + + "1=Changeset{revision='rev-1', author='john', date=123456789}, " + + "2=Changeset{revision='rev-2', author='henry', date=1234567810}, " + + "3=Changeset{revision='rev-1', author='john', date=123456789}, " + + "4=Changeset{revision='rev-1', author='john', date=123456789}" + + "}}"); } private static ScmInfo createScmInfoWithTwoChangestOnFourLines() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java index bd7ac6f2e56..f84703c3044 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/scm/ScmInfoRepositoryImplTest.java @@ -20,26 +20,41 @@ package org.sonar.server.computation.task.projectanalysis.scm; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.List; +import java.util.Optional; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.sonar.api.utils.log.LogTester; +import org.sonar.db.protobuf.DbFileSources.Line; import org.sonar.scanner.protocol.output.ScannerReport; +import org.sonar.scanner.protocol.output.ScannerReport.Changesets; +import org.sonar.server.computation.task.projectanalysis.analysis.AnalysisMetadataHolderRule; import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReader; import org.sonar.server.computation.task.projectanalysis.batch.BatchReportReaderRule; import org.sonar.server.computation.task.projectanalysis.component.Component; +import org.sonar.server.computation.task.projectanalysis.component.Component.Status; +import org.sonar.server.computation.task.projectanalysis.component.Component.Type; +import org.sonar.server.computation.task.projectanalysis.component.FileAttributes; import org.sonar.server.computation.task.projectanalysis.component.ReportComponent; import org.sonar.server.computation.task.projectanalysis.component.ViewsComponent; +import org.sonar.server.computation.task.projectanalysis.source.SourceHashRepository; +import org.sonar.server.computation.task.projectanalysis.source.SourceLinesDiff; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.guava.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import static org.sonar.api.utils.log.LoggerLevel.TRACE; @@ -48,7 +63,9 @@ import static org.sonar.server.computation.task.projectanalysis.component.Report @RunWith(DataProviderRunner.class) public class ScmInfoRepositoryImplTest { static final int FILE_REF = 1; - static final Component FILE = builder(Component.Type.FILE, FILE_REF).setKey("FILE_KEY").setUuid("FILE_UUID").build(); + static final FileAttributes attributes = new FileAttributes(false, "java", 3); + static final Component FILE = builder(Component.Type.FILE, FILE_REF).setKey("FILE_KEY").setUuid("FILE_UUID").setFileAttributes(attributes).build(); + static final Component FILE_SAME = builder(Component.Type.FILE, FILE_REF).setStatus(Status.SAME).setKey("FILE_KEY").setUuid("FILE_UUID").setFileAttributes(attributes).build(); static final long DATE_1 = 123456789L; static final long DATE_2 = 1234567810L; @@ -58,63 +75,151 @@ public class ScmInfoRepositoryImplTest { public LogTester logTester = new LogTester(); @Rule public BatchReportReaderRule reportReader = new BatchReportReaderRule(); + @Rule + public AnalysisMetadataHolderRule analysisMetadata = new AnalysisMetadataHolderRule(); + private SourceHashRepository sourceHashRepository = mock(SourceHashRepository.class); + private SourceLinesDiff diff = mock(SourceLinesDiff.class); private ScmInfoDbLoader dbLoader = mock(ScmInfoDbLoader.class); + private Date analysisDate = new Date(); + + private ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, analysisMetadata, dbLoader, diff, sourceHashRepository); - private ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(reportReader, dbLoader); + @Before + public void setUp() { + analysisMetadata.setAnalysisDate(analysisDate); + } @Test - public void read_from_report() { - addChangesetInReport("john", DATE_1, "rev-1"); + public void return_empty_if_component_is_not_file() { + Component c = mock(Component.class); + when(c.getType()).thenReturn(Type.DIRECTORY); + assertThat(underTest.getScmInfo(c)).isAbsent(); + } + @Test + public void load_scm_info_from_cache_when_already_loaded() { + addChangesetInReport("john", DATE_1, "rev-1"); ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); assertThat(scmInfo.getAllChangesets()).hasSize(1); - assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from report for file 'FILE_KEY'"); - } + assertThat(logTester.logs(TRACE)).hasSize(1); + logTester.clear(); - @Test - public void getScmInfo_returns_absent_if_CopyFromPrevious_is_false_and_there_is_no_changeset_in_report() { - addFileSourceInReport(1); + underTest.getScmInfo(FILE); + assertThat(logTester.logs(TRACE)).isEmpty(); - assertThat(underTest.getScmInfo(FILE)).isAbsent(); verifyZeroInteractions(dbLoader); + verifyZeroInteractions(sourceHashRepository); + verifyZeroInteractions(diff); } @Test - public void read_from_report_even_if_data_in_db_exists() { - addChangesetInReport("john", DATE_2, "rev-2"); + public void read_from_report() { + addChangesetInReport("john", DATE_1, "rev-1"); ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(1); Changeset changeset = scmInfo.getChangesetForLine(1); assertThat(changeset.getAuthor()).isEqualTo("john"); - assertThat(changeset.getDate()).isEqualTo(DATE_2); - assertThat(changeset.getRevision()).isEqualTo("rev-2"); + assertThat(changeset.getDate()).isEqualTo(DATE_1); + assertThat(changeset.getRevision()).isEqualTo("rev-1"); + + assertThat(logTester.logs(TRACE)).containsOnly("Reading SCM info from report for file 'FILE_KEY'"); + verifyZeroInteractions(dbLoader); + verifyZeroInteractions(sourceHashRepository); + verifyZeroInteractions(diff); } @Test - public void read_from_db_even_if_data_in_report_exists_when_CopyFromPrevious_is_true() { - ScmInfo info = mock(ScmInfo.class); - when(dbLoader.getScmInfoFromDb(FILE)).thenReturn(info); + public void read_from_DB_if_no_report_and_file_unchanged() { + createDbScmInfoWithOneLine("hash"); + when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash"); + + // should clear revision and author + ScmInfo scmInfo = underTest.getScmInfo(FILE_SAME).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(1); + assertChangeset(scmInfo.getChangesetForLine(1), null, null, 10L); + + verify(sourceHashRepository).getRawSourceHash(FILE_SAME); + verify(dbLoader).getScmInfo(FILE_SAME); + verifyNoMoreInteractions(dbLoader); + verifyNoMoreInteractions(sourceHashRepository); + verifyZeroInteractions(diff); + } + + @Test + public void read_from_DB_if_no_report_and_file_unchanged_and_copyFromPrevious_is_true() { + createDbScmInfoWithOneLine("hash"); + when(sourceHashRepository.getRawSourceHash(FILE_SAME)).thenReturn("hash"); addFileSourceInReport(1); - addChangesetInReport("john", DATE_2, "rev-2", true); + addCopyFromPrevious(); + + ScmInfo scmInfo = underTest.getScmInfo(FILE_SAME).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(1); + assertChangeset(scmInfo.getChangesetForLine(1), "rev1", "author1", 10L); + verify(sourceHashRepository).getRawSourceHash(FILE_SAME); + verify(dbLoader).getScmInfo(FILE_SAME); + + verifyNoMoreInteractions(dbLoader); + verifyNoMoreInteractions(sourceHashRepository); + verifyZeroInteractions(diff); + } + + @Test + public void generate_scm_info_when_nothing_in_report_nor_db() { + when(dbLoader.getScmInfo(FILE)).thenReturn(Optional.empty()); ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); - assertThat(scmInfo).isEqualTo(info); + assertThat(scmInfo.getAllChangesets()).hasSize(3); + + for (int i = 1; i <= 3; i++) { + assertChangeset(scmInfo.getChangesetForLine(i), null, null, analysisDate.getTime()); + } + + verify(dbLoader).getScmInfo(FILE); + verifyNoMoreInteractions(dbLoader); + verifyZeroInteractions(sourceHashRepository); + verifyZeroInteractions(diff); } @Test - public void return_nothing_when_no_data_in_report_nor_db() { - assertThat(underTest.getScmInfo(FILE)).isAbsent(); + public void generate_scm_info_when_nothing_in_db_and_report_is_has_no_changesets() { + when(dbLoader.getScmInfo(FILE)).thenReturn(Optional.empty()); + addFileSourceInReport(3); + ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(3); + + for (int i = 1; i <= 3; i++) { + assertChangeset(scmInfo.getChangesetForLine(i), null, null, analysisDate.getTime()); + } + + verify(dbLoader).getScmInfo(FILE); + verifyNoMoreInteractions(dbLoader); + verifyZeroInteractions(sourceHashRepository); + verifyZeroInteractions(diff); } @Test - public void return_nothing_when_nothing_in_report_and_db_has_no_scm() { - addFileSourceInReport(1); - assertThat(underTest.getScmInfo(FILE)).isAbsent(); + public void generate_scm_info_for_new_and_changed_lines_when_report_is_empty() { + createDbScmInfoWithOneLine("hash"); + when(diff.getNewOrChangedLines(FILE)).thenReturn(ImmutableSet.of(2, 3)); + addFileSourceInReport(3); + ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); + assertThat(scmInfo.getAllChangesets()).hasSize(3); + + assertChangeset(scmInfo.getChangesetForLine(1), null, null, 10L); + assertChangeset(scmInfo.getChangesetForLine(2), null, null, analysisDate.getTime()); + assertChangeset(scmInfo.getChangesetForLine(3), null, null, analysisDate.getTime()); + + verify(dbLoader).getScmInfo(FILE); + verify(diff).getNewOrChangedLines(FILE); + verifyNoMoreInteractions(dbLoader); + verifyZeroInteractions(sourceHashRepository); + verifyNoMoreInteractions(diff); } @Test @@ -125,6 +230,17 @@ public class ScmInfoRepositoryImplTest { underTest.getScmInfo(null); } + @Test + @UseDataProvider("allTypeComponentButFile") + public void do_not_query_db_nor_report_if_component_type_is_not_FILE(Component component) { + BatchReportReader batchReportReader = mock(BatchReportReader.class); + ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, analysisMetadata, dbLoader, diff, sourceHashRepository); + + assertThat(underTest.getScmInfo(component)).isAbsent(); + + verifyZeroInteractions(batchReportReader, dbLoader); + } + @DataProvider public static Object[][] allTypeComponentButFile() { Object[][] res = new Object[Component.Type.values().length - 1][1]; @@ -140,28 +256,10 @@ public class ScmInfoRepositoryImplTest { return res; } - @Test - @UseDataProvider("allTypeComponentButFile") - public void do_not_query_db_nor_report_if_component_type_is_not_FILE(Component component) { - BatchReportReader batchReportReader = mock(BatchReportReader.class); - ScmInfoRepositoryImpl underTest = new ScmInfoRepositoryImpl(batchReportReader, dbLoader); - - assertThat(underTest.getScmInfo(component)).isAbsent(); - - verifyZeroInteractions(batchReportReader, dbLoader); - } - - @Test - public void load_scm_info_from_cache_when_already_read() { - addChangesetInReport("john", DATE_1, "rev-1"); - ScmInfo scmInfo = underTest.getScmInfo(FILE).get(); - assertThat(scmInfo.getAllChangesets()).hasSize(1); - - assertThat(logTester.logs(TRACE)).hasSize(1); - logTester.clear(); - - underTest.getScmInfo(FILE); - assertThat(logTester.logs(TRACE)).isEmpty(); + private void assertChangeset(Changeset changeset, String revision, String author, long date) { + assertThat(changeset.getAuthor()).isEqualTo(author); + assertThat(changeset.getRevision()).isEqualTo(revision); + assertThat(changeset.getDate()).isEqualTo(date); } private void addChangesetInReport(String author, Long date, String revision) { @@ -181,6 +279,21 @@ public class ScmInfoRepositoryImplTest { .build()); } + private void addCopyFromPrevious() { + reportReader.putChangesets(Changesets.newBuilder().setComponentRef(FILE_REF).setCopyFromPrevious(true).build()); + } + + private DbScmInfo createDbScmInfoWithOneLine(String hash) { + Line line1 = Line.newBuilder().setLine(1) + .setScmRevision("rev1") + .setScmAuthor("author1") + .setScmDate(10L) + .build(); + DbScmInfo scmInfo = DbScmInfo.create(Collections.singleton(line1), hash).get(); + when(dbLoader.getScmInfo(FILE)).thenReturn(Optional.of(scmInfo)); + return scmInfo; + } + private void addFileSourceInReport(int lineCount) { reportReader.putFileSourceLines(FILE_REF, generateLines(lineCount)); reportReader.putComponent(ScannerReport.Component.newBuilder() @@ -196,4 +309,5 @@ public class ScmInfoRepositoryImplTest { } return builder.build(); } + } diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinderTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinderTest.java new file mode 100644 index 00000000000..7aa1288d0f0 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffFinderTest.java @@ -0,0 +1,282 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.source; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SourceLinesDiffFinderTest { + + @Test + public void shouldFindNothingWhenContentAreIdentical() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + database.add("line - 4"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2"); + report.add("line - 3"); + report.add("line - 4"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).isEmpty(); + + } + + @Test + public void shouldFindNothingWhenContentAreIdentical2() { + + List database = new ArrayList<>(); + database.add("package sample;\n"); + database.add("\n"); + database.add("public class Sample {\n"); + database.add("\n"); + database.add(" private String myMethod() {\n"); + database.add(" }\n"); + database.add("}\n"); + + List report = new ArrayList<>(); + report.add("package sample;\n"); + report.add("\n"); + report.add("public class Sample {\n"); + report.add("\n"); + report.add(" private String attr;\n"); + report.add("\n"); + report.add(" public Sample(String attr) {\n"); + report.add(" this.attr = attr;\n"); + report.add(" }\n"); + report.add("\n"); + report.add(" private String myMethod() {\n"); + report.add(" }\n"); + report.add("}\n"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(5, 6, 7, 8, 10, 11, 12); + + } + + @Test + public void shouldDetectWhenStartingWithModifiedLines() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + + List report = new ArrayList<>(); + report.add("line - 0 - modified"); + report.add("line - 1 - modified"); + report.add("line - 2"); + report.add("line - 3"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(1, 2); + + } + + @Test + public void shouldDetectWhenEndingWithModifiedLines() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2 - modified"); + report.add("line - 3 - modified"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(3, 4); + + } + + @Test + public void shouldDetectModifiedLinesInMiddleOfTheFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + database.add("line - 4"); + database.add("line - 5"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2 - modified"); + report.add("line - 3 - modified"); + report.add("line - 4"); + report.add("line - 5"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(3, 4); + + } + + @Test + public void shouldDetectNewLinesAtBeginningOfFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + + List report = new ArrayList<>(); + report.add("line - new"); + report.add("line - new"); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(1, 2); + + } + + @Test + public void shouldDetectNewLinesInMiddleOfFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - new"); + report.add("line - new"); + report.add("line - 2"); + report.add("line - 3"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(3, 4); + + } + + @Test + public void shouldDetectNewLinesAtEndOfFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2"); + report.add("line - new"); + report.add("line - new"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).containsExactlyInAnyOrder(4, 5); + + } + + @Test + public void shouldIgnoreDeletedLinesAtEndOfFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + database.add("line - 4"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 2"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).isEmpty(); + + } + + @Test + public void shouldIgnoreDeletedLinesInTheMiddleOfFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + database.add("line - 4"); + database.add("line - 5"); + + List report = new ArrayList<>(); + report.add("line - 0"); + report.add("line - 1"); + report.add("line - 4"); + report.add("line - 5"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).isEmpty(); + + } + + @Test + public void shouldIgnoreDeletedLinesAtTheStartOfTheFile() { + + List database = new ArrayList<>(); + database.add("line - 0"); + database.add("line - 1"); + database.add("line - 2"); + database.add("line - 3"); + + List report = new ArrayList<>(); + report.add("line - 2"); + report.add("line - 3"); + + Set diff = new SourceLinesDiffFinder(database, report).findNewOrChangedLines(); + + assertThat(diff).isEmpty(); + + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImplTest.java new file mode 100644 index 00000000000..25653123901 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/source/SourceLinesDiffImplTest.java @@ -0,0 +1,115 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.computation.task.projectanalysis.source; + +import com.google.common.base.Splitter; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.core.hash.SourceLinesHashesComputer; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.component.ComponentDao; +import org.sonar.db.source.FileSourceDao; +import org.sonar.db.source.FileSourceDto; +import org.sonar.server.computation.task.projectanalysis.component.Component; + +import static com.google.common.base.Joiner.on; +import static java.lang.String.valueOf; +import static java.util.Arrays.stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.sonar.server.computation.task.projectanalysis.component.Component.Type.FILE; +import static org.sonar.server.computation.task.projectanalysis.component.ReportComponent.builder; + +public class SourceLinesDiffImplTest { + + @Rule + public SourceLinesRepositoryRule sourceLinesRepository = new SourceLinesRepositoryRule(); + + private DbClient dbClient = mock(DbClient.class); + private DbSession dbSession = mock(DbSession.class); + private ComponentDao componentDao = mock(ComponentDao.class); + private FileSourceDao fileSourceDao = mock(FileSourceDao.class); + + private static final Splitter END_OF_LINE_SPLITTER = Splitter.on('\n'); + + private SourceLinesDiffImpl underTest = new SourceLinesDiffImpl(dbClient, fileSourceDao, sourceLinesRepository); + + private static final int FILE_REF = 1; + private static final String FILE_KEY = valueOf(FILE_REF); + + private static final String[] CONTENT = { + "package org.sonar.server.computation.task.projectanalysis.source_diff;", + "", + "public class Foo {", + " public String bar() {", + " return \"Doh!\";", + " }", + "}" + }; + + @Before + public void setUp() throws Exception { + when(dbClient.openSession(false)).thenReturn(dbSession); + when(dbClient.componentDao()).thenReturn(componentDao); + when(dbClient.fileSourceDao()).thenReturn(fileSourceDao); + } + + @Test + public void should_find_no_diff_when_report_and_db_content_are_identical() { + + mockContentOfFileInDb("" + FILE_KEY, CONTENT); + setFileContentInReport(FILE_REF, CONTENT); + + Component component = fileComponent(FILE_REF); + assertThat(underTest.getNewOrChangedLines(component)).isEmpty(); + + } + + private void mockContentOfFileInDb(String key, @Nullable String[] content) { + FileSourceDto dto = new FileSourceDto(); + if (content != null) { + SourceLinesHashesComputer linesHashesComputer = new SourceLinesHashesComputer(); + stream(content).forEach(linesHashesComputer::addLine); + dto.setLineHashes(on('\n').join(linesHashesComputer.getLineHashes())); + } + + when(fileSourceDao.selectLineHashes(dbSession, componentUuidOf(key))) + .thenReturn(END_OF_LINE_SPLITTER.splitToList(dto.getLineHashes())); + } + + private static String componentUuidOf(String key) { + return "uuid_" + key; + } + + private static Component fileComponent(int ref) { + return builder(FILE, ref) + .setPath("report_path" + ref) + .setUuid(componentUuidOf("" + ref)) + .build(); + } + + private void setFileContentInReport(int ref, String[] content) { + sourceLinesRepository.addLines(ref, content); + } +} -- 2.39.5