*/
String getName();
+ /**
+ * Get component old relative path.
+ */
+ String getOldName();
+
/**
* The component short name. For files and directories this is the parent relative path (ie filename for files). For projects and view this is the same as {@link #getName()}
*/
return this.name;
}
+ @Override
+ public String getOldName() {
+ return this.getFileAttributes().getOldName();
+ }
+
@Override
public String getShortName() {
return this.shortName;
private String uuid;
private String key;
private String name;
+ private String oldName;
private String shortName;
private String description;
private FileAttributes fileAttributes;
component.getIsTest(),
lang != null ? lang.intern() : null,
component.getLines(),
- component.getMarkedAsUnchanged());
+ component.getMarkedAsUnchanged(),
+ component.getOldRelativeFilePath()
+ );
}
private static class Node {
import javax.annotation.concurrent.Immutable;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.commons.lang.StringUtils.abbreviate;
+import static org.apache.commons.lang.StringUtils.trimToNull;
+import static org.sonar.db.component.ComponentValidator.MAX_COMPONENT_NAME_LENGTH;
/**
* The attributes specific to a Component of type {@link Component.Type#FILE}.
private final String languageKey;
private final boolean markedAsUnchanged;
private final int lines;
+ private String oldName;
public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines) {
- this(unitTest, languageKey, lines, false);
+ this(unitTest, languageKey, lines, false, null);
}
- public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged) {
+ public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged, @Nullable String oldName) {
this.unitTest = unitTest;
this.languageKey = languageKey;
this.markedAsUnchanged = markedAsUnchanged;
checkArgument(lines > 0, "Number of lines must be greater than zero");
this.lines = lines;
+ this.oldName = formatOldName(oldName);
}
public boolean isMarkedAsUnchanged() {
return languageKey;
}
+ @CheckForNull
+ public String getOldName() {
+ return oldName;
+ }
+
/**
* Number of lines of the file, can never be less than 1
*/
", unitTest=" + unitTest +
", lines=" + lines +
", markedAsUnchanged=" + markedAsUnchanged +
+ ", oldName=" + oldName +
'}';
}
+
+ private String formatOldName(@Nullable String name) {
+ return abbreviate(trimToNull(name), MAX_COMPONENT_NAME_LENGTH);
+ }
}
@Override
public void execute(ComputationStep.Context context) {
+ if (analysisMetadataHolder.isPullRequest()) {
+ LOG.debug("Currently within Pull Request scope. Do nothing.");
+ return;
+ }
+
// do nothing if no files in db (first analysis)
if (analysisMetadataHolder.isFirstAnalysis()) {
LOG.debug("First analysis. Do nothing.");
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.filemove;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import javax.annotation.concurrent.Immutable;
+import org.apache.ibatis.session.ResultHandler;
+import org.jetbrains.annotations.NotNull;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
+import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
+import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
+import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter;
+import org.sonar.ce.task.step.ComputationStep;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.component.BranchDto;
+import org.sonar.db.component.FileMoveRowDto;
+
+import static java.util.stream.Collectors.toMap;
+import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER;
+
+public class PullRequestFileMoveDetectionStep implements ComputationStep {
+ private static final Logger LOG = Loggers.get(PullRequestFileMoveDetectionStep.class);
+
+ private final AnalysisMetadataHolder analysisMetadataHolder;
+ private final TreeRootHolder rootHolder;
+ private final DbClient dbClient;
+ private final MutableMovedFilesRepository movedFilesRepository;
+ private final MutableAddedFileRepository addedFileRepository;
+
+ public PullRequestFileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient,
+ MutableMovedFilesRepository movedFilesRepository, MutableAddedFileRepository addedFileRepository) {
+ this.analysisMetadataHolder = analysisMetadataHolder;
+ this.rootHolder = rootHolder;
+ this.dbClient = dbClient;
+ this.movedFilesRepository = movedFilesRepository;
+ this.addedFileRepository = addedFileRepository;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Detect file moves in Pull Request scope";
+ }
+
+ @Override
+ public void execute(ComputationStep.Context context) {
+ if (!analysisMetadataHolder.isPullRequest()) {
+ LOG.debug("Currently not within Pull Request scope. Do nothing.");
+ return;
+ }
+
+ Map<String, Component> reportFilesByUuid = getReportFilesByUuid(this.rootHolder.getRoot());
+ context.getStatistics().add("reportFiles", reportFilesByUuid.size());
+
+ if (reportFilesByUuid.isEmpty()) {
+ LOG.debug("No files in report. No file move detection.");
+ return;
+ }
+
+ Map<String, DbComponent> targetBranchDbFilesByUuid = getTargetBranchDbFilesByUuid(analysisMetadataHolder);
+ context.getStatistics().add("dbFiles", targetBranchDbFilesByUuid.size());
+
+ if (targetBranchDbFilesByUuid.isEmpty()) {
+ registerNewlyAddedFiles(reportFilesByUuid);
+ context.getStatistics().add("addedFiles", reportFilesByUuid.size());
+ LOG.debug("Previous snapshot has no file. No file move detection.");
+ return;
+ }
+
+ Map<String, Component> movedFilesByUuid = getMovedFilesByUuid(reportFilesByUuid);
+ context.getStatistics().add("movedFiles", movedFilesByUuid.size());
+
+ Map<String, Component> newlyAddedFilesByUuid = getNewlyAddedFilesByUuid(reportFilesByUuid, targetBranchDbFilesByUuid);
+ context.getStatistics().add("addedFiles", newlyAddedFilesByUuid.size());
+
+ // Do we need to register the moved file in the moved files repo and use the data in the related steps/visitors?
+// registerMovedFiles(movedFilesByUuid, targetBranchDbFilesByUuid);
+ registerNewlyAddedFiles(newlyAddedFilesByUuid);
+ }
+
+ private void registerMovedFiles(Map<String, Component> movedFilesByUuid, Map<String, DbComponent> dbFilesByUuid) {
+ movedFilesByUuid
+ .forEach((movedFileUuid, movedFile) -> {
+ DbComponent oldFile = getOldFile(dbFilesByUuid.values(), movedFile.getOldName());
+ movedFilesRepository.setOriginalFile(movedFile, toOriginalFile(oldFile));
+ });
+ }
+
+ private void registerNewlyAddedFiles(Map<String, Component> newAddedFilesByUuid) {
+ newAddedFilesByUuid
+ .values()
+ .forEach(addedFileRepository::register);
+ }
+
+ @NotNull
+ private Map<String, Component> getNewlyAddedFilesByUuid(Map<String, Component> reportFilesByUuid, Map<String, DbComponent> dbFilesByUuid) {
+ return reportFilesByUuid
+ .values()
+ .stream()
+ .filter(file -> Objects.isNull(file.getOldName()))
+ .filter(file -> !dbFilesByUuid.containsKey(file.getUuid()))
+ .collect(toMap(Component::getUuid, Function.identity()));
+ }
+
+ private Map<String, Component> getMovedFilesByUuid(Map<String, Component> reportFilesByUuid) {
+ return reportFilesByUuid
+ .values()
+ .stream()
+ .filter(file -> Objects.nonNull(file.getOldName()))
+ .collect(toMap(Component::getUuid, Function.identity()));
+ }
+
+ private DbComponent getOldFile(Collection<DbComponent> dbFiles, String oldFilePath) {
+ return dbFiles
+ .stream()
+ .filter(file -> file.getPath().equals(oldFilePath))
+ .findFirst()
+ .get();
+ }
+
+ public Set<String> difference(Set<String> set1, Set<String> set2) {
+ if (set1.isEmpty() || set2.isEmpty()) {
+ return set1;
+ }
+
+ return Sets.difference(set1, set2).immutableCopy();
+ }
+
+ private Map<String, DbComponent> getTargetBranchDbFilesByUuid(AnalysisMetadataHolder analysisMetadataHolder) {
+ try (DbSession dbSession = dbClient.openSession(false)) {
+ String targetBranchUuid = getTargetBranchUuid(dbSession, analysisMetadataHolder.getProject().getUuid(), analysisMetadataHolder.getBranch().getTargetBranchName());
+
+ return getTargetBranchDbFiles(dbSession, targetBranchUuid)
+ .stream()
+ .collect(toMap(DbComponent::getUuid, Function.identity()));
+ }
+ }
+
+ private List<DbComponent> getTargetBranchDbFiles(DbSession dbSession, String targetBranchUuid) {
+ List<DbComponent> files = new LinkedList();
+
+ ResultHandler<FileMoveRowDto> storeFileMove = resultContext -> {
+ FileMoveRowDto row = resultContext.getResultObject();
+ files.add(new DbComponent(row.getKey(), row.getUuid(), row.getPath(), row.getLineCount()));
+ };
+
+ dbClient.componentDao().scrollAllFilesForFileMove(dbSession, targetBranchUuid, storeFileMove);
+
+ return files;
+ }
+
+ private String getTargetBranchUuid(DbSession dbSession, String projectUuid, String targetBranchName) {
+ return dbClient.branchDao().selectByBranchKey(dbSession, projectUuid, targetBranchName)
+ .map(BranchDto::getUuid)
+ .orElseThrow(() -> new IllegalStateException("Pull Request has no target branch"));
+ }
+
+ private static Map<String, Component> getReportFilesByUuid(Component root) {
+ final ImmutableMap.Builder<String, Component> builder = ImmutableMap.builder();
+
+ new DepthTraversalTypeAwareCrawler(
+ new TypeAwareVisitorAdapter(CrawlerDepthLimit.FILE, POST_ORDER) {
+ @Override
+ public void visitFile(Component file) {
+ builder.put(file.getUuid(), file);
+ }
+ }).visit(root);
+
+ return builder.build();
+ }
+
+ private static MovedFilesRepository.OriginalFile toOriginalFile(DbComponent dbComponent) {
+ return new MovedFilesRepository.OriginalFile(dbComponent.getUuid(), dbComponent.getKey());
+ }
+
+ @Immutable
+ private static final class DbComponent {
+ private final String key;
+ private final String uuid;
+ private final String path;
+ private final int lineCount;
+
+ private DbComponent(String key, String uuid, String path, int lineCount) {
+ this.key = key;
+ this.uuid = uuid;
+ this.path = path;
+ this.lineCount = lineCount;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public int getLineCount() {
+ return lineCount;
+ }
+ }
+}
public TrackingResult track(Component component, Input<DefaultIssue> rawInput) {
if (analysisMetadataHolder.isPullRequest()) {
return standardResult(pullRequestTracker.track(component, rawInput));
- } else if (isFirstAnalysisSecondaryBranch()) {
+ }
+
+ if (isFirstAnalysisSecondaryBranch()) {
Tracking<DefaultIssue, DefaultIssue> tracking = referenceBranchTracker.track(component, rawInput);
return new TrackingResult(tracking.getMatchedRaws(), emptyMap(), empty(), tracking.getUnmatchedRaws());
- } else {
- return standardResult(tracker.track(component, rawInput));
}
+
+ return standardResult(tracker.track(component, rawInput));
}
private static TrackingResult standardResult(Tracking<DefaultIssue, DefaultIssue> tracking) {
"Issue %s doesn't belong to file %s registered as original file of current file %s",
issue, originalFile.getUuid(), component);
- // changes the issue's component uuid, and set issue as changed to enforce it is persisted to DB
+ // changes the issue's component uuid, and set issue as changed, to enforce it is persisted to DB
issueUpdater.setIssueComponent(issue, component.getUuid(), component.getKey(), new Date(analysisMetadataHolder.getAnalysisDate()));
}
}
public Input<DefaultIssue> create(Component component) {
if (component.getType() == Component.Type.PROJECT) {
return new ProjectTrackerBaseLazyInput(analysisMetadataHolder, componentsWithUnprocessedIssues, dbClient, issueUpdater, issuesLoader, reportModulesPath, component);
- } else if (component.getType() == Component.Type.DIRECTORY) {
+ }
+
+ if (component.getType() == Component.Type.DIRECTORY) {
// Folders have no issues
return new EmptyTrackerBaseLazyInput(dbClient, component);
}
+
return new FileTrackerBaseLazyInput(dbClient, component, movedFilesRepository.getOriginalFile(component).orElse(null));
}
import java.util.List;
import org.sonar.ce.task.container.TaskContainer;
import org.sonar.ce.task.projectanalysis.filemove.FileMoveDetectionStep;
+import org.sonar.ce.task.projectanalysis.filemove.PullRequestFileMoveDetectionStep;
import org.sonar.ce.task.projectanalysis.language.HandleUnanalyzedLanguagesStep;
import org.sonar.ce.task.projectanalysis.measure.PostMeasuresComputationChecksStep;
import org.sonar.ce.task.projectanalysis.purge.PurgeDatastoresStep;
LoadQualityGateStep.class,
LoadPeriodsStep.class,
FileMoveDetectionStep.class,
+ PullRequestFileMoveDetectionStep.class,
// load duplications related stuff
LoadDuplicationsFromReportStep.class,
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
*/
@Immutable
public class DefaultIndexedFile extends DefaultInputComponent implements IndexedFile {
- private static AtomicInteger intGenerator = new AtomicInteger(0);
+ private static final AtomicInteger intGenerator = new AtomicInteger(0);
private final String projectRelativePath;
private final String moduleRelativePath;
private final Type type;
private final Path absolutePath;
private final SensorStrategy sensorStrategy;
+ private final String oldFilePath;
/**
* Testing purposes only!
*/
public DefaultIndexedFile(String projectKey, Path baseDir, String relativePath, @Nullable String language) {
this(baseDir.resolve(relativePath), projectKey, relativePath, relativePath, Type.MAIN, language, intGenerator.getAndIncrement(),
- new SensorStrategy());
+ new SensorStrategy(), null);
}
public DefaultIndexedFile(Path absolutePath, String projectKey, String projectRelativePath, String moduleRelativePath, Type type, @Nullable String language, int batchId,
SensorStrategy sensorStrategy) {
+ this(absolutePath, projectKey, projectRelativePath, moduleRelativePath, type, language, batchId, sensorStrategy, null);
+ }
+
+ public DefaultIndexedFile(Path absolutePath, String projectKey, String projectRelativePath, String moduleRelativePath, Type type, @Nullable String language, int batchId,
+ SensorStrategy sensorStrategy, @Nullable String oldFilePath) {
super(batchId);
this.projectKey = projectKey;
this.projectRelativePath = PathUtils.sanitize(projectRelativePath);
this.language = language;
this.sensorStrategy = sensorStrategy;
this.absolutePath = absolutePath;
+ this.oldFilePath = oldFilePath;
}
@Override
return absolutePath;
}
+ @CheckForNull
+ public String oldPath() {
+ return oldFilePath;
+ }
+
+ public boolean isMovedFile() {
+ return Objects.nonNull(this.oldPath());
+ }
+
@Override
public InputStream inputStream() throws IOException {
return Files.newInputStream(path());
*/
@Override
public String key() {
- return new StringBuilder().append(projectKey).append(":").append(projectRelativePath).toString();
+ return String.join(":", projectKey, projectRelativePath);
}
@Override
return indexedFile.absolutePath();
}
+ @CheckForNull
+ public String oldPath() {
+ return indexedFile.oldPath();
+ }
+
+ public boolean isMovedFile() {
+ return indexedFile.isMovedFile();
+ }
+
@Override
public File file() {
return indexedFile.file();
import org.sonar.scanner.scan.branch.BranchConfiguration;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scm.ScmConfiguration;
+import org.sonar.scm.git.GitScmProvider;
import static java.util.Optional.empty;
Map<Path, DefaultInputFile> changedFiles = StreamSupport.stream(inputComponentStore.allChangedFilesToPublish().spliterator(), false)
.collect(Collectors.toMap(DefaultInputFile::path, f -> f));
- Map<Path, Set<Integer>> pathSetMap = provider.branchChangedLines(targetScmBranch, rootBaseDir, changedFiles.keySet());
+ Map<Path, Set<Integer>> pathSetMap = ((GitScmProvider) provider).branchChangedLines(targetScmBranch, rootBaseDir, changedFiles); // TODO: Extend ScmProvider abstract
int count = 0;
if (pathSetMap == null) {
fileBuilder.setStatus(convert(file.status()));
fileBuilder.setMarkedAsUnchanged(file.isMarkedAsUnchanged());
+ if (file.isMovedFile()) {
+ fileBuilder.setOldRelativeFilePath(file.oldPath());
+ }
+
String lang = getLanguageKey(file);
if (lang != null) {
fileBuilder.setLanguage(lang);
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.repository.language.Language;
import org.sonar.scanner.scan.ScanProperties;
+import org.sonar.scanner.scm.ScmChangedFiles;
import org.sonar.scanner.util.ProgressReport;
import static java.lang.String.format;
private final InputComponentStore componentStore;
private final SensorStrategy sensorStrategy;
private final LanguageDetection langDetection;
+ private final ScmChangedFiles scmChangedFiles;
private boolean warnInclusionsAlreadyLogged;
private boolean warnExclusionsAlreadyLogged;
public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore,
ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties,
- InputFileFilter[] filters) {
+ InputFileFilter[] filters, ScmChangedFiles scmChangedFiles) {
this.project = project;
this.scannerComponentIdGenerator = scannerComponentIdGenerator;
this.componentStore = componentStore;
this.properties = properties;
this.filters = filters;
this.projectExclusionFilters = projectExclusionFilters;
+ this.scmChangedFiles = scmChangedFiles;
}
void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
return;
}
- DefaultIndexedFile indexedFile = new DefaultIndexedFile(realAbsoluteFile, project.key(),
+ DefaultIndexedFile indexedFile = new DefaultIndexedFile(
+ realAbsoluteFile,
+ project.key(),
projectRelativePath.toString(),
moduleRelativePath.toString(),
- type, language != null ? language.key() : null, scannerComponentIdGenerator.getAsInt(), sensorStrategy);
+ type,
+ language != null ? language.key() : null,
+ scannerComponentIdGenerator.getAsInt(),
+ sensorStrategy,
+ scmChangedFiles.getFileOldPath(realAbsoluteFile)
+ );
+
DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()));
if (language != null && language.isPublishAllFiles()) {
inputFile.setPublished(true);
import java.nio.file.Path;
import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Predicate;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;
+import org.sonar.scm.git.ChangedFile;
@Immutable
public class ScmChangedFiles {
@Nullable
- private final Collection<Path> fileCollection;
+ private final Collection<ChangedFile> changedFiles;
- public ScmChangedFiles(@Nullable Collection<Path> changedFiles) {
- this.fileCollection = changedFiles;
+ public ScmChangedFiles(@Nullable Collection<ChangedFile> changedFiles) {
+ this.changedFiles = changedFiles;
}
public boolean isChanged(Path file) {
throw new IllegalStateException("Scm didn't provide valid data");
}
- return fileCollection.contains(file);
+ return this.findFile(file).isPresent();
}
public boolean isValid() {
- return fileCollection != null;
+ return changedFiles != null;
}
@CheckForNull
- Collection<Path> get() {
- return fileCollection;
+ public Collection<ChangedFile> get() {
+ return changedFiles;
+ }
+
+ @CheckForNull
+ public String getFileOldPath(Path absoluteFilePath) {
+ return this.findFile(absoluteFilePath)
+ .filter(ChangedFile::isMoved)
+ .map(ChangedFile::getOldFilePath)
+ .orElse(null);
+ }
+
+ private Optional<ChangedFile> findFile(Path absoluteFilePath) {
+ Predicate<ChangedFile> isTargetFile = file -> file.getAbsolutFilePath().equals(absoluteFilePath);
+
+ return Optional.ofNullable(this.get())
+ .orElseGet(List::of)
+ .stream()
+ .filter(isTargetFile)
+ .findFirst();
}
}
import java.nio.file.Path;
import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
-import org.sonar.api.batch.scm.ScmProvider;
import org.sonar.api.impl.utils.ScannerUtils;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.api.utils.log.Profiler;
import org.sonar.scanner.scan.branch.BranchConfiguration;
+import org.sonar.scm.git.ChangedFile;
+import org.sonar.scm.git.GitScmProvider;
import org.springframework.context.annotation.Bean;
public class ScmChangedFilesProvider {
@Bean("ScmChangedFiles")
public ScmChangedFiles provide(ScmConfiguration scmConfiguration, BranchConfiguration branchConfiguration, DefaultInputProject project) {
Path rootBaseDir = project.getBaseDir();
- Collection<Path> changedFiles = loadChangedFilesIfNeeded(scmConfiguration, branchConfiguration, rootBaseDir);
- validatePaths(changedFiles);
+ Collection<ChangedFile> changedFiles = loadChangedFilesIfNeeded(scmConfiguration, branchConfiguration, rootBaseDir);
+
+ if (changedFiles != null) {
+ validatePaths(getFilePaths(changedFiles));
+ }
+
return new ScmChangedFiles(changedFiles);
}
- private static void validatePaths(@javax.annotation.Nullable Collection<Path> paths) {
- if (paths != null && paths.stream().anyMatch(p -> !p.isAbsolute())) {
+ private static void validatePaths(Set<Path> changedFilePaths) {
+ if (changedFilePaths != null && changedFilePaths.stream().anyMatch(p -> !p.isAbsolute())) {
throw new IllegalStateException("SCM provider returned a changed file with a relative path but paths must be absolute. Please fix the provider.");
}
}
+ private static Set<Path> getFilePaths(Collection<ChangedFile> changedFiles) {
+ return changedFiles
+ .stream()
+ .map(ChangedFile::getAbsolutFilePath)
+ .collect(Collectors.toSet());
+ }
+
@CheckForNull
- private static Collection<Path> loadChangedFilesIfNeeded(ScmConfiguration scmConfiguration, BranchConfiguration branchConfiguration, Path rootBaseDir) {
+ private static Collection<ChangedFile> loadChangedFilesIfNeeded(ScmConfiguration scmConfiguration, BranchConfiguration branchConfiguration, Path rootBaseDir) {
final String targetBranchName = branchConfiguration.targetBranchName();
if (branchConfiguration.isPullRequest() && targetBranchName != null) {
- ScmProvider scmProvider = scmConfiguration.provider();
+ GitScmProvider scmProvider = (GitScmProvider) scmConfiguration.provider();
if (scmProvider != null) {
Profiler profiler = Profiler.create(LOG).startInfo(LOG_MSG);
- Collection<Path> changedFiles = scmProvider.branchChangedFiles(targetBranchName, rootBaseDir);
+ Collection<ChangedFile> changedFiles = scmProvider.branchModifiedFiles(targetBranchName, rootBaseDir);
profiler.stopInfo();
if (changedFiles != null) {
LOG.debug("SCM reported {} {} changed in the branch", changedFiles.size(), ScannerUtils.pluralize("file", changedFiles.size()));
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scm.git;
+
+import java.nio.file.Path;
+import java.util.Objects;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+public class ChangedFile {
+ @Nullable
+ private final String oldFilePath;
+ private final String filePath;
+ private final Path absoluteFilePath;
+
+ public ChangedFile(String filePath, Path absoluteFilePath) {
+ this(filePath, absoluteFilePath, null);
+ }
+
+ public ChangedFile(String filePath, Path absoluteFilePath, @Nullable String oldFilePath) {
+ this.filePath = filePath;
+ this.oldFilePath = oldFilePath;
+ this.absoluteFilePath = absoluteFilePath;
+ }
+
+ @CheckForNull
+ public String getOldFilePath() {
+ return oldFilePath;
+ }
+
+ public boolean isMoved() {
+ return Objects.nonNull(this.getOldFilePath());
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public Path getAbsolutFilePath() {
+ return absoluteFilePath;
+ }
+}
import java.io.IOException;
import java.nio.file.Path;
import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.CheckForNull;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.DiffAlgorithm;
import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.diff.RawTextComparator;
+import org.eclipse.jgit.diff.RenameDetector;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.FileTreeIterator;
import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
+import org.eclipse.jgit.treewalk.filter.TreeFilter;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.batch.scm.BlameCommand;
import org.sonar.api.batch.scm.ScmProvider;
import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
+import static org.eclipse.jgit.diff.DiffEntry.ChangeType.ADD;
+import static org.eclipse.jgit.diff.DiffEntry.ChangeType.MODIFY;
+import static org.eclipse.jgit.diff.DiffEntry.ChangeType.RENAME;
+
public class GitScmProvider extends ScmProvider {
private static final Logger LOG = Loggers.get(GitScmProvider.class);
return null;
}
+ // TODO: Adjust ScmProvider abstract
+ @CheckForNull
+ public Collection<ChangedFile> branchModifiedFiles(String targetBranchName, Path rootBaseDir) {
+ try (Repository repo = buildRepo(rootBaseDir)) {
+ Ref targetRef = resolveTargetRef(targetBranchName, repo);
+ if (targetRef == null) {
+ addWarningTargetNotFound(targetBranchName);
+ return null;
+ }
+
+ if (isDiffAlgoInvalid(repo.getConfig())) {
+ LOG.warn("The diff algorithm configured in git is not supported. "
+ + "No information regarding changes in the branch will be collected, which can lead to unexpected results.");
+ return null;
+ }
+
+ Optional<RevCommit> mergeBaseCommit = findMergeBase(repo, targetRef);
+ if (mergeBaseCommit.isEmpty()) {
+ LOG.warn("No merge base found between HEAD and " + targetRef.getName());
+ return null;
+ }
+ AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit.get());
+
+ // we compare a commit with HEAD, so no point ignoring line endings (it will be whatever is committed)
+ try (Git git = newGit(repo)) {
+ List<DiffEntry> diffEntries = git.diff()
+ .setShowNameAndStatusOnly(true)
+ .setOldTree(mergeBaseTree)
+ .setNewTree(prepareNewTree(repo))
+ .call();
+
+ return computeChangedFiles(repo, diffEntries);
+ }
+ } catch (IOException | GitAPIException e) {
+ LOG.warn(e.getMessage(), e);
+ }
+ return null;
+ }
+
+ private static List<ChangedFile> computeChangedFiles(Repository repository, List<DiffEntry> diffEntries) throws IOException {
+ Path workingDirectory = repository.getWorkTree().toPath();
+
+ Map<String, String> renamedFilePaths = computeRenamedFilePaths(repository, diffEntries);
+ Set<String> changedFilePaths = computeChangedFilePaths(diffEntries);
+
+ List<ChangedFile> changedFiles = new LinkedList<>();
+
+ Consumer<String> collectChangedFiles = filePath -> changedFiles.add(new ChangedFile(filePath, workingDirectory.resolve(filePath), renamedFilePaths.getOrDefault(filePath, null)));
+ changedFilePaths.forEach(collectChangedFiles);
+
+ return changedFiles;
+ }
+
+ private static Map<String, String> computeRenamedFilePaths(Repository repository, List<DiffEntry> diffEntries) throws IOException {
+ RenameDetector renameDetector = new RenameDetector(repository);
+ renameDetector.addAll(diffEntries);
+
+ return renameDetector
+ .compute()
+ .stream()
+ .filter(entry -> RENAME.equals(entry.getChangeType()))
+ .collect(Collectors.toUnmodifiableMap(DiffEntry::getNewPath, DiffEntry::getOldPath));
+ }
+
+ private static Set<String> computeChangedFilePaths(List<DiffEntry> diffEntries) {
+ return diffEntries
+ .stream()
+ .filter(isAllowedChangeType(ADD, MODIFY))
+ .map(DiffEntry::getNewPath)
+ .collect(Collectors.toSet());
+ }
+
+ private static Predicate<DiffEntry> isAllowedChangeType(ChangeType ...changeTypes) {
+ Function<ChangeType, Predicate<DiffEntry>> isChangeType = type -> entry -> type.equals(entry.getChangeType());
+
+ return Arrays
+ .stream(changeTypes)
+ .map(isChangeType)
+ .reduce(x -> false, Predicate::or);
+ }
+
@CheckForNull
@Override
public Map<Path, Set<Integer>> branchChangedLines(String targetBranchName, Path projectBaseDir, Set<Path> changedFiles) {
return null;
}
+ // TODO: Adjust ScmProvider abstract
+ public Map<Path, Set<Integer>> branchChangedLines(String targetBranchName, Path projectBaseDir, Map<Path, DefaultInputFile> changedFiles) {
+ try (Repository repo = buildRepo(projectBaseDir)) {
+ Ref targetRef = resolveTargetRef(targetBranchName, repo);
+ if (targetRef == null) {
+ addWarningTargetNotFound(targetBranchName);
+ return null;
+ }
+
+ if (isDiffAlgoInvalid(repo.getConfig())) {
+ // we already print a warning when branchChangedFiles is called
+ return null;
+ }
+
+ // force ignore different line endings when comparing a commit with the workspace
+ repo.getConfig().setBoolean("core", null, "autocrlf", true);
+
+ Optional<RevCommit> mergeBaseCommit = findMergeBase(repo, targetRef);
+
+ if (mergeBaseCommit.isEmpty()) {
+ LOG.warn("No merge base found between HEAD and " + targetRef.getName());
+ return null;
+ }
+
+ Map<Path, Set<Integer>> changedLines = new HashMap<>();
+
+ for (Map.Entry<Path, DefaultInputFile> entry : changedFiles.entrySet()) {
+ collectChangedLines(repo, mergeBaseCommit.get(), changedLines, entry.getKey(), entry.getValue());
+ }
+
+ return changedLines;
+ } catch (Exception e) {
+ LOG.warn("Failed to get changed lines from git", e);
+ }
+ return null;
+ }
+
private void addWarningTargetNotFound(String targetBranchName) {
analysisWarnings.addUnique(String.format(COULD_NOT_FIND_REF
+ ". You may see unexpected issues and changes. "
}
}
+ // TODO: Adjust ScmProvider abstract
+ private void collectChangedLines(Repository repo, RevCommit mergeBaseCommit, Map<Path, Set<Integer>> changedLines, Path changedFilePath, DefaultInputFile changedFileData) {
+ ChangedLinesComputer computer = new ChangedLinesComputer();
+
+ try (DiffFormatter diffFmt = new DiffFormatter(new BufferedOutputStream(computer.receiver()))) {
+ diffFmt.setRepository(repo);
+ diffFmt.setProgressMonitor(NullProgressMonitor.INSTANCE);
+ diffFmt.setDiffComparator(RawTextComparator.WS_IGNORE_ALL);
+
+ diffFmt.setDetectRenames(changedFileData.isMovedFile());
+
+ Path workTree = repo.getWorkTree().toPath();
+ TreeFilter treeFilter = getTreeFilter(changedFileData, workTree);
+ diffFmt.setPathFilter(treeFilter);
+
+ AbstractTreeIterator mergeBaseTree = prepareTreeParser(repo, mergeBaseCommit);
+ List<DiffEntry> diffEntries = diffFmt.scan(mergeBaseTree, new FileTreeIterator(repo));
+
+ diffFmt.format(diffEntries);
+ diffFmt.flush();
+
+ diffEntries.stream()
+ .filter(isAllowedChangeType(ADD, MODIFY, RENAME))
+ .findAny()
+ .ifPresent(diffEntry -> changedLines.put(changedFilePath, computer.changedLines()));
+ } catch (Exception e) {
+ LOG.warn("Failed to get changed lines from git for file " + changedFilePath, e);
+ }
+ }
+
@Override
@CheckForNull
public Instant forkDate(String referenceBranchName, Path projectBaseDir) {
return path.replaceAll(Pattern.quote(File.separator), "/");
}
+ private TreeFilter getTreeFilter(DefaultInputFile changedFile, Path baseDir) {
+ String oldPath = toGitPath(changedFile.oldPath());
+ String path = toGitPath(relativizeFilePath(baseDir, changedFile.path()));
+
+ if (changedFile.isMovedFile()) {
+ return PathFilterGroup.createFromStrings(path, oldPath);
+ }
+
+ return PathFilter.create(path);
+ }
+
@CheckForNull
private Ref resolveTargetRef(String targetBranchName, Repository repo) throws IOException {
String localRef = "refs/heads/" + targetBranchName;
}
}
+ private static String relativizeFilePath(Path baseDirectory, Path filePath) {
+ return baseDirectory.relativize(filePath).toString();
+ }
+
AbstractTreeIterator prepareTreeParser(Repository repo, RevCommit commit) throws IOException {
CanonicalTreeParser treeParser = new CanonicalTreeParser();
try (ObjectReader objectReader = repo.newObjectReader()) {
*/
package org.sonar.scanner.scan.filesystem;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.junit.Test;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.scanner.repository.ProjectRepositories;
import org.sonar.scanner.repository.SingleProjectRepository;
import org.sonar.scanner.scm.ScmChangedFiles;
+import org.sonar.scm.git.ChangedFile;
import static org.assertj.core.api.Assertions.assertThat;
@Test
public void detect_status_branches_confirm() {
- ScmChangedFiles changedFiles = new ScmChangedFiles(Collections.singletonList(Paths.get("module", "src", "Foo.java")));
- StatusDetection statusDetection = new StatusDetection(projectRepositories, changedFiles);
+ Path filePath = Paths.get("module", "src", "Foo.java");
+ ScmChangedFiles changedFiles = new ScmChangedFiles(List.of(new ChangedFile(filePath.toString(), filePath)));
+ StatusDetection statusDetection = new StatusDetection(projectRepositories, changedFiles);
assertThat(statusDetection.status("foo", createFile("src/Foo.java"), "XXXXX")).isEqualTo(InputFile.Status.CHANGED);
}
import org.sonar.api.batch.scm.ScmProvider;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.scan.branch.BranchConfiguration;
+import org.sonar.scm.git.ChangedFile;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
when(scmProvider.branchChangedFiles("target", rootBaseDir)).thenReturn(Collections.singleton(Paths.get("changedFile").toAbsolutePath()));
ScmChangedFiles scmChangedFiles = provider.provide(scmConfiguration, branchConfiguration, project);
- assertThat(scmChangedFiles.get()).containsOnly(Paths.get("changedFile").toAbsolutePath());
+ Path filePath = Paths.get("changedFile").toAbsolutePath();
+ ChangedFile changedFile = new ChangedFile(filePath.toString(), filePath);
+ assertThat(scmChangedFiles.get()).containsOnly(changedFile);
verify(scmProvider).branchChangedFiles("target", rootBaseDir);
}
import java.util.Collection;
import java.util.Collections;
import org.junit.Test;
+import org.sonar.scm.git.ChangedFile;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@Test
public void testGetter() {
- Collection<Path> files = Collections.singletonList(Paths.get("files"));
+ Path filePath = Paths.get("files");
+ ChangedFile file = new ChangedFile(filePath.toString(), filePath);
+ Collection<ChangedFile> files = Collections.singletonList(file);
scmChangedFiles = new ScmChangedFiles(files);
- assertThat(scmChangedFiles.get()).containsOnly(Paths.get("files"));
+ assertThat(scmChangedFiles.get()).containsOnly(file);
}
@Test
@Test
public void testConfirm() {
- Collection<Path> files = Collections.singletonList(Paths.get("files"));
+ Path filePath = Paths.get("files");
+ Collection<ChangedFile> files = Collections.singletonList(new ChangedFile(filePath.toString(), filePath));
scmChangedFiles = new ScmChangedFiles(files);
assertThat(scmChangedFiles.isValid()).isTrue();
assertThat(scmChangedFiles.isChanged(Paths.get("files"))).isTrue();
// Path relative to project base directory
string project_relative_path = 14;
bool markedAsUnchanged = 15;
+ string old_relative_file_path = 16;
enum ComponentType {
UNSET = 0;
projects / sonarqube.git / commitdiff