SONAR-13579 Detect files moves in Pull Request scope

SONAR-13579 Get database files from target branch instead of snapshot
SONAR-13579 Store old relative file path to `FileAttributes` class

10 files changed, 283 insertions, 8 deletions

diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java
index 4889401417a..69302fcbfcd 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/Component.java
@@ -88,6 +88,11 @@ public interface Component {
   String getName();
 
   /**
+   * Get component old relative path.
+   */
+  String getOldName();
+
+  /**
    * The component short name. For files and directories this is the parent relative path (ie filename for files). For projects and view this is the same as {@link #getName()}
    */
   String getShortName();
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentImpl.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentImpl.java
index a9ac6fbbd26..b06e66bb465 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentImpl.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentImpl.java
@@ -93,6 +93,11 @@ public class ComponentImpl implements Component {
   }
 
   @Override
+  public String getOldName() {
+    return this.getFileAttributes().getOldName();
+  }
+
+  @Override
   public String getShortName() {
     return this.shortName;
   }
@@ -159,6 +164,7 @@ public class ComponentImpl implements Component {
     private String uuid;
     private String key;
     private String name;
+    private String oldName;
     private String shortName;
     private String description;
     private FileAttributes fileAttributes;
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
index 4ca4a8ab048..d4f07d5cbdc 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/ComponentTreeBuilder.java
@@ -335,7 +335,9 @@ public class ComponentTreeBuilder {
       component.getIsTest(),
       lang != null ? lang.intern() : null,
       component.getLines(),
-      component.getMarkedAsUnchanged());
+      component.getMarkedAsUnchanged(),
+      component.getOldRelativeFilePath()
+    );
   }
 
   private static class Node {
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
index c1c868ec1f9..fc62e592e14 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/component/FileAttributes.java
@@ -24,6 +24,9 @@ import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.commons.lang.StringUtils.abbreviate;
+import static org.apache.commons.lang.StringUtils.trimToNull;
+import static org.sonar.db.component.ComponentValidator.MAX_COMPONENT_NAME_LENGTH;
 
 /**
  * The attributes specific to a Component of type {@link Component.Type#FILE}.
@@ -35,17 +38,19 @@ public class FileAttributes {
   private final String languageKey;
   private final boolean markedAsUnchanged;
   private final int lines;
+  private String oldName;
 
   public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines) {
-    this(unitTest, languageKey, lines, false);
+    this(unitTest, languageKey, lines, false, null);
   }
 
-  public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged) {
+  public FileAttributes(boolean unitTest, @Nullable String languageKey, int lines, boolean markedAsUnchanged, @Nullable String oldName) {
     this.unitTest = unitTest;
     this.languageKey = languageKey;
     this.markedAsUnchanged = markedAsUnchanged;
     checkArgument(lines > 0, "Number of lines must be greater than zero");
     this.lines = lines;
+    this.oldName = formatOldName(oldName);
   }
 
   public boolean isMarkedAsUnchanged() {
@@ -61,6 +66,11 @@ public class FileAttributes {
     return languageKey;
   }
 
+  @CheckForNull
+  public String getOldName() {
+    return oldName;
+  }
+
   /**
    * Number of lines of the file, can never be less than 1
    */
@@ -75,6 +85,11 @@ public class FileAttributes {
       ", unitTest=" + unitTest +
       ", lines=" + lines +
       ", markedAsUnchanged=" + markedAsUnchanged +
+      ", oldName=" + oldName +
       '}';
   }
+
+  private String formatOldName(@Nullable String name) {
+    return abbreviate(trimToNull(name), MAX_COMPONENT_NAME_LENGTH);
+  }
 }
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStep.java
index 2b484eea014..7bddb40e717 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/FileMoveDetectionStep.java
@@ -97,6 +97,11 @@ public class FileMoveDetectionStep implements ComputationStep {
 
   @Override
   public void execute(ComputationStep.Context context) {
+    if (analysisMetadataHolder.isPullRequest()) {
+      LOG.debug("Currently within Pull Request scope. Do nothing.");
+      return;
+    }
+
     // do nothing if no files in db (first analysis)
     if (analysisMetadataHolder.isFirstAnalysis()) {
       LOG.debug("First analysis. Do nothing.");
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/PullRequestFileMoveDetectionStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/PullRequestFileMoveDetectionStep.java
new file mode 100644
index 00000000000..77f172f8763
--- /dev/null
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/filemove/PullRequestFileMoveDetectionStep.java
@@ -0,0 +1,235 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.ce.task.projectanalysis.filemove;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import javax.annotation.concurrent.Immutable;
+import org.apache.ibatis.session.ResultHandler;
+import org.jetbrains.annotations.NotNull;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
+import org.sonar.ce.task.projectanalysis.component.Component;
+import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit;
+import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler;
+import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
+import org.sonar.ce.task.projectanalysis.component.TypeAwareVisitorAdapter;
+import org.sonar.ce.task.step.ComputationStep;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.component.BranchDto;
+import org.sonar.db.component.FileMoveRowDto;
+
+import static java.util.stream.Collectors.toMap;
+import static org.sonar.ce.task.projectanalysis.component.ComponentVisitor.Order.POST_ORDER;
+
+public class PullRequestFileMoveDetectionStep implements ComputationStep {
+  private static final Logger LOG = Loggers.get(PullRequestFileMoveDetectionStep.class);
+
+  private final AnalysisMetadataHolder analysisMetadataHolder;
+  private final TreeRootHolder rootHolder;
+  private final DbClient dbClient;
+  private final MutableMovedFilesRepository movedFilesRepository;
+  private final MutableAddedFileRepository addedFileRepository;
+
+  public PullRequestFileMoveDetectionStep(AnalysisMetadataHolder analysisMetadataHolder, TreeRootHolder rootHolder, DbClient dbClient,
+    MutableMovedFilesRepository movedFilesRepository, MutableAddedFileRepository addedFileRepository) {
+    this.analysisMetadataHolder = analysisMetadataHolder;
+    this.rootHolder = rootHolder;
+    this.dbClient = dbClient;
+    this.movedFilesRepository = movedFilesRepository;
+    this.addedFileRepository = addedFileRepository;
+  }
+
+  @Override
+  public String getDescription() {
+    return "Detect file moves in Pull Request scope";
+  }
+
+  @Override
+  public void execute(ComputationStep.Context context) {
+    if (!analysisMetadataHolder.isPullRequest()) {
+      LOG.debug("Currently not within Pull Request scope. Do nothing.");
+      return;
+    }
+
+    Map<String, Component> reportFilesByUuid = getReportFilesByUuid(this.rootHolder.getRoot());
+    context.getStatistics().add("reportFiles", reportFilesByUuid.size());
+
+    if (reportFilesByUuid.isEmpty()) {
+      LOG.debug("No files in report. No file move detection.");
+      return;
+    }
+
+    Map<String, DbComponent> targetBranchDbFilesByUuid = getTargetBranchDbFilesByUuid(analysisMetadataHolder);
+    context.getStatistics().add("dbFiles", targetBranchDbFilesByUuid.size());
+
+    if (targetBranchDbFilesByUuid.isEmpty()) {
+      registerNewlyAddedFiles(reportFilesByUuid);
+      context.getStatistics().add("addedFiles", reportFilesByUuid.size());
+      LOG.debug("Previous snapshot has no file. No file move detection.");
+      return;
+    }
+
+    Map<String, Component> movedFilesByUuid = getMovedFilesByUuid(reportFilesByUuid);
+    context.getStatistics().add("movedFiles", movedFilesByUuid.size());
+
+    Map<String, Component> newlyAddedFilesByUuid = getNewlyAddedFilesByUuid(reportFilesByUuid, targetBranchDbFilesByUuid);
+    context.getStatistics().add("addedFiles", newlyAddedFilesByUuid.size());
+
+    // Do we need to register the moved file in the moved files repo and use the data in the related steps/visitors?
+//    registerMovedFiles(movedFilesByUuid, targetBranchDbFilesByUuid);
+    registerNewlyAddedFiles(newlyAddedFilesByUuid);
+  }
+
+  private void registerMovedFiles(Map<String, Component> movedFilesByUuid, Map<String, DbComponent> dbFilesByUuid) {
+    movedFilesByUuid
+      .forEach((movedFileUuid, movedFile) -> {
+        DbComponent oldFile = getOldFile(dbFilesByUuid.values(), movedFile.getOldName());
+        movedFilesRepository.setOriginalFile(movedFile, toOriginalFile(oldFile));
+      });
+  }
+
+  private void registerNewlyAddedFiles(Map<String, Component> newAddedFilesByUuid) {
+    newAddedFilesByUuid
+      .values()
+      .forEach(addedFileRepository::register);
+  }
+
+  @NotNull
+  private Map<String, Component> getNewlyAddedFilesByUuid(Map<String, Component> reportFilesByUuid, Map<String, DbComponent> dbFilesByUuid) {
+    return reportFilesByUuid
+      .values()
+      .stream()
+      .filter(file -> Objects.isNull(file.getOldName()))
+      .filter(file -> !dbFilesByUuid.containsKey(file.getUuid()))
+      .collect(toMap(Component::getUuid, Function.identity()));
+  }
+
+  private Map<String, Component> getMovedFilesByUuid(Map<String, Component> reportFilesByUuid) {
+    return reportFilesByUuid
+      .values()
+      .stream()
+      .filter(file -> Objects.nonNull(file.getOldName()))
+      .collect(toMap(Component::getUuid, Function.identity()));
+  }
+
+  private DbComponent getOldFile(Collection<DbComponent> dbFiles, String oldFilePath) {
+    return dbFiles
+      .stream()
+      .filter(file -> file.getPath().equals(oldFilePath))
+      .findFirst()
+      .get();
+  }
+
+  public Set<String> difference(Set<String> set1, Set<String> set2) {
+    if (set1.isEmpty() || set2.isEmpty()) {
+      return set1;
+    }
+
+    return Sets.difference(set1, set2).immutableCopy();
+  }
+
+  private Map<String, DbComponent> getTargetBranchDbFilesByUuid(AnalysisMetadataHolder analysisMetadataHolder) {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      String targetBranchUuid = getTargetBranchUuid(dbSession, analysisMetadataHolder.getProject().getUuid(), analysisMetadataHolder.getBranch().getTargetBranchName());
+
+      return getTargetBranchDbFiles(dbSession, targetBranchUuid)
+        .stream()
+        .collect(toMap(DbComponent::getUuid, Function.identity()));
+    }
+  }
+
+  private List<DbComponent> getTargetBranchDbFiles(DbSession dbSession, String targetBranchUuid) {
+     List<DbComponent> files = new LinkedList();
+
+    ResultHandler<FileMoveRowDto> storeFileMove = resultContext -> {
+      FileMoveRowDto row = resultContext.getResultObject();
+      files.add(new DbComponent(row.getKey(), row.getUuid(), row.getPath(), row.getLineCount()));
+    };
+
+    dbClient.componentDao().scrollAllFilesForFileMove(dbSession, targetBranchUuid, storeFileMove);
+
+    return files;
+  }
+
+  private String getTargetBranchUuid(DbSession dbSession, String projectUuid, String targetBranchName) {
+      return dbClient.branchDao().selectByBranchKey(dbSession, projectUuid, targetBranchName)
+        .map(BranchDto::getUuid)
+        .orElseThrow(() -> new IllegalStateException("Pull Request has no target branch"));
+  }
+
+  private static Map<String, Component> getReportFilesByUuid(Component root) {
+    final ImmutableMap.Builder<String, Component> builder = ImmutableMap.builder();
+
+    new DepthTraversalTypeAwareCrawler(
+      new TypeAwareVisitorAdapter(CrawlerDepthLimit.FILE, POST_ORDER) {
+        @Override
+        public void visitFile(Component file) {
+          builder.put(file.getUuid(), file);
+        }
+      }).visit(root);
+
+    return builder.build();
+  }
+
+  private static MovedFilesRepository.OriginalFile toOriginalFile(DbComponent dbComponent) {
+    return new MovedFilesRepository.OriginalFile(dbComponent.getUuid(), dbComponent.getKey());
+  }
+
+  @Immutable
+  private static final class DbComponent {
+    private final String key;
+    private final String uuid;
+    private final String path;
+    private final int lineCount;
+
+    private DbComponent(String key, String uuid, String path, int lineCount) {
+      this.key = key;
+      this.uuid = uuid;
+      this.path = path;
+      this.lineCount = lineCount;
+    }
+
+    public String getKey() {
+      return key;
+    }
+
+    public String getUuid() {
+      return uuid;
+    }
+
+    public String getPath() {
+      return path;
+    }
+
+    public int getLineCount() {
+      return lineCount;
+    }
+  }
+}
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueTrackingDelegator.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueTrackingDelegator.java
index f1226671b35..0aab874f3b1 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueTrackingDelegator.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/IssueTrackingDelegator.java
@@ -45,12 +45,14 @@ public class IssueTrackingDelegator {
   public TrackingResult track(Component component, Input<DefaultIssue> rawInput) {
     if (analysisMetadataHolder.isPullRequest()) {
       return standardResult(pullRequestTracker.track(component, rawInput));
-    } else if (isFirstAnalysisSecondaryBranch()) {
+    }
+
+    if (isFirstAnalysisSecondaryBranch()) {
       Tracking<DefaultIssue, DefaultIssue> tracking = referenceBranchTracker.track(component, rawInput);
       return new TrackingResult(tracking.getMatchedRaws(), emptyMap(), empty(), tracking.getUnmatchedRaws());
-    } else {
-      return standardResult(tracker.track(component, rawInput));
     }
+
+    return standardResult(tracker.track(component, rawInput));
   }
 
   private static TrackingResult standardResult(Tracking<DefaultIssue, DefaultIssue> tracking) {
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/MovedIssueVisitor.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/MovedIssueVisitor.java
index e1091282746..4c31b335b46 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/MovedIssueVisitor.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/MovedIssueVisitor.java
@@ -55,7 +55,7 @@ public class MovedIssueVisitor extends IssueVisitor {
       "Issue %s doesn't belong to file %s registered as original file of current file %s",
       issue, originalFile.getUuid(), component);
 
-    // changes the issue's component uuid, and set issue as changed to enforce it is persisted to DB
+    // changes the issue's component uuid, and set issue as changed, to enforce it is persisted to DB
     issueUpdater.setIssueComponent(issue, component.getUuid(), component.getKey(), new Date(analysisMetadataHolder.getAnalysisDate()));
   }
 }
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerBaseInputFactory.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerBaseInputFactory.java
index 2ba64a8e3e7..370dbd93c44 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerBaseInputFactory.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/issue/TrackerBaseInputFactory.java
@@ -59,10 +59,13 @@ public class TrackerBaseInputFactory extends BaseInputFactory {
   public Input<DefaultIssue> create(Component component) {
     if (component.getType() == Component.Type.PROJECT) {
       return new ProjectTrackerBaseLazyInput(analysisMetadataHolder, componentsWithUnprocessedIssues, dbClient, issueUpdater, issuesLoader, reportModulesPath, component);
-    } else if (component.getType() == Component.Type.DIRECTORY) {
+    }
+
+    if (component.getType() == Component.Type.DIRECTORY) {
       // Folders have no issues
       return new EmptyTrackerBaseLazyInput(dbClient, component);
     }
+
     return new FileTrackerBaseLazyInput(dbClient, component, movedFilesRepository.getOriginalFile(component).orElse(null));
   }
 
diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
index a482de93c0d..e5a7b038052 100644
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.List;
 import org.sonar.ce.task.container.TaskContainer;
 import org.sonar.ce.task.projectanalysis.filemove.FileMoveDetectionStep;
+import org.sonar.ce.task.projectanalysis.filemove.PullRequestFileMoveDetectionStep;
 import org.sonar.ce.task.projectanalysis.language.HandleUnanalyzedLanguagesStep;
 import org.sonar.ce.task.projectanalysis.measure.PostMeasuresComputationChecksStep;
 import org.sonar.ce.task.projectanalysis.purge.PurgeDatastoresStep;
@@ -54,6 +55,7 @@ public class ReportComputationSteps extends AbstractComputationSteps {
     LoadQualityGateStep.class,
     LoadPeriodsStep.class,
     FileMoveDetectionStep.class,
+    PullRequestFileMoveDetectionStep.class,
 
     // load duplications related stuff
     LoadDuplicationsFromReportStep.class,