SONAR-16080 Git submodules not considered when loading exclusions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java
index d3c463dbb30c815c3ae786c181d472a3661e9169..73bc85aeb26a7383bd77ca3038e429398cacf324 100644 (file)
--- a/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scm/git/IncludedFilesRepository.java
@@ -24,6 +24,7 @@ import java.nio.file.Path;
 import java.util.HashSet;
 import java.util.Set;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.submodule.SubmoduleWalk;
 import org.eclipse.jgit.treewalk.FileTreeIterator;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.treewalk.WorkingTreeIterator;
@@ -47,23 +48,40 @@ public class IncludedFilesRepository {
 
   private void indexFiles(Path baseDir) throws IOException {
     try (Repository repo = JGitUtils.buildRepository(baseDir)) {
-      Path workTreeRoot = repo.getWorkTree().toPath();
-      FileTreeIterator workingTreeIt = new FileTreeIterator(repo);
-      try (TreeWalk treeWalk = new TreeWalk(repo)) {
-        treeWalk.setRecursive(true);
-        if (!baseDir.equals(workTreeRoot)) {
-          Path relativeBaseDir = workTreeRoot.relativize(baseDir);
-          treeWalk.setFilter(PathFilterGroup.createFromStrings(relativeBaseDir.toString().replace('\\', '/')));
+      collectFilesIterative(repo, baseDir);
+    }
+  }
+
+  private void collectFiles(Repository repo, Path baseDir) throws IOException {
+    Path workTreeRoot = repo.getWorkTree().toPath();
+    FileTreeIterator workingTreeIt = new FileTreeIterator(repo);
+    try (TreeWalk treeWalk = new TreeWalk(repo)) {
+      treeWalk.setRecursive(true);
+      // with submodules, the baseDir may be the parent of the workTreeRoot. In that case, we don't want to set a filter.
+      if (!workTreeRoot.equals(baseDir) && baseDir.startsWith(workTreeRoot)) {
+        Path relativeBaseDir = workTreeRoot.relativize(baseDir);
+        treeWalk.setFilter(PathFilterGroup.createFromStrings(relativeBaseDir.toString().replace('\\', '/')));
+      }
+      treeWalk.addTree(workingTreeIt);
+      while (treeWalk.next()) {
+
+        WorkingTreeIterator workingTreeIterator = treeWalk
+          .getTree(0, WorkingTreeIterator.class);
+
+        if (!workingTreeIterator.isEntryIgnored()) {
+          includedFiles.add(workTreeRoot.resolve(treeWalk.getPathString()));
         }
-        treeWalk.addTree(workingTreeIt);
-        while (treeWalk.next()) {
+      }
+    }
+  }
 
-          WorkingTreeIterator workingTreeIterator = treeWalk
-            .getTree(0, WorkingTreeIterator.class);
+  private void collectFilesIterative(Repository repo, Path baseDir) throws IOException {
+    collectFiles(repo, baseDir);
 
-          if (!workingTreeIterator.isEntryIgnored()) {
-            includedFiles.add(workTreeRoot.resolve(treeWalk.getPathString()));
-          }
+    try (SubmoduleWalk submoduleWalk = SubmoduleWalk.forIndex(repo)) {
+      while (submoduleWalk.next()) {
+        try (Repository submoduleRepo = submoduleWalk.getRepository()) {
+          collectFilesIterative(submoduleRepo, baseDir);
         }
       }
     }

diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java
index cf1d8e7b409376a69d1502ac448b80fdb8e4e224..ed43b81be6ffda7e377e75de4a36bf3f01f17a53 100644 (file)
--- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/GitIgnoreCommandTest.java
@@ -20,18 +20,22 @@
 package org.sonar.scm.git;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.SubmoduleAddCommand;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.Repository;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.sonar.api.utils.log.LogTester;
 import org.sonar.api.utils.log.LoggerLevel;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.sonar.scm.git.Utils.javaUnzip;
 
@@ -64,13 +68,10 @@ public class GitIgnoreCommandTest {
 
   @Test
   public void test_pattern_on_deep_repo() throws Exception {
-    Path projectDir = temp.newFolder().toPath();
-    Git.init().setDirectory(projectDir.toFile()).call();
-
-    Files.write(projectDir.resolve(".gitignore"), Arrays.asList("**/*.java"), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
+    Path projectDir = createGitRepoWithIgnore();
     int child_folders_per_folder = 2;
     int folder_depth = 10;
-    createDeepFolderStructure(projectDir, child_folders_per_folder, 0, folder_depth);
+    createFolderStructure(projectDir, child_folders_per_folder, 0, folder_depth);
 
     logTester.setLevel(LoggerLevel.DEBUG);
 
@@ -79,24 +80,48 @@ public class GitIgnoreCommandTest {
 
     assertThat(underTest
       .isIgnored(projectDir.resolve("folder_0_0/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.java")))
-        .isTrue();
+      .isTrue();
     assertThat(underTest
       .isIgnored(projectDir.resolve("folder_0_0/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php")))
-        .isFalse();
+      .isFalse();
 
     int expectedIncludedFiles = (int) Math.pow(child_folders_per_folder, folder_depth) + 1; // The .gitignore file is indexed
     assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(expectedIncludedFiles + " non excluded files in this Git repository");
   }
 
+  @Test
+  public void include_submodules() throws IOException, GitAPIException {
+    Path projectDir = temp.newFolder().toPath();
+    Git git = Git.init().setDirectory(projectDir.toFile()).call();
+
+    createSubmoduleWithFiles(git, "module1");
+
+    Files.write(projectDir.resolve(".gitignore"), Arrays.asList("**/*.java"), UTF_8, TRUNCATE_EXISTING, CREATE);
+    createFolderStructure(projectDir, 1, 0, 1);
+
+    logTester.setLevel(LoggerLevel.DEBUG);
+
+    GitIgnoreCommand underTest = new GitIgnoreCommand();
+    underTest.init(projectDir);
+
+    assertThat(underTest.isIgnored(projectDir.resolve("folder_0_0/Foo.java"))).isTrue();
+    assertThat(underTest.isIgnored(projectDir.resolve("folder_0_0/Foo.php"))).isFalse();
+
+    // also applies to files in submodule
+    assertThat(underTest.isIgnored(projectDir.resolve("module1/folder_0_0/Foo.java"))).isTrue();
+    assertThat(underTest.isIgnored(projectDir.resolve("module1/folder_0_0/Foo.php"))).isFalse();
+
+    int expectedIncludedFiles = 6;
+    assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(expectedIncludedFiles + " non excluded files in this Git repository");
+  }
+
   @Test
   public void dont_index_files_outside_basedir() throws Exception {
-    Path repoRoot = temp.newFolder().toPath();
-    Git.init().setDirectory(repoRoot.toFile()).call();
+    Path repoRoot = createGitRepoWithIgnore();
 
-    Files.write(repoRoot.resolve(".gitignore"), Arrays.asList("**/*.java"), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
     int child_folders_per_folder = 2;
     int folder_depth = 10;
-    createDeepFolderStructure(repoRoot, child_folders_per_folder, 0, folder_depth);
+    createFolderStructure(repoRoot, child_folders_per_folder, 0, folder_depth);
 
     logTester.setLevel(LoggerLevel.DEBUG);
 
@@ -107,16 +132,45 @@ public class GitIgnoreCommandTest {
 
     assertThat(underTest
       .isIgnored(projectBasedir.resolve("folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php")))
-        .isFalse();
+      .isFalse();
     assertThat(underTest
       .isIgnored(repoRoot.resolve("folder_0_1/folder_1_0/folder_2_0/folder_3_0/folder_4_0/folder_5_0/folder_6_0/folder_7_0/folder_8_0/folder_9_0/Foo.php")))
-        .isTrue();
+      .isTrue();
 
     int expectedIncludedFiles = (int) Math.pow(child_folders_per_folder, folder_depth - 1);
     assertThat(logTester.logs(LoggerLevel.DEBUG)).contains(expectedIncludedFiles + " non excluded files in this Git repository");
   }
 
-  private void createDeepFolderStructure(Path current, int childCount, int currentDepth, int maxDepth) throws IOException {
+  private Path createGitRepoWithIgnore() throws IOException, GitAPIException {
+    Path repoRoot = temp.newFolder().toPath();
+    try (Git git = Git.init().setDirectory(repoRoot.toFile()).call()) {
+    }
+    Files.write(repoRoot.resolve(".gitignore"), Arrays.asList("**/*.java"), UTF_8, TRUNCATE_EXISTING, CREATE);
+    return repoRoot;
+  }
+
+  private void createSubmoduleWithFiles(Git git, String path) throws IOException, GitAPIException {
+    // create the other git repository
+    Path subRoot = temp.newFolder().toPath();
+    Files.write(subRoot.resolve(".gitignore"), Arrays.asList("**/*.java"), UTF_8, TRUNCATE_EXISTING, CREATE);
+    createFolderStructure(subRoot, 1, 0, 1);
+
+    try (Git subGit = Git.init().setDirectory(subRoot.toFile()).call()) {
+      subGit.add().addFilepattern(".").call();
+      subGit.commit().setMessage("first").call();
+    }
+
+    // add the other git repo as a submodule
+    SubmoduleAddCommand addCommand = git.submoduleAdd()
+      .setURI(subRoot.toUri().toString())
+      .setPath(path);
+    try (Repository module = addCommand.call()) {
+
+    }
+    git.submoduleUpdate().call();
+  }
+
+  private void createFolderStructure(Path current, int childCount, int currentDepth, int maxDepth) throws IOException {
     if (currentDepth >= maxDepth) {
       Path javaFile = current.resolve("Foo.java");
       Path phpFile = current.resolve("Foo.php");
@@ -133,7 +187,7 @@ public class GitIgnoreCommandTest {
       if (!Files.exists(newPath)) {
         Files.createDirectory(newPath);
       }
-      createDeepFolderStructure(newPath, childCount, currentDepth + 1, maxDepth);
+      createFolderStructure(newPath, childCount, currentDepth + 1, maxDepth);
     }
   }