SONAR-21195 Refactor file indexing into two distinct steps

3 files changed, 160 insertions, 7 deletions

diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
new file mode 100644
index 00000000000..b7794c959a8
--- /dev/null
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/DirectoryFileVisitorTest.java
@@ -0,0 +1,93 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystemLoopException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import org.apache.commons.lang.SystemUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.scanner.fs.InputModuleHierarchy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class DirectoryFileVisitorTest {
+
+  @ClassRule
+  public static TemporaryFolder temp = new TemporaryFolder();
+
+  private final DefaultInputModule module = mock();
+  private final ModuleExclusionFilters moduleExclusionFilters = mock();
+  private final InputModuleHierarchy inputModuleHierarchy = mock();
+  private final InputFile.Type type = mock();
+
+  @Test
+  public void visit_hidden_file() throws IOException {
+    DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+    File hidden = temp.newFile(".hidden");
+    if (SystemUtils.IS_OS_WINDOWS) {
+      Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS);
+    }
+
+
+    DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+    underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class));
+
+    verify(action, never()).execute(any(Path.class));
+  }
+
+  @Test
+  public void test_visit_file_failed_generic_io_exception() throws IOException {
+    DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+    File file = temp.newFile("failed");
+
+    DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+    assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException()));
+  }
+
+  @Test
+  public void test_visit_file_failed_file_system_loop_exception() throws IOException {
+    DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+    File file = temp.newFile("symlink");
+
+    DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+    FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath()));
+
+    assertThat(result).isEqualTo(FileVisitResult.CONTINUE);
+  }
+
+}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
new file mode 100644
index 00000000000..7be01e0a79f
--- /dev/null
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileFilterRepositoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class InputFileFilterRepositoryTest {
+
+  @Test
+  public void should_not_return_null_if_initialized_with_no_filters() {
+    InputFileFilterRepository underTest = new InputFileFilterRepository();
+    assertThat(underTest.getInputFileFilters()).isNotNull();
+  }
+
+  @Test
+  public void should_return_filters_from_initialization() {
+    InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true);
+    assertThat(underTest.getInputFileFilters()).isNotNull();
+    assertThat(underTest.getInputFileFilters()).hasSize(1);
+  }
+
+}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
index 34637645cbc..1440d8f427b 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/LanguageDetectionTest.java
@@ -24,6 +24,8 @@ import com.tngtech.java.junit.dataprovider.DataProviderRunner;
 import com.tngtech.java.junit.dataprovider.UseDataProvider;
 import java.io.File;
 import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -38,7 +40,11 @@ import org.sonar.scanner.repository.language.LanguagesRepository;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.endsWith;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 @RunWith(DataProviderRunner.class)
 public class LanguageDetectionTest {
@@ -72,7 +78,7 @@ public class LanguageDetectionTest {
   @Test
   public void detectLanguageKey_shouldDetectByFileExtension() {
     LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
 
     assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
     assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java");
@@ -94,7 +100,7 @@ public class LanguageDetectionTest {
       new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
       new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
       new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
     assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey);
   }
 
@@ -117,7 +123,7 @@ public class LanguageDetectionTest {
 
   @Test
   public void detectLanguageKey_shouldNotFailIfNoLanguage() {
-    LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages())));
+    LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>()));
     assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
   }
 
@@ -125,14 +131,14 @@ public class LanguageDetectionTest {
   public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
     LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
 
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
     assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
   }
 
   @Test
   public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
     LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
     assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
       .isInstanceOf(MessageException.class)
       .hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
@@ -146,7 +152,7 @@ public class LanguageDetectionTest {
 
     settings.setProperty("sonar.lang.patterns.xml", "xml/**");
     settings.setProperty("sonar.lang.patterns.web", "web/**");
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
     assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml");
     assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web");
   }
@@ -157,7 +163,7 @@ public class LanguageDetectionTest {
     settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
     settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
 
-    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
 
     assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
     assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol");
@@ -168,6 +174,19 @@ public class LanguageDetectionTest {
       .hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt");
   }
 
+  @Test
+  public void should_cache_detected_language_by_file_path() {
+    Map<String, org.sonar.scanner.repository.language.Language> languageCacheSpy = spy(new HashMap<>());
+    LanguagesRepository languages = new FakeLanguagesRepository(new Languages(
+      new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
+    LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy);
+
+    assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
+    assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
+    verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class));
+    verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java"));
+  }
+
   private String detectLanguageKey(LanguageDetection detection, String path) {
     org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path));
     return language != null ? language.key() : null;