SONAR-11096 Discard files from analysis scope greater 20MB

SONAR-11096 Discard files from analysis scope greater when than 20 MB

- introduce property 'sonar.filesize.limit'

7 files changed, 64 insertions, 8 deletions

diff --git a/server/sonar-docs/src/pages/analysis/analysis-parameters.md b/server/sonar-docs/src/pages/analysis/analysis-parameters.md
index 4292ecde1eb..58a63573012 100644
--- a/server/sonar-docs/src/pages/analysis/analysis-parameters.md
+++ b/server/sonar-docs/src/pages/analysis/analysis-parameters.md
@@ -83,6 +83,7 @@ Key | Description | Default
 `sonar.buildString`| The string passed with this property will be stored with the analysis and available in the results of `api/project_analyses/search`, thus allowing you to later identify a specific analysis and obtain its ID for use with `api/project_analyses/set_baseline`. | |
 `sonar.analysis.[yourKey]`| This property stub allows you to insert custom key/value pairs into the analysis context, which will also be passed forward to [webhooks](/project-administration/webhooks/). | |
 `sonar.newCode.referenceBranch`| Sets the [New Code](/project-administration/new-code-period/) definition to `Reference Branch` for this analysis, overriding the configuration on the server. The `New Code` will be calculated based on the differences between the branch under analysis and the provided branch. This parameter is intended to be set in a configuration file (ex: `sonar-project.properties`), specific to a given branch.
+`sonar.filesize.limit`| Sets the limit in MB for files to be discarded from the analysis scope if the size is greater than specified. | 20
 
 ### Duplications
 Key | Description | Default
diff --git a/server/sonar-docs/src/pages/setup/upgrade-notes.md b/server/sonar-docs/src/pages/setup/upgrade-notes.md
index f568017cb11..79089559ad6 100644
--- a/server/sonar-docs/src/pages/setup/upgrade-notes.md
+++ b/server/sonar-docs/src/pages/setup/upgrade-notes.md
@@ -4,6 +4,9 @@ url: /setup/upgrade-notes/
 ---
 
 ## Release 9.8 Upgrade notes
+**Analysis discards from its scope all files greater than 20 MB**
+SonarQube analysis discards from its scope all files greater than 20 MB. This setting can be changed at scanner level using following property `sonar.filesize.limit`. ([SONAR-11096](https://sonarsource.atlassian.net/browse/SONAR-11096)) 
+
 **Dropping the embedded documentation**
 SonarQube no longer ships with an embedded version of the documentation. The documentation will now only be available at [docs.sonarqube.org](https://docs.sonarqube.org/). ([SONAR-17221](https://sonarsource.atlassian.net/browse/SONAR-17221))  
 * All in-app links will now point to the documentation website.
diff --git a/sonar-core/src/main/java/org/sonar/core/config/ScannerProperties.java b/sonar-core/src/main/java/org/sonar/core/config/ScannerProperties.java
index fa38ca14887..6b9ede6192f 100644
--- a/sonar-core/src/main/java/org/sonar/core/config/ScannerProperties.java
+++ b/sonar-core/src/main/java/org/sonar/core/config/ScannerProperties.java
@@ -26,6 +26,7 @@ import org.sonar.api.resources.Qualifiers;
 
 import static java.util.Arrays.asList;
 import static org.sonar.api.PropertyType.BOOLEAN;
+import static org.sonar.api.PropertyType.INTEGER;
 
 public class ScannerProperties {
 
@@ -38,7 +39,7 @@ public class ScannerProperties {
   public static final String PULL_REQUEST_KEY = "sonar.pullrequest.key";
   public static final String PULL_REQUEST_BRANCH = "sonar.pullrequest.branch";
   public static final String PULL_REQUEST_BASE = "sonar.pullrequest.base";
-
+  public static final String FILE_SIZE_LIMIT = "sonar.filesize.limit";
   public static final String LINKS_SOURCES_DEV = "sonar.links.scm_dev";
   public static final String DISABLE_PROJECT_AND_ORG_AUTODETECTION = "sonar.keys_autodetection.disabled";
 
@@ -84,6 +85,14 @@ public class ScannerProperties {
         .description("Disables auto-detection of project keys from scanner execution environment.")
         .type(BOOLEAN)
         .hidden()
+        .build(),
+      PropertyDefinition.builder(FILE_SIZE_LIMIT)
+        .name("Limit of a file size excluded from analysis in MB")
+        .type(INTEGER)
+        .defaultValue("20")
+        .description(
+          "Allows discarding files from analysis exceeding certain sizes.")
+        .hidden()
         .build());
   }
 }
diff --git a/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java b/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java
index 00d1068c7a3..fada8ca9c48 100644
--- a/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java
+++ b/sonar-core/src/test/java/org/sonar/core/config/CorePropertyDefinitionsTest.java
@@ -30,7 +30,7 @@ public class CorePropertyDefinitionsTest {
   @Test
   public void all() {
     List<PropertyDefinition> defs = CorePropertyDefinitions.all();
-    assertThat(defs).hasSize(52);
+    assertThat(defs).hasSize(53);
   }
 
   @Test
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ScanProperties.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ScanProperties.java
index 250d9d22482..095887f8159 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ScanProperties.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ScanProperties.java
@@ -27,6 +27,7 @@ import org.sonar.api.config.Configuration;
 import org.sonar.api.utils.MessageException;
 
 import static org.sonar.core.config.ScannerProperties.BRANCH_NAME;
+import static org.sonar.core.config.ScannerProperties.FILE_SIZE_LIMIT;
 
 /**
  * Properties that can be passed to the scanners and are not exposed in SonarQube.
@@ -93,6 +94,10 @@ public class ScanProperties {
     return configuration.getInt(REPORT_PUBLISH_TIMEOUT_IN_SEC).orElse(60);
   }
 
+  public long fileSizeLimit() {
+    return configuration.getInt(FILE_SIZE_LIMIT).orElse(20);
+  }
+
   /**
    * This should be called in the beginning of the analysis to fail fast
    */
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
index f62617d166a..caf0ad0d76f 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java
@@ -20,6 +20,7 @@
 package org.sonar.scanner.scan.filesystem;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -119,6 +120,12 @@ public class FileIndexer {
       return;
     }
 
+    long maxFileSize = properties.fileSizeLimit();
+    if (Files.size(realAbsoluteFile) > maxFileSize * 1024L * 1024L) {
+      LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), maxFileSize);
+      return;
+    }
+
     Language language = langDetection.language(realAbsoluteFile, projectRelativePath);
 
     if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
@@ -141,7 +148,7 @@ public class FileIndexer {
 
     DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()));
     if (language != null && language.isPublishAllFiles()) {
-        inputFile.setPublished(true);
+      inputFile.setPublished(true);
     }
     if (!accept(inputFile)) {
       return;
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java
index f6daaed28b1..687d65a1017 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java
@@ -22,6 +22,7 @@ package org.sonar.scanner.mediumtest.fs;
 import com.google.common.collect.ImmutableMap;
 import java.io.File;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -50,6 +51,7 @@ import org.sonar.xoo.global.DeprecatedGlobalSensor;
 import org.sonar.xoo.global.GlobalProjectSensor;
 import org.sonar.xoo.rule.XooRulesDefinition;
 
+import static java.lang.String.format;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.assertj.core.api.Assertions.tuple;
@@ -121,7 +123,7 @@ public class FileSystemMediumTest {
         .put("sonar.sources", "src")
         .build())
       .execute())
-      .isInstanceOf(MessageException.class);
+        .isInstanceOf(MessageException.class);
   }
 
   @Test
@@ -417,6 +419,26 @@ public class FileSystemMediumTest {
   }
 
   @Test
+  public void ignoreFilesWhenGreaterThanDefinedSize() throws IOException {
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+
+    File fileGreaterThanLimit = writeFile(srcDir, "sample.xoo", 1024 * 1024 + 1);
+    writeFile(srcDir, "another.xoo", "Sample xoo 2\ncontent");
+
+    AnalysisResult result = tester.newAnalysis()
+      .properties(builder
+        // set limit to 1MB
+        .put("sonar.filesize.limit", "1")
+        .build())
+      .execute();
+
+    assertThat(result.inputFiles()).hasSize(1);
+    assertThat(logTester.logs())
+      .contains(format("File '%s' is bigger than 1MB and as consequence is removed from the analysis scope.", fileGreaterThanLimit.getAbsolutePath()));
+  }
+
+  @Test
   public void test_inclusions_on_multi_modules() throws IOException {
     File baseDir = temp.getRoot();
     File baseDirModuleA = new File(baseDir, "moduleA");
@@ -644,8 +666,8 @@ public class FileSystemMediumTest {
         .put("sonar.sources", "src,src/sample.xoo")
         .build())
       .execute())
-      .isInstanceOf(MessageException.class)
-      .hasMessage("File src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files");
+        .isInstanceOf(MessageException.class)
+        .hasMessage("File src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files");
   }
 
   // SONAR-9574
@@ -663,8 +685,8 @@ public class FileSystemMediumTest {
         .put("module1.sonar.sources", "src")
         .build())
       .execute())
-      .isInstanceOf(MessageException.class)
-      .hasMessage("File module1/src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files");
+        .isInstanceOf(MessageException.class)
+        .hasMessage("File module1/src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files");
   }
 
   // SONAR-5330
@@ -1048,4 +1070,13 @@ public class FileSystemMediumTest {
     FileUtils.write(file, content, StandardCharsets.UTF_8);
     return file;
   }
+
+  private File writeFile(File parent, String name, long size) throws IOException {
+    File file = new File(parent, name);
+    RandomAccessFile raf = new RandomAccessFile(file, "rw");
+    raf.setLength(size);
+    raf.close();
+    return file;
+  }
+
 }