SONAR-11077 add size of uncompressed report in to CE DEBUG logs

diff --git a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java
index fc77e9b74d90fbd16ae8f281292b60aca93b7b7d..cade181ec265c88a7629236e8b8fc1b14119e9db 100644 (file)
--- a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java
+++ b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java
@@ -116,6 +116,22 @@ public final class FileUtils2 {
     }
   }
 
+  /**
+   * Size of file or directory, in bytes. In case of a directory,
+   * the size is the sum of the sizes of all files recursively traversed.
+   *
+   * This implementation is recommended over commons-io
+   * {@code FileUtils#sizeOf(File)} which suffers from slow usage of Java IO.
+   *
+   * @throws IOException if files can't be traversed or size attribute is not present
+   * @see BasicFileAttributes#size()
+   */
+  public static long sizeOf(Path path) throws IOException {
+    SizeVisitor visitor = new SizeVisitor();
+    Files.walkFileTree(path, visitor);
+    return visitor.size;
+  }
+
   private static void cleanDirectoryImpl(Path path) throws IOException {
     if (!path.toFile().isDirectory()) {
       throw new IllegalArgumentException(format("'%s' is not a directory", path));
@@ -177,4 +193,18 @@ public final class FileUtils2 {
       return FileVisitResult.CONTINUE;
     }
   }
+
+  private static final class SizeVisitor extends SimpleFileVisitor<Path> {
+    private long size = 0;
+
+    @Override
+    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+      // size is specified on regular files only
+      // https://docs.oracle.com/javase/8/docs/api/java/nio/file/attribute/BasicFileAttributes.html#size--
+      if (attrs.isRegularFile()) {
+        size += attrs.size();
+      }
+      return FileVisitResult.CONTINUE;
+    }
+  }
 }

diff --git a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java
index b8f7268c346738864caf6c46fdbf96e971ff080a..ffd59a4f9e550e471fdf4fd4897f21c84728a0cc 100644 (file)
--- a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java
+++ b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java
@@ -25,12 +25,14 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import javax.annotation.CheckForNull;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.SystemUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assume.assumeTrue;
 
@@ -237,6 +239,63 @@ public class FileUtils2Test {
     assertThat(childDir2).doesNotExist();
   }
 
+  @Test
+  public void sizeOf_sums_sizes_of_all_files_in_directory() throws IOException {
+    File dir = temporaryFolder.newFolder();
+    File child = new File(dir, "child.txt");
+    File grandChild1 = new File(dir, "grand/child1.txt");
+    File grandChild2 = new File(dir, "grand/child2.txt");
+    FileUtils.write(child, "foo", UTF_8);
+    FileUtils.write(grandChild1, "bar", UTF_8);
+    FileUtils.write(grandChild2, "baz", UTF_8);
+
+    long childSize = FileUtils2.sizeOf(child.toPath());
+    assertThat(childSize).isPositive();
+    long grandChild1Size = FileUtils2.sizeOf(grandChild1.toPath());
+    assertThat(grandChild1Size).isPositive();
+    long grandChild2Size = FileUtils2.sizeOf(grandChild2.toPath());
+    assertThat(grandChild2Size).isPositive();
+
+    assertThat(FileUtils2.sizeOf(dir.toPath()))
+      .isEqualTo(childSize + grandChild1Size + grandChild2Size);
+
+    // sanity check by comparing commons-io
+    assertThat(FileUtils2.sizeOf(dir.toPath()))
+      .isEqualTo(FileUtils.sizeOfDirectory(dir));
+  }
+
+  @Test
+  public void sizeOf_is_zero_on_empty_files() throws IOException {
+    File file = temporaryFolder.newFile();
+
+    assertThat(FileUtils2.sizeOf(file.toPath())).isEqualTo(0);
+  }
+
+  @Test
+  public void sizeOf_throws_IOE_if_path_does_not_exist() throws IOException {
+    Path path = temporaryFolder.newFile().toPath();
+    Files.delete(path);
+
+    expectedException.expect(IOException.class);
+
+    FileUtils2.sizeOf(path);
+  }
+
+  @Test
+  public void sizeOf_ignores_size_of_non_regular_files() throws IOException {
+    File outside = temporaryFolder.newFile();
+    FileUtils.write(outside, "outside!!!", UTF_8);
+    File dir = temporaryFolder.newFolder();
+    File child = new File(dir, "child1.txt");
+    FileUtils.write(child, "inside!!!", UTF_8);
+    File symlink = new File(dir, "child2.txt");
+    Files.createSymbolicLink(symlink.toPath(), outside.toPath());
+
+    assertThat(FileUtils2.sizeOf(dir.toPath()))
+      .isPositive()
+      .isEqualTo(FileUtils2.sizeOf(child.toPath()));
+  }
+
   private void expectDirectoryCanNotBeNullNPE() {
     expectedException.expect(NullPointerException.class);
     expectedException.expectMessage("Directory can not be null");

diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java
index da704c578200788c380ecbe7f4de1e97f4ba6129..73e14791f6eee6c49caa818113a8bc32d16dd084 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java
@@ -24,13 +24,17 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Optional;
+import org.apache.commons.io.FileUtils;
 import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.TempFolder;
 import org.sonar.api.utils.ZipUtils;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.ce.queue.CeTask;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.ce.CeTaskInputDao;
+import org.sonar.process.FileUtils2;
 import org.sonar.server.computation.task.projectanalysis.batch.MutableBatchReportDirectoryHolder;
 import org.sonar.server.computation.task.step.ComputationStep;
 
@@ -40,6 +44,8 @@ import org.sonar.server.computation.task.step.ComputationStep;
  */
 public class ExtractReportStep implements ComputationStep {
 
+  private static final Logger LOGGER = Loggers.get(ExtractReportStep.class);
+
   private final DbClient dbClient;
   private final CeTask task;
   private final TempFolder tempFolder;
@@ -66,6 +72,16 @@ public class ExtractReportStep implements ComputationStep {
           throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e);
         }
         reportDirectoryHolder.setDirectory(unzippedDir);
+        if (LOGGER.isDebugEnabled()) {
+          // size is not added to context statistics because computation
+          // can take time. It's enabled only if log level is DEBUG.
+          try {
+            String dirSize = FileUtils.byteCountToDisplaySize(FileUtils2.sizeOf(unzippedDir.toPath()));
+            LOGGER.debug("Analysis report is {} uncompressed", dirSize);
+          } catch (IOException e) {
+            LOGGER.warn("Fail to compute size of directory " + unzippedDir, e);
+          }
+        }
       } else {
         throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database");
       }