From fe003e5e825a357e0afda331b2374393404973d6 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Fri, 3 Aug 2018 11:41:12 +0200 Subject: [PATCH] SONAR-11077 add size of uncompressed report in to CE DEBUG logs --- .../java/org/sonar/process/FileUtils2.java | 30 ++++++++++ .../org/sonar/process/FileUtils2Test.java | 59 +++++++++++++++++++ .../step/ExtractReportStep.java | 16 +++++ 3 files changed, 105 insertions(+) diff --git a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java index fc77e9b74d9..cade181ec26 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java +++ b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java @@ -116,6 +116,22 @@ public final class FileUtils2 { } } + /** + * Size of file or directory, in bytes. In case of a directory, + * the size is the sum of the sizes of all files recursively traversed. + * + * This implementation is recommended over commons-io + * {@code FileUtils#sizeOf(File)} which suffers from slow usage of Java IO. + * + * @throws IOException if files can't be traversed or size attribute is not present + * @see BasicFileAttributes#size() + */ + public static long sizeOf(Path path) throws IOException { + SizeVisitor visitor = new SizeVisitor(); + Files.walkFileTree(path, visitor); + return visitor.size; + } + private static void cleanDirectoryImpl(Path path) throws IOException { if (!path.toFile().isDirectory()) { throw new IllegalArgumentException(format("'%s' is not a directory", path)); @@ -177,4 +193,18 @@ public final class FileUtils2 { return FileVisitResult.CONTINUE; } } + + private static final class SizeVisitor extends SimpleFileVisitor { + private long size = 0; + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + // size is specified on regular files only + // https://docs.oracle.com/javase/8/docs/api/java/nio/file/attribute/BasicFileAttributes.html#size-- + if (attrs.isRegularFile()) { + size += attrs.size(); + } + return FileVisitResult.CONTINUE; + } + } } diff --git a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java index b8f7268c346..ffd59a4f9e5 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java +++ b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java @@ -25,12 +25,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import javax.annotation.CheckForNull; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.SystemUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assume.assumeTrue; @@ -237,6 +239,63 @@ public class FileUtils2Test { assertThat(childDir2).doesNotExist(); } + @Test + public void sizeOf_sums_sizes_of_all_files_in_directory() throws IOException { + File dir = temporaryFolder.newFolder(); + File child = new File(dir, "child.txt"); + File grandChild1 = new File(dir, "grand/child1.txt"); + File grandChild2 = new File(dir, "grand/child2.txt"); + FileUtils.write(child, "foo", UTF_8); + FileUtils.write(grandChild1, "bar", UTF_8); + FileUtils.write(grandChild2, "baz", UTF_8); + + long childSize = FileUtils2.sizeOf(child.toPath()); + assertThat(childSize).isPositive(); + long grandChild1Size = FileUtils2.sizeOf(grandChild1.toPath()); + assertThat(grandChild1Size).isPositive(); + long grandChild2Size = FileUtils2.sizeOf(grandChild2.toPath()); + assertThat(grandChild2Size).isPositive(); + + assertThat(FileUtils2.sizeOf(dir.toPath())) + .isEqualTo(childSize + grandChild1Size + grandChild2Size); + + // sanity check by comparing commons-io + assertThat(FileUtils2.sizeOf(dir.toPath())) + .isEqualTo(FileUtils.sizeOfDirectory(dir)); + } + + @Test + public void sizeOf_is_zero_on_empty_files() throws IOException { + File file = temporaryFolder.newFile(); + + assertThat(FileUtils2.sizeOf(file.toPath())).isEqualTo(0); + } + + @Test + public void sizeOf_throws_IOE_if_path_does_not_exist() throws IOException { + Path path = temporaryFolder.newFile().toPath(); + Files.delete(path); + + expectedException.expect(IOException.class); + + FileUtils2.sizeOf(path); + } + + @Test + public void sizeOf_ignores_size_of_non_regular_files() throws IOException { + File outside = temporaryFolder.newFile(); + FileUtils.write(outside, "outside!!!", UTF_8); + File dir = temporaryFolder.newFolder(); + File child = new File(dir, "child1.txt"); + FileUtils.write(child, "inside!!!", UTF_8); + File symlink = new File(dir, "child2.txt"); + Files.createSymbolicLink(symlink.toPath(), outside.toPath()); + + assertThat(FileUtils2.sizeOf(dir.toPath())) + .isPositive() + .isEqualTo(FileUtils2.sizeOf(child.toPath())); + } + private void expectDirectoryCanNotBeNullNPE() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("Directory can not be null"); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java index da704c57820..73e14791f6e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/step/ExtractReportStep.java @@ -24,13 +24,17 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Optional; +import org.apache.commons.io.FileUtils; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.TempFolder; import org.sonar.api.utils.ZipUtils; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.ce.queue.CeTask; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.ce.CeTaskInputDao; +import org.sonar.process.FileUtils2; import org.sonar.server.computation.task.projectanalysis.batch.MutableBatchReportDirectoryHolder; import org.sonar.server.computation.task.step.ComputationStep; @@ -40,6 +44,8 @@ import org.sonar.server.computation.task.step.ComputationStep; */ public class ExtractReportStep implements ComputationStep { + private static final Logger LOGGER = Loggers.get(ExtractReportStep.class); + private final DbClient dbClient; private final CeTask task; private final TempFolder tempFolder; @@ -66,6 +72,16 @@ public class ExtractReportStep implements ComputationStep { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); } reportDirectoryHolder.setDirectory(unzippedDir); + if (LOGGER.isDebugEnabled()) { + // size is not added to context statistics because computation + // can take time. It's enabled only if log level is DEBUG. + try { + String dirSize = FileUtils.byteCountToDisplaySize(FileUtils2.sizeOf(unzippedDir.toPath())); + LOGGER.debug("Analysis report is {} uncompressed", dirSize); + } catch (IOException e) { + LOGGER.warn("Fail to compute size of directory " + unzippedDir, e); + } + } } else { throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); } -- 2.39.5