From b6d687934b6987d528449fe49e27dcf01cd66e04 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Mon, 30 Jul 2018 17:41:16 +0200 Subject: [PATCH] SONAR-11077 add size of uncompressed report in to CE DEBUG logs --- .../step/ExtractReportStep.java | 17 +++++++ .../step/ExtractReportStepTest.java | 5 ++- .../java/org/sonar/process/FileUtils2.java | 26 +++++++++++ .../org/sonar/process/FileUtils2Test.java | 44 +++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java index 27ac1d80195..119d16e5a77 100644 --- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java +++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java @@ -24,15 +24,19 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Optional; +import org.apache.commons.io.FileUtils; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.TempFolder; import org.sonar.api.utils.ZipUtils; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; import org.sonar.ce.task.CeTask; import org.sonar.ce.task.projectanalysis.batch.MutableBatchReportDirectoryHolder; import org.sonar.ce.task.step.ComputationStep; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.ce.CeTaskInputDao; +import org.sonar.process.FileUtils2; /** * Extracts the content zip file of the {@link CeTask} to a temp directory and adds a {@link File} @@ -40,6 +44,8 @@ import org.sonar.db.ce.CeTaskInputDao; */ public class ExtractReportStep implements ComputationStep { + private static final Logger LOGGER = Loggers.get(ExtractReportStep.class); + private final DbClient dbClient; private final CeTask task; private final TempFolder tempFolder; @@ -66,6 +72,17 @@ public class ExtractReportStep implements ComputationStep { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); } reportDirectoryHolder.setDirectory(unzippedDir); + if (LOGGER.isDebugEnabled()) { + // size is not added to context statistics because computation + // can take time. It's enabled only if log level is DEBUG. + try { + String dirSize = FileUtils.byteCountToDisplaySize(FileUtils2.sizeOf(unzippedDir.toPath())); + LOGGER.debug("Analysis report is {} uncompressed", dirSize); + } catch (IOException e) { + LOGGER.warn("Fail to compute size of directory " + unzippedDir, e); + } + } + } else { throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); } diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java index 715753e1753..efa4d91273f 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStepTest.java @@ -49,7 +49,7 @@ public class ExtractReportStepTest { public JUnitTempFolder tempFolder = new JUnitTempFolder(); @Rule - public LogTester logTester = new LogTester().setLevel(LoggerLevel.INFO); + public LogTester logTester = new LogTester(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -76,6 +76,7 @@ public class ExtractReportStepTest { @Test public void unzip_report() throws Exception { + logTester.setLevel(LoggerLevel.DEBUG); File reportFile = generateReport(); try (InputStream input = FileUtils.openInputStream(reportFile)) { dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), TASK_UUID, input); @@ -90,6 +91,8 @@ public class ExtractReportStepTest { assertThat(unzippedDir).isDirectory().exists(); assertThat(unzippedDir.listFiles()).hasSize(1); assertThat(new File(unzippedDir, "metadata.pb")).hasContent("{metadata}"); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).anyMatch(log -> log.matches("Analysis report is \\d+ bytes uncompressed")); } private File generateReport() throws IOException { diff --git a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java index 8690e7776e2..d275c960fa9 100644 --- a/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java +++ b/server/sonar-process/src/main/java/org/sonar/process/FileUtils2.java @@ -116,6 +116,22 @@ public final class FileUtils2 { } } + /** + * Size of file or directory, in bytes. In case of a directory, + * the size is the sum of the sizes of all files recursively traversed. + * + * This implementation is recommended over commons-io + * {@code FileUtils#sizeOf(File)} which suffers from slow usage of Java IO. + * + * @throws IOException if files can't be traversed or size attribute is not present + * @see BasicFileAttributes#size() + */ + public static long sizeOf(Path path) throws IOException { + SizeVisitor visitor = new SizeVisitor(); + Files.walkFileTree(path, visitor); + return visitor.size; + } + private static void cleanDirectoryImpl(Path path) throws IOException { if (!path.toFile().isDirectory()) { throw new IllegalArgumentException(format("'%s' is not a directory", path)); @@ -177,4 +193,14 @@ public final class FileUtils2 { return FileVisitResult.CONTINUE; } } + + private static final class SizeVisitor extends SimpleFileVisitor { + private long size = 0; + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + size += attrs.size(); + return FileVisitResult.CONTINUE; + } + } } diff --git a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java index e75a1fed445..890a21d782a 100644 --- a/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java +++ b/server/sonar-process/src/test/java/org/sonar/process/FileUtils2Test.java @@ -25,12 +25,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import javax.annotation.CheckForNull; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.SystemUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assume.assumeTrue; @@ -237,6 +239,48 @@ public class FileUtils2Test { assertThat(childDir2).doesNotExist(); } + @Test + public void sizeOf_sums_sizes_of_all_files_in_directory() throws IOException { + File dir = temporaryFolder.newFolder(); + File child = new File(dir, "child.txt"); + File grandChild1 = new File(dir, "grand/child1.txt"); + File grandChild2 = new File(dir, "grand/child2.txt"); + FileUtils.write(child, "foo", UTF_8); + FileUtils.write(grandChild1, "bar", UTF_8); + FileUtils.write(grandChild2, "baz", UTF_8); + + long childSize = FileUtils2.sizeOf(child.toPath()); + assertThat(childSize).isPositive(); + long grandChild1Size = FileUtils2.sizeOf(grandChild1.toPath()); + assertThat(grandChild1Size).isPositive(); + long grandChild2Size = FileUtils2.sizeOf(grandChild2.toPath()); + assertThat(grandChild2Size).isPositive(); + + assertThat(FileUtils2.sizeOf(dir.toPath())) + .isEqualTo(childSize + grandChild1Size + grandChild2Size); + + // sanity check by comparing commons-io + assertThat(FileUtils2.sizeOf(dir.toPath())) + .isEqualTo(FileUtils.sizeOfDirectory(dir)); + } + + @Test + public void sizeOf_is_zero_on_empty_files() throws IOException { + File file = temporaryFolder.newFile(); + + assertThat(FileUtils2.sizeOf(file.toPath())).isEqualTo(0); + } + + @Test + public void sizeOf_throws_IOE_if_path_does_not_exist() throws IOException { + Path path = temporaryFolder.newFile().toPath(); + Files.delete(path); + + expectedException.expect(IOException.class); + + FileUtils2.sizeOf(path); + } + private void expectDirectoryCanNotBeNullNPE() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("Directory can not be null"); -- 2.39.5