SONAR-14917 Misleading memory amounts displayed in Administration > System

diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java
index bd0302be14a9e024127919958509f96a927cc27d..a3c1668554e83fd80cf63810b98982ada163060f 100644 (file)
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ExtractReportStep.java
@@ -24,7 +24,6 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Optional;
-import org.apache.commons.io.FileUtils;
 import org.sonar.api.utils.MessageException;
 import org.sonar.api.utils.TempFolder;
 import org.sonar.api.utils.ZipUtils;
@@ -38,6 +37,8 @@ import org.sonar.db.DbSession;
 import org.sonar.db.ce.CeTaskInputDao;
 import org.sonar.process.FileUtils2;
 
+import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
+
 /**
  * Extracts the content zip file of the {@link CeTask} to a temp directory and adds a {@link File}
  * representing that temp directory to the {@link MutableBatchReportDirectoryHolder}.
@@ -76,7 +77,7 @@ public class ExtractReportStep implements ComputationStep {
           // size is not added to context statistics because computation
           // can take time. It's enabled only if log level is DEBUG.
           try {
-            String dirSize = FileUtils.byteCountToDisplaySize(FileUtils2.sizeOf(unzippedDir.toPath()));
+            String dirSize = humanReadableByteCountSI(FileUtils2.sizeOf(unzippedDir.toPath()));
             LOGGER.debug("Analysis report is {} uncompressed", dirSize);
           } catch (IOException e) {
             LOGGER.warn("Fail to compute size of directory " + unzippedDir, e);

diff --git a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java
index bf355f2d4f6ee84f82938f14e06c7862335a1c54..ba32960616a979ecd083717bd6db731108fdc7f0 100644 (file)
--- a/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java
+++ b/server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/PersistIssuesStep.java
@@ -22,7 +22,6 @@ package org.sonar.ce.task.projectanalysis.step;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.io.FileUtils;
 import org.sonar.api.utils.System2;
 import org.sonar.ce.task.projectanalysis.issue.ProtoIssueCache;
 import org.sonar.ce.task.projectanalysis.issue.RuleRepository;
@@ -39,6 +38,7 @@ import org.sonar.db.issue.IssueDto;
 import org.sonar.db.issue.IssueMapper;
 import org.sonar.server.issue.IssueStorage;
 
+import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
 import static org.sonar.core.util.stream.MoreCollectors.toList;
 import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
 
@@ -68,7 +68,7 @@ public class PersistIssuesStep implements ComputationStep {
 
   @Override
   public void execute(ComputationStep.Context context) {
-    context.getStatistics().add("cacheSize", FileUtils.byteCountToDisplaySize(protoIssueCache.fileSize()));
+    context.getStatistics().add("cacheSize", humanReadableByteCountSI(protoIssueCache.fileSize()));
     IssueStatistics statistics = new IssueStatistics();
     try (DbSession dbSession = dbClient.openSession(true);
 

diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java
index ffc38bfc979c4e64616b9ade34351c7efbfe2715..12d50d9c84c603d8c7da820575918accc626172c 100644 (file)
--- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java
+++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java
@@ -29,7 +29,7 @@ import org.sonar.server.es.EsClient;
 import org.sonar.server.es.response.IndexStats;
 import org.sonar.server.es.response.IndicesStatsResponse;
 
-import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
 import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute;
 
 @ServerSide
@@ -60,7 +60,7 @@ public class EsIndexesSection implements SystemInfoSection, Global {
       String prefix = "Index " + indexStats.getName() + " - ";
       setAttribute(protobuf, prefix + "Docs", indexStats.getDocCount());
       setAttribute(protobuf, prefix + "Shards", indexStats.getShardsCount());
-      setAttribute(protobuf, prefix + "Store Size", byteCountToDisplaySize(indexStats.getStoreSizeBytes()));
+      setAttribute(protobuf, prefix + "Store Size", humanReadableByteCountSI(indexStats.getStoreSizeBytes()));
     }
   }
 }

diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java
index 1329bc940ede7858ba3b89b1f83ace1ea9506351..d4f62864601e6f1b3c16128989af51a807d2da42 100644 (file)
--- a/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java
+++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java
@@ -31,7 +31,7 @@ import org.sonar.server.es.response.NodeStats;
 import org.sonar.server.es.response.NodeStatsResponse;
 
 import static java.lang.String.format;
-import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
 import static org.sonar.process.systeminfo.SystemInfoUtils.setAttribute;
 
 public class EsStateSection implements SystemInfoSection {
@@ -70,28 +70,28 @@ public class EsStateSection implements SystemInfoSection {
 
   public static void toProtobuf(NodeStats stats, ProtobufSystemInfo.Section.Builder protobuf) {
     setAttribute(protobuf, "CPU Usage (%)", stats.getCpuUsage());
-    setAttribute(protobuf, "Disk Available", byteCountToDisplaySize(stats.getDiskAvailableBytes()));
-    setAttribute(protobuf, "Store Size", byteCountToDisplaySize(stats.getIndicesStats().getStoreSizeInBytes()));
-    setAttribute(protobuf, "Translog Size", byteCountToDisplaySize(stats.getIndicesStats().getTranslogSizeInBytes()));
+    setAttribute(protobuf, "Disk Available", humanReadableByteCountSI(stats.getDiskAvailableBytes()));
+    setAttribute(protobuf, "Store Size", humanReadableByteCountSI(stats.getIndicesStats().getStoreSizeInBytes()));
+    setAttribute(protobuf, "Translog Size", humanReadableByteCountSI(stats.getIndicesStats().getTranslogSizeInBytes()));
     setAttribute(protobuf, "Open File Descriptors", stats.getOpenFileDescriptors());
     setAttribute(protobuf, "Max File Descriptors", stats.getMaxFileDescriptors());
     setAttribute(protobuf, "JVM Heap Usage", formatPercent(stats.getJvmStats().getHeapUsedPercent()));
-    setAttribute(protobuf, "JVM Heap Used", byteCountToDisplaySize(stats.getJvmStats().getHeapUsedInBytes()));
-    setAttribute(protobuf, "JVM Heap Max", byteCountToDisplaySize(stats.getJvmStats().getHeapMaxInBytes()));
-    setAttribute(protobuf, "JVM Non Heap Used", byteCountToDisplaySize(stats.getJvmStats().getNonHeapUsedInBytes()));
+    setAttribute(protobuf, "JVM Heap Used", humanReadableByteCountSI(stats.getJvmStats().getHeapUsedInBytes()));
+    setAttribute(protobuf, "JVM Heap Max", humanReadableByteCountSI(stats.getJvmStats().getHeapMaxInBytes()));
+    setAttribute(protobuf, "JVM Non Heap Used", humanReadableByteCountSI(stats.getJvmStats().getNonHeapUsedInBytes()));
     setAttribute(protobuf, "JVM Threads", stats.getJvmStats().getThreadCount());
-    setAttribute(protobuf, "Field Data Memory", byteCountToDisplaySize(stats.getIndicesStats().getFieldDataMemorySizeInBytes()));
+    setAttribute(protobuf, "Field Data Memory", humanReadableByteCountSI(stats.getIndicesStats().getFieldDataMemorySizeInBytes()));
     setAttribute(protobuf, "Field Data Circuit Breaker Limit",
-      byteCountToDisplaySize(stats.getFieldDataCircuitBreakerLimit()));
+      humanReadableByteCountSI(stats.getFieldDataCircuitBreakerLimit()));
     setAttribute(protobuf, "Field Data Circuit Breaker Estimation",
-      byteCountToDisplaySize(stats.getFieldDataCircuitBreakerEstimation()));
+      humanReadableByteCountSI(stats.getFieldDataCircuitBreakerEstimation()));
     setAttribute(protobuf, "Request Circuit Breaker Limit",
-      byteCountToDisplaySize(stats.getRequestCircuitBreakerLimit()));
+      humanReadableByteCountSI(stats.getRequestCircuitBreakerLimit()));
     setAttribute(protobuf, "Request Circuit Breaker Estimation",
-      byteCountToDisplaySize(stats.getRequestCircuitBreakerEstimation()));
-    setAttribute(protobuf, "Query Cache Memory", byteCountToDisplaySize(stats.getIndicesStats().getQueryCacheMemorySizeInBytes()));
+      humanReadableByteCountSI(stats.getRequestCircuitBreakerEstimation()));
+    setAttribute(protobuf, "Query Cache Memory", humanReadableByteCountSI(stats.getIndicesStats().getQueryCacheMemorySizeInBytes()));
     setAttribute(protobuf, "Request Cache Memory",
-      byteCountToDisplaySize(stats.getIndicesStats().getRequestCacheMemorySizeInBytes()));
+      humanReadableByteCountSI(stats.getIndicesStats().getRequestCacheMemorySizeInBytes()));
   }
 
   private static String formatPercent(long amount) {

diff --git a/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java b/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java
index 109e6769ce98350da4452258e95163762b8a6c43..ad058a6f45b61bac1903cd12354448e8bb6f1a78 100644 (file)
--- a/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java
+++ b/sonar-core/src/main/java/org/sonar/core/util/FileUtils.java
@@ -27,6 +27,8 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
 import java.util.EnumSet;
 import javax.annotation.Nullable;
 
@@ -161,6 +163,26 @@ public final class FileUtils {
     checkIO(!file.exists(), "Unable to delete directory '%s'", path);
   }
 
+  /**
+   * Replaces the use of Apache Commons FileUtils #byteCountToDisplaySize (see SONAR-14917)
+   * @param bytes number of bytes
+   * @return human readable byte count, with 1 decimal place
+   */
+  public static String humanReadableByteCountSI(long bytes) {
+    if (-1000 < bytes && bytes < 1000) {
+      if (bytes == 1) {
+        return bytes + " byte";
+      }
+      return bytes + " bytes";
+    }
+    CharacterIterator ci = new StringCharacterIterator("kMGTPE");
+    while (bytes <= -999_950 || bytes >= 999_950) {
+      bytes /= 1000;
+      ci.next();
+    }
+    return String.format("%.1f %cB", bytes / 1000.0, ci.current());
+  }
+
   /**
    * This visitor is intended to be used to visit direct children of directory <strong>or a symLink to a directory</strong>,
    * so, with a max depth of {@link #VISIT_MAX_DEPTH 1}. Each direct children will either be directly deleted (if file)

diff --git a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java
index 89eefb5b4a978fab2912bdc4001fd295e9813b6f..de333333ccb221aa2163a0053378a6555dea8868 100644 (file)
--- a/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java
+++ b/sonar-core/src/test/java/org/sonar/core/util/FileUtilsTest.java
@@ -159,6 +159,31 @@ public class FileUtilsTest {
     assertThat(childDir2).doesNotExist();
   }
 
+  @Test
+  public void humanReadableByteCountSI_returns_bytes() {
+    assertThat(FileUtils.humanReadableByteCountSI(123)).isEqualTo("123 bytes");
+    assertThat(FileUtils.humanReadableByteCountSI(0)).isEqualTo("0 bytes");
+    assertThat(FileUtils.humanReadableByteCountSI(1)).isEqualTo("1 byte");
+  }
+
+  @Test
+  public void humanReadableByteCountSI_returns_kbs() {
+    assertThat(FileUtils.humanReadableByteCountSI(1_234)).isEqualTo("1.2 kB");
+    assertThat(FileUtils.humanReadableByteCountSI(1_000)).isEqualTo("1.0 kB");
+    assertThat(FileUtils.humanReadableByteCountSI(9_999)).isEqualTo("10.0 kB");
+    assertThat(FileUtils.humanReadableByteCountSI(999_949)).isEqualTo("999.9 kB");
+  }
+
+  @Test
+  public void humanReadableByteCountSI_returns_tbs() {
+    assertThat(FileUtils.humanReadableByteCountSI(1_234_000_000_000L)).isEqualTo("1.2 TB");
+  }
+
+  @Test
+  public void humanReadableByteCountSI_returns_mbs() {
+    assertThat(FileUtils.humanReadableByteCountSI(1234567)).isEqualTo("1.2 MB");
+  }
+
   @Test
   public void deleteQuietly_deletes_symbolicLink() throws IOException {
     assumeTrue(SystemUtils.IS_OS_UNIX);

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java
index f15f4d03f08522e65d0dbd71fb4b098d9544792f..f5a6e50b33e58c794b19336b117d2658e37a1220 100644 (file)
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/ReportPublisher.java
@@ -58,6 +58,7 @@ import static java.net.URLEncoder.encode;
 import static org.apache.commons.lang.StringUtils.EMPTY;
 import static org.apache.commons.lang.StringUtils.isBlank;
 import static org.sonar.core.util.FileUtils.deleteQuietly;
+import static org.sonar.core.util.FileUtils.humanReadableByteCountSI;
 import static org.sonar.scanner.scan.branch.BranchType.PULL_REQUEST;
 
 public class ReportPublisher implements Startable {
@@ -162,13 +163,13 @@ public class ReportPublisher implements Startable {
         publisher.publish(writer);
       }
       long stopTime = System.currentTimeMillis();
-      LOG.info("Analysis report generated in {}ms, dir size={}", stopTime - startTime, FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(reportDir.toFile())));
+      LOG.info("Analysis report generated in {}ms, dir size={}", stopTime - startTime, humanReadableByteCountSI(FileUtils.sizeOfDirectory(reportDir.toFile())));
 
       startTime = System.currentTimeMillis();
       File reportZip = temp.newFile("scanner-report", ".zip");
       ZipUtils.zipDir(reportDir.toFile(), reportZip);
       stopTime = System.currentTimeMillis();
-      LOG.info("Analysis report compressed in {}ms, zip size={}", stopTime - startTime, FileUtils.byteCountToDisplaySize(FileUtils.sizeOf(reportZip)));
+      LOG.info("Analysis report compressed in {}ms, zip size={}", stopTime - startTime, humanReadableByteCountSI(FileUtils.sizeOf(reportZip)));
       return reportZip;
     } catch (IOException e) {
       throw new IllegalStateException("Unable to prepare analysis report", e);