Estimate the amount of memory used by a PackWriter

Memory usage is dominated by three terms:
 - The maximum memory allocated to each delta window.
 - The maximum size of a single file held in memory during delta search.
 - ObjectToPack instances owned by the writer.

For the first two terms, rather than doing complex instrumentation of
the DeltaWindows, we just overestimate based on the config parameters
(though we may underestimate if the maximum size is not set).

For the ObjectToPack instances, we do some rough byte accounting of the
underlying Java object representation.

Change-Id: I23fe3cf9d260a91f1aeb6ea22d75af8ddb9b1939

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
index 12d601f1031e03e3f3bdeb625167c272428af81f..aac288ec5e7b04b3d586c1214d8b688d75dadb1e 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
@@ -2139,14 +2139,45 @@ public class PackWriter {
        }
 
        private class MutableState {
+               /** Estimated size of a single ObjectToPack instance. */
+               // Assume 64-bit pointers, since this is just an estimate.
+               private static final long OBJECT_TO_PACK_SIZE =
+                               (2 * 8)               // Object header
+                               + (2 * 8) + (2 * 8)   // ObjectToPack fields
+                               + (8 + 8)             // PackedObjectInfo fields
+                               + 8                   // ObjectIdOwnerMap fields
+                               + 40                  // AnyObjectId fields
+                               + 8;                  // Reference in BlockList
+
+               private final long totalDeltaSearchBytes;
+
                private volatile PackingPhase phase;
 
                MutableState() {
                        phase = PackingPhase.COUNTING;
+                       if (config.isDeltaCompress()) {
+                               int threads = config.getThreads();
+                               if (threads <= 0)
+                                       threads = Runtime.getRuntime().availableProcessors();
+                               totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit())
+                                               + config.getBigFileThreshold();
+                       } else
+                               totalDeltaSearchBytes = 0;
                }
 
                State snapshot() {
-                       return new State(phase);
+                       long objCnt = 0;
+                       objCnt += objectsLists[Constants.OBJ_COMMIT].size();
+                       objCnt += objectsLists[Constants.OBJ_TREE].size();
+                       objCnt += objectsLists[Constants.OBJ_BLOB].size();
+                       objCnt += objectsLists[Constants.OBJ_TAG].size();
+                       // Exclude CachedPacks.
+
+                       long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt;
+                       PackingPhase curr = phase;
+                       if (curr == PackingPhase.COMPRESSING)
+                               bytesUsed += totalDeltaSearchBytes;
+                       return new State(curr, bytesUsed);
                }
        }
 
@@ -2172,8 +2203,11 @@ public class PackWriter {
        public class State {
                private final PackingPhase phase;
 
-               State(PackingPhase phase) {
+               private final long bytesUsed;
+
+               State(PackingPhase phase, long bytesUsed) {
                        this.phase = phase;
+                       this.bytesUsed = bytesUsed;
                }
 
                /** @return the PackConfig used to build the writer. */
@@ -2186,9 +2220,14 @@ public class PackWriter {
                        return phase;
                }
 
+               /** @return an estimate of the total memory used by the writer. */
+               public long estimateBytesUsed() {
+                       return bytesUsed;
+               }
+
                @Override
                public String toString() {
-                       return "PackWriter.State[" + phase + "]";
+                       return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
                }
        }
 }