From f26b79d04453b64ab9eead7130b099c7c90fd9ab Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Thu, 10 Nov 2011 12:53:50 -0800 Subject: [PATCH] Estimate the amount of memory used by a PackWriter Memory usage is dominated by three terms: - The maximum memory allocated to each delta window. - The maximum size of a single file held in memory during delta search. - ObjectToPack instances owned by the writer. For the first two terms, rather than doing complex instrumentation of the DeltaWindows, we just overestimate based on the config parameters (though we may underestimate if the maximum size is not set). For the ObjectToPack instances, we do some rough byte accounting of the underlying Java object representation. Change-Id: I23fe3cf9d260a91f1aeb6ea22d75af8ddb9b1939 --- .../eclipse/jgit/storage/pack/PackWriter.java | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index 12d601f103..aac288ec5e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -2139,14 +2139,45 @@ public class PackWriter { } private class MutableState { + /** Estimated size of a single ObjectToPack instance. */ + // Assume 64-bit pointers, since this is just an estimate. + private static final long OBJECT_TO_PACK_SIZE = + (2 * 8) // Object header + + (2 * 8) + (2 * 8) // ObjectToPack fields + + (8 + 8) // PackedObjectInfo fields + + 8 // ObjectIdOwnerMap fields + + 40 // AnyObjectId fields + + 8; // Reference in BlockList + + private final long totalDeltaSearchBytes; + private volatile PackingPhase phase; MutableState() { phase = PackingPhase.COUNTING; + if (config.isDeltaCompress()) { + int threads = config.getThreads(); + if (threads <= 0) + threads = Runtime.getRuntime().availableProcessors(); + totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit()) + + config.getBigFileThreshold(); + } else + totalDeltaSearchBytes = 0; } State snapshot() { - return new State(phase); + long objCnt = 0; + objCnt += objectsLists[Constants.OBJ_COMMIT].size(); + objCnt += objectsLists[Constants.OBJ_TREE].size(); + objCnt += objectsLists[Constants.OBJ_BLOB].size(); + objCnt += objectsLists[Constants.OBJ_TAG].size(); + // Exclude CachedPacks. + + long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt; + PackingPhase curr = phase; + if (curr == PackingPhase.COMPRESSING) + bytesUsed += totalDeltaSearchBytes; + return new State(curr, bytesUsed); } } @@ -2172,8 +2203,11 @@ public class PackWriter { public class State { private final PackingPhase phase; - State(PackingPhase phase) { + private final long bytesUsed; + + State(PackingPhase phase, long bytesUsed) { this.phase = phase; + this.bytesUsed = bytesUsed; } /** @return the PackConfig used to build the writer. */ @@ -2186,9 +2220,14 @@ public class PackWriter { return phase; } + /** @return an estimate of the total memory used by the writer. */ + public long estimateBytesUsed() { + return bytesUsed; + } + @Override public String toString() { - return "PackWriter.State[" + phase + "]"; + return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; } } } -- 2.39.5