Honor pack.windowlimit to cap memory usage during packing

The pack.windowlimit configuration parameter places an upper bound
on the number of bytes used by the DeltaWindow class as it scans
through the object list.  If memory usage would exceed the limit
the window is temporarily decreased in size to keep memory used
within that bound.

Change-Id: I09521b8f335475d8aee6125826da8ba2e545060d
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java
index c45a076b358f7a94871eae93109e77d2922a1400..e548cc936591ce8964da6ba5aaeaa1231cd72f4c 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaIndex.java
@@ -70,6 +70,19 @@ public class DeltaIndex {
        /** Number of bytes in a block. */
        static final int BLKSZ = 16; // must be 16, see unrolled loop in hashBlock
 
+       /**
+        * Estimate the size of an index for a given source.
+        * <p>
+        * This is roughly a worst-case estimate. The actual index may be smaller.
+        *
+        * @param sourceLength
+        *            length of the source, in bytes.
+        * @return estimated size. Approximately {@code 1.75 * sourceLength}.
+        */
+       public static long estimateIndexSize(int sourceLength) {
+               return sourceLength + (sourceLength * 3 / 4);
+       }
+
        /**
         * Maximum number of positions to consider for a given content hash.
         * <p>

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java
index fa577b669926cd64c5424870d9d0c8f7cb32b3c5..ee865fbe5a569b8d70e5409b821855d152fc17e4 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java
@@ -68,9 +68,15 @@ class DeltaWindow {
 
        private final DeltaWindowEntry[] window;
 
+       /** Maximum number of bytes to admit to the window at once. */
+       private final long maxMemory;
+
        /** Maximum depth we should create for any delta chain. */
        private final int maxDepth;
 
+       /** Amount of memory we have loaded right now. */
+       private long loaded;
+
        // The object we are currently considering needs a lot of state:
 
        /** Position of {@link #res} within {@link #window} array. */
@@ -115,6 +121,7 @@ class DeltaWindow {
                for (int i = 0; i < window.length; i++)
                        window[i] = new DeltaWindowEntry();
 
+               maxMemory = pw.getDeltaSearchMemoryLimit();
                maxDepth = pw.getMaxDeltaDepth();
        }
 
@@ -125,6 +132,15 @@ class DeltaWindow {
                                monitor.update(1);
 
                                res = window[resSlot];
+                               if (0 < maxMemory) {
+                                       clear(res);
+                                       int tail = next(resSlot);
+                                       final long need = estimateSize(toSearch[off]);
+                                       while (maxMemory < loaded + need && tail != resSlot) {
+                                               clear(window[tail]);
+                                               tail = next(tail);
+                                       }
+                               }
                                res.set(toSearch[off]);
 
                                if (res.object.isDoNotDelta()) {
@@ -148,6 +164,18 @@ class DeltaWindow {
                }
        }
 
+       private static long estimateSize(ObjectToPack ent) {
+               return DeltaIndex.estimateIndexSize(ent.getWeight());
+       }
+
+       private void clear(DeltaWindowEntry ent) {
+               if (ent.index != null)
+                       loaded -= ent.index.getIndexSize();
+               else if (res.buffer != null)
+                       loaded -= ent.buffer.length;
+               ent.set(null);
+       }
+
        private void search() throws IOException {
                // TODO(spearce) If the object is used as a base for other
                // objects in this pack we should limit the depth we create
@@ -336,8 +364,13 @@ class DeltaWindow {
        }
 
        private void keepInWindow() {
-               if (++resSlot == window.length)
-                       resSlot = 0;
+               resSlot = next(resSlot);
+       }
+
+       private int next(int slot) {
+               if (++slot == window.length)
+                       return 0;
+               return slot;
        }
 
        private int prior(int slot) {
@@ -397,6 +430,8 @@ class DeltaWindow {
                                e.initCause(noMemory);
                                throw e;
                        }
+                       if (0 < maxMemory)
+                               loaded += idx.getIndexSize() - idx.getSourceSize();
                        ent.index = idx;
                }
                return idx;
@@ -405,8 +440,12 @@ class DeltaWindow {
        private byte[] buffer(DeltaWindowEntry ent) throws MissingObjectException,
                        IncorrectObjectTypeException, IOException, LargeObjectException {
                byte[] buf = ent.buffer;
-               if (buf == null)
-                       ent.buffer = buf = writer.buffer(reader, ent.object);
+               if (buf == null) {
+                       buf = writer.buffer(reader, ent.object);
+                       if (0 < maxMemory)
+                               loaded += buf.length;
+                       ent.buffer = buf;
+               }
                return buf;
        }
 

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
index 3769d6b4bfe449cd3a38c40a8eb4ae8c3e798afe..38f00b5739f58d325133a62e3cf0025f06d03e72 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
@@ -229,6 +229,8 @@ public class PackWriter {
 
        private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE;
 
+       private long deltaSearchMemoryLimit;
+
        private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE;
 
        private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT;
@@ -289,6 +291,7 @@ public class PackWriter {
 
                final PackConfig pc = configOf(repo).get(PackConfig.KEY);
                deltaSearchWindowSize = pc.deltaWindow;
+               deltaSearchMemoryLimit = pc.deltaWindowMemory;
                deltaCacheSize = pc.deltaCacheSize;
                deltaCacheLimit = pc.deltaCacheLimit;
                maxDeltaDepth = pc.deltaDepth;
@@ -485,6 +488,36 @@ public class PackWriter {
                        deltaSearchWindowSize = objectCount;
        }
 
+       /**
+        * Get maximum number of bytes to put into the delta search window.
+        * <p>
+        * Default setting is 0, for an unlimited amount of memory usage. Actual
+        * memory used is the lower limit of either this setting, or the sum of
+        * space used by at most {@link #getDeltaSearchWindowSize()} objects.
+        * <p>
+        * This limit is per thread, if 4 threads are used the actual memory
+        * limit will be 4 times this value.
+        *
+        * @return the memory limit.
+        */
+       public long getDeltaSearchMemoryLimit() {
+               return deltaSearchMemoryLimit;
+       }
+
+       /**
+        * Set the maximum number of bytes to put into the delta search window.
+        * <p>
+        * Default setting is 0, for an unlimited amount of memory usage. If the
+        * memory limit is reached before {@link #getDeltaSearchWindowSize()} the
+        * window size is temporarily lowered.
+        *
+        * @param memoryLimit
+        *            Maximum number of bytes to load at once, 0 for unlimited.
+        */
+       public void setDeltaSearchMemoryLimit(long memoryLimit) {
+               deltaSearchMemoryLimit = memoryLimit;
+       }
+
        /**
         * Get the size of the in-memory delta cache.
         * <p>