TemporaryBuffer: Allow presizing block pointer list

Callers may wish to use TemporaryBuffer as an essentially unbounded
buffer by passing Integer.MAX_VALUE as the size. (This makes it
behave like ByteArrayOutputStream, only without requiring contiguous
memory.) Unfortunately, it was always allocating an array in the
backing block pointer list to hold enough blocks to MAX_VALUE--all
262,016 of them. It wasn't allocating the blocks themselves, but this
array was still extremely wasteful, using about 2MiB of memory on a
64-bit system.

Tweak the interface to specify an estimated size, and only allocate
the block pointer list enough entries to hold that size. It's an
ArrayList, so if that estimate was wrong, it'll grow. We assume the
cost of finding enough contiguous memory to grow that array is
acceptable.

While we're in there, fix an off-by-one error: due to integer division
we were undercounting the number of blocks needed to store n bytes of
data as (n / SZ).

Change-Id: I794eca3ac4472bcc605b3641e177922aca92b9c0

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java
index 9817cdc0a1e2697d973330257b3a4d04b9a70c3e..d978804d579e6e72a7ecb511c98eadb40037dd4a 100644 (file)
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java
@@ -53,7 +53,9 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+
 import org.eclipse.jgit.junit.TestRng;
+import org.eclipse.jgit.util.TemporaryBuffer.Block;
 import org.junit.Test;
 
 public class TemporaryBufferTest {
@@ -424,4 +426,27 @@ public class TemporaryBufferTest {
                        assertEquals("In-memory buffer limit exceeded", e.getMessage());
                }
        }
+
+       @Test
+       public void testHeapWithEstimatedSize() throws IOException {
+               int sz = 2 * Block.SZ;
+               try (TemporaryBuffer b = new TemporaryBuffer.Heap(sz / 2, sz)) {
+                       for (int i = 0; i < sz; i++) {
+                               b.write('x');
+                       }
+                       try {
+                               b.write(1);
+                               fail("accepted too many bytes of data");
+                       } catch (IOException e) {
+                               assertEquals("In-memory buffer limit exceeded", e.getMessage());
+                       }
+
+                       try (InputStream in = b.openInputStream()) {
+                               for (int i = 0; i < sz; i++) {
+                                       assertEquals('x', in.read());
+                               }
+                               assertEquals(-1, in.read());
+                       }
+               }
+       }
 }

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
index 10aade4e11e070808b0100a00415a5e3bbac5bec..0a8c5945d9959b074010326275e594d8c3820863 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
@@ -78,6 +78,9 @@ public abstract class TemporaryBuffer extends OutputStream {
         */
        private int inCoreLimit;
 
+       /** Initial size of block list. */
+       private int initialBlocks;
+
        /** If {@link #inCoreLimit} has been reached, remainder goes here. */
        private OutputStream overflow;
 
@@ -86,10 +89,28 @@ public abstract class TemporaryBuffer extends OutputStream {
         *
         * @param limit
         *            maximum number of bytes to store in memory before entering the
-        *            overflow output path.
+        *            overflow output path; also used as the estimated size.
         */
        protected TemporaryBuffer(final int limit) {
-               inCoreLimit = limit;
+               this(limit, limit);
+       }
+
+       /**
+        * Create a new empty temporary buffer.
+        *
+        * @param estimatedSize
+        *            estimated size of storage used, to size the initial list of
+        *            block pointers.
+        * @param limit
+        *            maximum number of bytes to store in memory before entering the
+        *            overflow output path.
+        * @since 4.0
+        */
+       protected TemporaryBuffer(final int estimatedSize, final int limit) {
+               if (estimatedSize > limit)
+                       throw new IllegalArgumentException();
+               this.inCoreLimit = limit;
+               this.initialBlocks = (estimatedSize - 1) / Block.SZ + 1;
                reset();
        }
 
@@ -274,7 +295,7 @@ public abstract class TemporaryBuffer extends OutputStream {
                        blocks = new ArrayList<Block>(1);
                        blocks.add(new Block(inCoreLimit));
                } else {
-                       blocks = new ArrayList<Block>(inCoreLimit / Block.SZ);
+                       blocks = new ArrayList<Block>(initialBlocks);
                        blocks.add(new Block());
                }
        }
@@ -498,12 +519,28 @@ public abstract class TemporaryBuffer extends OutputStream {
                 * Create a new heap buffer with a maximum storage limit.
                 *
                 * @param limit
+                *            maximum number of bytes that can be stored in this buffer;
+                *            also used as the estimated size. Storing beyond this many
+                *            will cause an IOException to be thrown during write.
+                */
+               public Heap(final int limit) {
+                       super(limit);
+               }
+
+               /**
+                * Create a new heap buffer with a maximum storage limit.
+                *
+                * @param estimatedSize
+                *            estimated size of storage used, to size the initial list of
+                *            block pointers.
+                * @param limit
                 *            maximum number of bytes that can be stored in this buffer.
                 *            Storing beyond this many will cause an IOException to be
                 *            thrown during write.
+                * @since 4.0
                 */
-               public Heap(final int limit) {
-                       super(limit);
+               public Heap(final int estimatedSize, final int limit) {
+                       super(estimatedSize, limit);
                }
 
                @Override