Remove streaming delta support from JGit

Streaming packed deltas is so slow that it never feasibly completes
(it will take hours for it to stream a few hundred megabytes on
relatively fast systems with a large amount of storage).  This
was indicated as a "failed experiment" by Shawn in the following
mailing list post:
http://dev.eclipse.org/mhonarc/lists/jgit-dev/msg01674.html

Change-Id: Idc12f59e37b122f13856d7b533a5af9d8867a8a5
Signed-off-by: Doug Kelly <dougk.ff7@gmail.com>

2 files changed, 66 insertions, 372 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java
index ced4575f20..a19fcd9d50 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java
@@ -47,20 +47,27 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.security.MessageDigest;
 import java.text.MessageFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.zip.Deflater;
 
 import org.eclipse.jgit.errors.LargeObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.internal.storage.pack.DeltaEncoder;
+import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
 import org.eclipse.jgit.junit.TestRepository;
@@ -75,6 +82,7 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevBlob;
 import org.eclipse.jgit.storage.file.WindowCacheConfig;
 import org.eclipse.jgit.transport.PackParser;
+import org.eclipse.jgit.transport.PackedObjectInfo;
 import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.NB;
 import org.eclipse.jgit.util.TemporaryBuffer;
@@ -105,7 +113,7 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
 
 		WindowCacheConfig cfg = new WindowCacheConfig();
 		cfg.setStreamFileThreshold(streamThreshold);
-		WindowCache.reconfigure(cfg);
+		cfg.install();
 
 		repo = createBareRepository();
 		tr = new TestRepository<Repository>(repo);
@@ -116,7 +124,7 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
 	public void tearDown() throws Exception {
 		if (wc != null)
 			wc.release();
-		WindowCache.reconfigure(new WindowCacheConfig());
+		new WindowCacheConfig().install();
 		super.tearDown();
 	}
 
@@ -241,68 +249,65 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
 	}
 
 	@Test
-	public void testDelta_LargeObjectChain() throws Exception {
+	public void testDelta_FailsOver2GiB() throws Exception {
 		ObjectInserter.Formatter fmt = new ObjectInserter.Formatter();
-		byte[] data0 = new byte[streamThreshold + 5];
-		Arrays.fill(data0, (byte) 0xf3);
-		ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0);
+		byte[] base = new byte[] { 'a' };
+		ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base);
+		ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' });
 
-		TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024);
-		packHeader(pack, 4);
-		objectHeader(pack, Constants.OBJ_BLOB, data0.length);
-		deflate(pack, data0);
+		PackedObjectInfo a = new PackedObjectInfo(idA);
+		PackedObjectInfo b = new PackedObjectInfo(idB);
 
-		byte[] data1 = clone(0x01, data0);
-		byte[] delta1 = delta(data0, data1);
-		ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1);
-		objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length);
-		id0.copyRawTo(pack);
-		deflate(pack, delta1);
-
-		byte[] data2 = clone(0x02, data1);
-		byte[] delta2 = delta(data1, data2);
-		ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2);
-		objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length);
-		id1.copyRawTo(pack);
-		deflate(pack, delta2);
-
-		byte[] data3 = clone(0x03, data2);
-		byte[] delta3 = delta(data2, data3);
-		ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3);
-		objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length);
-		id2.copyRawTo(pack);
-		deflate(pack, delta3);
-
-		digest(pack);
-		PackParser ip = index(pack.toByteArray());
-		ip.setAllowThin(true);
-		ip.parse(NullProgressMonitor.INSTANCE);
+		TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024);
+		packHeader(pack, 2);
+		a.setOffset(pack.length());
+		objectHeader(pack, Constants.OBJ_BLOB, base.length);
+		deflate(pack, base);
 
-		assertTrue("has blob", wc.has(id3));
+		ByteArrayOutputStream tmp = new ByteArrayOutputStream();
+		DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30);
+		de.copy(0, 1);
+		byte[] delta = tmp.toByteArray();
+		b.setOffset(pack.length());
+		objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length);
+		idA.copyRawTo(pack);
+		deflate(pack, delta);
+		byte[] footer = digest(pack);
+
+		File packName = new File(new File(
+				((FileObjectDatabase) repo.getObjectDatabase()).getDirectory(),
+				"pack"), idA.name() + ".pack");
+		File idxName = new File(new File(
+				((FileObjectDatabase) repo.getObjectDatabase()).getDirectory(),
+				"pack"), idA.name() + ".idx");
+
+		FileOutputStream f = new FileOutputStream(packName);
+		try {
+			f.write(pack.toByteArray());
+		} finally {
+			f.close();
+		}
 
-		ObjectLoader ol = wc.open(id3);
-		assertNotNull("created loader", ol);
-		assertEquals(Constants.OBJ_BLOB, ol.getType());
-		assertEquals(data3.length, ol.getSize());
-		assertTrue("is large", ol.isLarge());
+		f = new FileOutputStream(idxName);
 		try {
-			ol.getCachedBytes();
-			fail("Should have thrown LargeObjectException");
-		} catch (LargeObjectException tooBig) {
-			assertEquals(MessageFormat.format(
-					JGitText.get().largeObjectException, id3.name()), tooBig
-					.getMessage());
+			List<PackedObjectInfo> list = new ArrayList<PackedObjectInfo>();
+			list.add(a);
+			list.add(b);
+			Collections.sort(list);
+			new PackIndexWriterV1(f).write(list, footer);
+		} finally {
+			f.close();
 		}
 
-		ObjectStream in = ol.openStream();
-		assertNotNull("have stream", in);
-		assertEquals(Constants.OBJ_BLOB, in.getType());
-		assertEquals(data3.length, in.getSize());
-		byte[] act = new byte[data3.length];
-		IO.readFully(in, act, 0, data3.length);
-		assertTrue("same content", Arrays.equals(act, data3));
-		assertEquals("stream at EOF", -1, in.read());
-		in.close();
+		PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit());
+		try {
+			packFile.get(wc, b);
+			fail("expected LargeObjectException.ExceedsByteArrayLimit");
+		} catch (LargeObjectException.ExceedsByteArrayLimit bad) {
+			assertNull(bad.getObjectId());
+		} finally {
+			packFile.close();
+		}
 	}
 
 	private static byte[] clone(int first, byte[] base) {
@@ -358,10 +363,13 @@ public class PackFileTest extends LocalDiskRepositoryTestCase {
 		deflater.end();
 	}
 
-	private static void digest(TemporaryBuffer.Heap buf) throws IOException {
+	private static byte[] digest(TemporaryBuffer.Heap buf)
+			throws IOException {
 		MessageDigest md = Constants.newMessageDigest();
 		md.update(buf.toByteArray());
-		buf.write(md.digest());
+		byte[] footer = md.digest();
+		buf.write(footer);
+		return footer;
 	}
 
 	private ObjectInserter inserter;
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/DeltaStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/DeltaStreamTest.java
deleted file mode 100644
index 3ea09179de..0000000000
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/DeltaStreamTest.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Copyright (C) 2010, Google Inc.
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- *   notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- *   copyright notice, this list of conditions and the following
- *   disclaimer in the documentation and/or other materials provided
- *   with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- *   names of its contributors may be used to endorse or promote
- *   products derived from this software without specific prior
- *   written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package org.eclipse.jgit.internal.storage.pack;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-
-import org.eclipse.jgit.errors.CorruptObjectException;
-import org.eclipse.jgit.internal.JGitText;
-import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
-import org.eclipse.jgit.internal.storage.pack.DeltaEncoder;
-import org.eclipse.jgit.internal.storage.pack.DeltaStream;
-import org.eclipse.jgit.junit.JGitTestUtil;
-import org.eclipse.jgit.junit.TestRng;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.util.IO;
-import org.junit.Before;
-import org.junit.Test;
-
-public class DeltaStreamTest {
-	private TestRng rng;
-
-	private ByteArrayOutputStream deltaBuf;
-
-	private DeltaEncoder deltaEnc;
-
-	private byte[] base;
-
-	private byte[] data;
-
-	private int dataPtr;
-
-	private byte[] delta;
-
-	private TestRng getRng() {
-		if (rng == null)
-			rng = new TestRng(JGitTestUtil.getName());
-		return rng;
-	}
-
-	@Before
-	public void setUp() throws Exception {
-		deltaBuf = new ByteArrayOutputStream();
-	}
-
-	@Test
-	public void testCopy_SingleOp() throws IOException {
-		init((1 << 16) + 1, (1 << 8) + 1);
-		copy(0, data.length);
-		assertValidState();
-	}
-
-	@Test
-	public void testCopy_MaxSize() throws IOException {
-		int max = (0xff << 16) + (0xff << 8) + 0xff;
-		init(1 + max, max);
-		copy(1, max);
-		assertValidState();
-	}
-
-	@Test
-	public void testCopy_64k() throws IOException {
-		init(0x10000 + 2, 0x10000 + 1);
-		copy(1, 0x10000);
-		copy(0x10001, 1);
-		assertValidState();
-	}
-
-	@Test
-	public void testCopy_Gap() throws IOException {
-		init(256, 8);
-		copy(4, 4);
-		copy(128, 4);
-		assertValidState();
-	}
-
-	@Test
-	public void testCopy_OutOfOrder() throws IOException {
-		init((1 << 16) + 1, (1 << 16) + 1);
-		copy(1 << 8, 1 << 8);
-		copy(0, data.length - dataPtr);
-		assertValidState();
-	}
-
-	@Test
-	public void testInsert_SingleOp() throws IOException {
-		init((1 << 16) + 1, 2);
-		insert("hi");
-		assertValidState();
-	}
-
-	@Test
-	public void testInsertAndCopy() throws IOException {
-		init(8, 512);
-		insert(new byte[127]);
-		insert(new byte[127]);
-		insert(new byte[127]);
-		insert(new byte[125]);
-		copy(2, 6);
-		assertValidState();
-	}
-
-	@Test
-	public void testSkip() throws IOException {
-		init(32, 15);
-		copy(2, 2);
-		insert("ab");
-		insert("cd");
-		copy(4, 4);
-		copy(0, 2);
-		insert("efg");
-		assertValidState();
-
-		for (int p = 0; p < data.length; p++) {
-			byte[] act = new byte[data.length];
-			System.arraycopy(data, 0, act, 0, p);
-			DeltaStream in = open();
-			IO.skipFully(in, p);
-			assertEquals(data.length - p, in.read(act, p, data.length - p));
-			assertEquals(-1, in.read());
-			assertTrue("skipping " + p, Arrays.equals(data, act));
-		}
-
-		// Skip all the way to the end should still recognize EOF.
-		DeltaStream in = open();
-		IO.skipFully(in, data.length);
-		assertEquals(-1, in.read());
-		assertEquals(0, in.skip(1));
-
-		// Skip should not open the base as we move past it, but it
-		// will open when we need to start copying data from it.
-		final boolean[] opened = new boolean[1];
-		in = new DeltaStream(new ByteArrayInputStream(delta)) {
-			@Override
-			protected long getBaseSize() throws IOException {
-				return base.length;
-			}
-
-			@Override
-			protected InputStream openBase() throws IOException {
-				opened[0] = true;
-				return new ByteArrayInputStream(base);
-			}
-		};
-		IO.skipFully(in, 7);
-		assertFalse("not yet open", opened[0]);
-		assertEquals(data[7], in.read());
-		assertTrue("now open", opened[0]);
-	}
-
-	@Test
-	public void testIncorrectBaseSize() throws IOException {
-		init(4, 4);
-		copy(0, 4);
-		assertValidState();
-
-		DeltaStream in = new DeltaStream(new ByteArrayInputStream(delta)) {
-			@Override
-			protected long getBaseSize() throws IOException {
-				return 128;
-			}
-
-			@Override
-			protected InputStream openBase() throws IOException {
-				return new ByteArrayInputStream(base);
-			}
-		};
-		try {
-			in.read(new byte[4]);
-			fail("did not throw an exception");
-		} catch (CorruptObjectException e) {
-			assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage());
-		}
-
-		in = new DeltaStream(new ByteArrayInputStream(delta)) {
-			@Override
-			protected long getBaseSize() throws IOException {
-				return 4;
-			}
-
-			@Override
-			protected InputStream openBase() throws IOException {
-				return new ByteArrayInputStream(new byte[0]);
-			}
-		};
-		try {
-			in.read(new byte[4]);
-			fail("did not throw an exception");
-		} catch (CorruptObjectException e) {
-			assertEquals(JGitText.get().baseLengthIncorrect, e.getMessage());
-		}
-	}
-
-	private void init(int baseSize, int dataSize) throws IOException {
-		base = getRng().nextBytes(baseSize);
-		data = new byte[dataSize];
-		deltaEnc = new DeltaEncoder(deltaBuf, baseSize, dataSize);
-	}
-
-	private void copy(int offset, int len) throws IOException {
-		System.arraycopy(base, offset, data, dataPtr, len);
-		deltaEnc.copy(offset, len);
-		assertEquals(deltaBuf.size(), deltaEnc.getSize());
-		dataPtr += len;
-	}
-
-	private void insert(String text) throws IOException {
-		insert(Constants.encode(text));
-	}
-
-	private void insert(byte[] text) throws IOException {
-		System.arraycopy(text, 0, data, dataPtr, text.length);
-		deltaEnc.insert(text);
-		assertEquals(deltaBuf.size(), deltaEnc.getSize());
-		dataPtr += text.length;
-	}
-
-	private void assertValidState() throws IOException {
-		assertEquals("test filled example result", data.length, dataPtr);
-
-		delta = deltaBuf.toByteArray();
-		assertEquals(base.length, BinaryDelta.getBaseSize(delta));
-		assertEquals(data.length, BinaryDelta.getResultSize(delta));
-		assertArrayEquals(data, BinaryDelta.apply(base, delta));
-
-		// Assert that a single bulk read produces the correct result.
-		//
-		byte[] act = new byte[data.length];
-		DeltaStream in = open();
-		assertEquals(data.length, in.getSize());
-		assertEquals(data.length, in.read(act));
-		assertEquals(-1, in.read());
-		assertTrue("bulk read has same content", Arrays.equals(data, act));
-
-		// Assert that smaller tiny reads have the same result too.
-		//
-		act = new byte[data.length];
-		in = open();
-		int read = 0;
-		while (read < data.length) {
-			int n = in.read(act, read, 128);
-			if (n <= 0)
-				break;
-			read += n;
-		}
-		assertEquals(data.length, read);
-		assertEquals(-1, in.read());
-		assertTrue("small reads have same content", Arrays.equals(data, act));
-	}
-
-	private DeltaStream open() throws IOException {
-		return new DeltaStream(new ByteArrayInputStream(delta)) {
-			@Override
-			protected long getBaseSize() throws IOException {
-				return base.length;
-			}
-
-			@Override
-			protected InputStream openBase() throws IOException {
-				return new ByteArrayInputStream(base);
-			}
-		};
-	}
-}