PackWriter: Validate reused cached packs

If object reuse validation is enabled, the output pack is going to
probably be stored locally.  When reusing an existing cached pack
to save object enumeration costs, ensure the cached pack has not
been corrupted by checking its SHA-1 trailer.  If it has, writing
will abort and the output pack won't be complete.  This prevents
anyone from trying to use the output pack, and catches corruption
before it can be carried any further.

Change-Id: If89d0d4e429d9f4c86f14de6c0020902705153e6
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

8 files changed, 77 insertions, 22 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteArrayWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteArrayWindow.java
index 5b5a511c17..95e3e00c75 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteArrayWindow.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteArrayWindow.java
@@ -46,6 +46,7 @@
 package org.eclipse.jgit.storage.file;
 
 import java.io.IOException;
+import java.security.MessageDigest;
 import java.util.zip.CRC32;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
@@ -83,8 +84,12 @@ final class ByteArrayWindow extends ByteWindow {
 	}
 
 	@Override
-	void write(PackOutputStream out, long pos, int cnt) throws IOException {
-		out.write(array, (int) (pos - start), cnt);
+	void write(PackOutputStream out, long pos, int cnt, MessageDigest digest)
+			throws IOException {
+		int ptr = (int) (pos - start);
+		out.write(array, ptr, cnt);
+		if (digest != null)
+			digest.update(array, ptr, cnt);
 	}
 
 	void check(Inflater inf, byte[] tmp, long pos, int cnt)
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteBufferWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteBufferWindow.java
index b9af158a05..85441589ad 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteBufferWindow.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteBufferWindow.java
@@ -47,6 +47,7 @@ package org.eclipse.jgit.storage.file;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.security.MessageDigest;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 
@@ -75,7 +76,8 @@ final class ByteBufferWindow extends ByteWindow {
 	}
 
 	@Override
-	void write(PackOutputStream out, long pos, int cnt) throws IOException {
+	void write(PackOutputStream out, long pos, int cnt, MessageDigest digest)
+			throws IOException {
 		final ByteBuffer s = buffer.slice();
 		s.position((int) (pos - start));
 
@@ -84,6 +86,8 @@ final class ByteBufferWindow extends ByteWindow {
 			int n = Math.min(cnt, buf.length);
 			s.get(buf, 0, n);
 			out.write(buf, 0, n);
+			if (digest != null)
+				digest.update(buf, 0, n);
 			cnt -= n;
 		}
 	}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteWindow.java
index 3e147898be..f0b43fdb49 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteWindow.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ByteWindow.java
@@ -45,6 +45,7 @@
 package org.eclipse.jgit.storage.file;
 
 import java.io.IOException;
+import java.security.MessageDigest;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 
@@ -120,8 +121,8 @@ abstract class ByteWindow {
 	 */
 	protected abstract int copy(int pos, byte[] dstbuf, int dstoff, int cnt);
 
-	abstract void write(PackOutputStream out, long pos, int cnt)
-			throws IOException;
+	abstract void write(PackOutputStream out, long pos, int cnt,
+			MessageDigest md) throws IOException;
 
 	final int setInput(long pos, Inflater inf) throws DataFormatException {
 		return setInput((int) (pos - start), inf);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalCachedPack.java
index 18b32c9eae..d0461640f2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalCachedPack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalCachedPack.java
@@ -87,9 +87,10 @@ class LocalCachedPack extends CachedPack {
 		return cnt;
 	}
 
-	void copyAsIs(PackOutputStream out, WindowCursor wc) throws IOException {
+	void copyAsIs(PackOutputStream out, boolean validate, WindowCursor wc)
+			throws IOException {
 		for (String packName : packNames)
-			getPackFile(packName).copyPackAsIs(out, wc);
+			getPackFile(packName).copyPackAsIs(out, validate, wc);
 	}
 
 	@Override
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java
index 80820b2ade..d8d986fb04 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java
@@ -305,11 +305,11 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 		return dstbuf;
 	}
 
-	void copyPackAsIs(PackOutputStream out, WindowCursor curs)
+	void copyPackAsIs(PackOutputStream out, boolean validate, WindowCursor curs)
 			throws IOException {
 		// Pin the first window, this ensures the length is accurate.
 		curs.pin(this, 0);
-		curs.copyPackAsIs(this, out, 12, length - (12 + 20));
+		curs.copyPackAsIs(this, length, validate, out);
 	}
 
 	final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
@@ -462,7 +462,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 			// and we have it pinned.  Write this out without copying.
 			//
 			out.writeHeader(src, inflatedLength);
-			quickCopy.write(out, dataOffset, (int) dataLength);
+			quickCopy.write(out, dataOffset, (int) dataLength, null);
 
 		} else if (dataLength <= buf.length) {
 			// Tiny optimization: Lots of objects are very small deltas or
@@ -499,6 +499,10 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 		return invalid;
 	}
 
+	void setInvalid() {
+		invalid = true;
+	}
+
 	private void readFully(final long position, final byte[] dstbuf,
 			int dstoff, final int cnt, final WindowCursor curs)
 			throws IOException {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
index 661df62cbe..e2dcae82bc 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
@@ -45,6 +45,9 @@
 package org.eclipse.jgit.storage.file;
 
 import java.io.IOException;
+import java.security.MessageDigest;
+import java.text.MessageFormat;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -52,6 +55,7 @@ import java.util.List;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 
+import org.eclipse.jgit.JGitText;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
@@ -197,21 +201,52 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs {
 		return cnt - need;
 	}
 
-	public void copyPackAsIs(PackOutputStream out, CachedPack pack)
-			throws IOException {
-		((LocalCachedPack) pack).copyAsIs(out, this);
+	public void copyPackAsIs(PackOutputStream out, CachedPack pack,
+			boolean validate) throws IOException {
+		((LocalCachedPack) pack).copyAsIs(out, validate, this);
 	}
 
-	void copyPackAsIs(final PackFile pack, final PackOutputStream out,
-			long position, long cnt) throws IOException {
-		while (0 < cnt) {
+	void copyPackAsIs(final PackFile pack, final long length, boolean validate,
+			final PackOutputStream out) throws IOException {
+		MessageDigest md = null;
+		if (validate) {
+			md = Constants.newMessageDigest();
+			byte[] buf = out.getCopyBuffer();
+			pin(pack, 0);
+			if (window.copy(0, buf, 0, 12) != 12) {
+				pack.setInvalid();
+				throw new IOException(JGitText.get().packfileIsTruncated);
+			}
+			md.update(buf, 0, 12);
+		}
+
+		long position = 12;
+		long remaining = length - (12 + 20);
+		while (0 < remaining) {
 			pin(pack, position);
 
 			int ptr = (int) (position - window.start);
-			int n = (int) Math.min(window.size() - ptr, cnt);
-			window.write(out, position, n);
+			int n = (int) Math.min(window.size() - ptr, remaining);
+			window.write(out, position, n, md);
 			position += n;
-			cnt -= n;
+			remaining -= n;
+		}
+
+		if (md != null) {
+			byte[] buf = new byte[20];
+			byte[] actHash = md.digest();
+
+			pin(pack, position);
+			if (window.copy(position, buf, 0, 20) != 20) {
+				pack.setInvalid();
+				throw new IOException(JGitText.get().packfileIsTruncated);
+			}
+			if (!Arrays.equals(actHash, buf)) {
+				pack.setInvalid();
+				throw new IOException(MessageFormat.format(
+						JGitText.get().packfileCorruptionDetected, pack
+								.getPackFile().getPath()));
+			}
 		}
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
index 8ad0b24197..6b10349bca 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
@@ -216,9 +216,14 @@ public interface ObjectReuseAsIs {
 	 *            stream to append the pack onto.
 	 * @param pack
 	 *            the cached pack to send.
+	 * @param validate
+	 *            if true the representation must be validated and not be
+	 *            corrupt before being reused. If false, validation may be
+	 *            skipped as it will be performed elsewhere in the processing
+	 *            pipeline.
 	 * @throws IOException
 	 *             the pack cannot be read, or stream did not accept a write.
 	 */
-	public abstract void copyPackAsIs(PackOutputStream out, CachedPack pack)
-			throws IOException;
+	public abstract void copyPackAsIs(PackOutputStream out, CachedPack pack,
+			boolean validate) throws IOException;
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
index 3d06048688..93c739f5b7 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
@@ -660,7 +660,7 @@ public class PackWriter {
 			stats.reusedObjects += pack.getObjectCount();
 			stats.reusedDeltas += deltaCnt;
 			stats.totalDeltas += deltaCnt;
-			reuseSupport.copyPackAsIs(out, pack);
+			reuseSupport.copyPackAsIs(out, pack, reuseValidate);
 		}
 		writeChecksum(out);
 		out.flush();