Add BlobObjectChecker

Some repositories can have a policy that do not accept certain blobs. To
check if the incoming pack file contains such blobs, ObjectChecker can
be used. However, this ObjectChecker is not called by PackParser if the
blob is stored as a whole. This is because the object can be so large
that it doesn't fit in memory.

This change introduces BlobObjectChecker. This interface takes chunks of
a blob instead of the entire object. ObjectChecker can optionally return
a BlobObjectChecker. This won't change existing ObjectChecker
implementation; existing implementation continues to receive deltified
blob objects only.

Change-Id: Ic33a92c2de42bd7a89786a4da26b7a648b25218d
Signed-off-by: Masaya Suzuki <masayasuzuki@google.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java

@@ -63,6 +63,7 @@ import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.HAS_DOTGIT;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.NULL_SHA1;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.TREE_NOT_SORTED;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.ZERO_PADDED_FILEMODE;
+import static org.eclipse.jgit.util.RawParseUtils.decode;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.fail;
@@ -73,11 +74,16 @@ import java.text.MessageFormat;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 public class ObjectCheckerTest {
 	private ObjectChecker checker;
 
+	@Rule
+	public final ExpectedException thrown = ExpectedException.none();
+
 	@Before
 	public void setUp() throws Exception {
 		checker = new ObjectChecker();
@@ -101,6 +107,98 @@ public class ObjectCheckerTest {
 		checker.check(OBJ_BLOB, new byte[1]);
 	}
 
+	@Test
+	public void testCheckBlobNotCorrupt() throws CorruptObjectException {
+		checker = new ObjectChecker() {
+			@Override
+			public void checkBlob(byte[] raw) throws CorruptObjectException {
+				String in = decode(raw);
+				if (in.contains("secret_key")) {
+					throw new CorruptObjectException("don't add a secret key");
+				}
+			}
+		};
+		checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\""));
+	}
+
+	@Test
+	public void testCheckBlobCorrupt() throws CorruptObjectException {
+		checker = new ObjectChecker() {
+			@Override
+			public void checkBlob(byte[] raw) throws CorruptObjectException {
+				String in = decode(raw);
+				if (in.contains("secret_key")) {
+					throw new CorruptObjectException("don't add a secret key");
+				}
+			}
+		};
+		thrown.expect(CorruptObjectException.class);
+		checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\""));
+	}
+
+	@Test
+	public void testCheckBlobWithBlobObjectCheckerNotCorrupt()
+			throws CorruptObjectException {
+		checker = new ObjectChecker() {
+			@Override
+			public BlobObjectChecker newBlobObjectChecker() {
+				return new BlobObjectChecker() {
+					private boolean containSecretKey;
+
+					@Override
+					public void update(byte[] in, int offset, int len) {
+						String str = decode(in, offset, offset + len);
+						if (str.contains("secret_key")) {
+							containSecretKey = true;
+						}
+					}
+
+					@Override
+					public void endBlob(AnyObjectId id)
+						throws CorruptObjectException {
+						if (containSecretKey) {
+							throw new CorruptObjectException(
+									"don't add a secret key");
+						}
+					}
+				};
+			}
+		};
+		checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\""));
+	}
+
+	@Test
+	public void testCheckBlobWithBlobObjectCheckerCorrupt()
+			throws CorruptObjectException {
+		checker = new ObjectChecker() {
+			@Override
+			public BlobObjectChecker newBlobObjectChecker() {
+				return new BlobObjectChecker() {
+					private boolean containSecretKey;
+
+					@Override
+					public void update(byte[] in, int offset, int len) {
+						String str = decode(in, offset, offset + len);
+						if (str.contains("secret_key")) {
+							containSecretKey = true;
+						}
+					}
+
+					@Override
+					public void endBlob(AnyObjectId id)
+						throws CorruptObjectException {
+						if (containSecretKey) {
+							throw new CorruptObjectException(
+									"don't add a secret key");
+						}
+					}
+				};
+			}
+		};
+		thrown.expect(CorruptObjectException.class);
+		checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\""));
+	}
+
 	@Test
 	public void testValidCommitNoParent() throws CorruptObjectException {
 		StringBuilder b = new StringBuilder();

org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+import org.eclipse.jgit.errors.CorruptObjectException;
+
+/**
+ * Verifies that a blob object is a valid object.
+ * <p>
+ * Unlike trees, commits and tags, there's no validity of blobs. Implementers
+ * can optionally implement this blob checker to reject certain blobs.
+ *
+ * @since 4.9
+ */
+public interface BlobObjectChecker {
+	/** No-op implementation of {@link BlobObjectChecker}. */
+	public static final BlobObjectChecker NULL_CHECKER =
+			new BlobObjectChecker() {
+				@Override
+				public void update(byte[] in, int p, int len) {
+					// Empty implementation.
+				}
+
+				@Override
+				public void endBlob(AnyObjectId id) {
+					// Empty implementation.
+				}
+			};
+
+	/**
+	 * Check a new fragment of the blob.
+	 *
+	 * @param in
+	 *            input array of bytes.
+	 * @param offset
+	 *            offset to start at from {@code in}.
+	 * @param len
+	 *            length of the fragment to check.
+	 */
+	void update(byte[] in, int offset, int len);
+
+	/**
+	 * Finalize the blob checking.
+	 *
+	 * @param id
+	 *            identity of the object being checked.
+	 * @throws CorruptObjectException
+	 *             if any error was detected.
+	 */
+	void endBlob(AnyObjectId id) throws CorruptObjectException;
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java

@@ -359,7 +359,13 @@ public class ObjectChecker {
 			checkTree(id, raw);
 			break;
 		case OBJ_BLOB:
-			checkBlob(raw);
+			BlobObjectChecker checker = newBlobObjectChecker();
+			if (checker == null) {
+				checkBlob(raw);
+			} else {
+				checker.update(raw, 0, raw.length);
+				checker.endBlob(id);
+			}
 			break;
 		default:
 			report(UNKNOWN_TYPE, id, MessageFormat.format(
@@ -1066,9 +1072,23 @@ public class ObjectChecker {
 		return '1' <= b && b <= '9';
 	}
 
+	/**
+	 * Create a new {@link BlobObjectChecker}.
+	 *
+	 * @return new BlobObjectChecker or null if it's not provided.
+	 * @since 4.9
+	 */
+	@Nullable
+	public BlobObjectChecker newBlobObjectChecker() {
+		return null;
+	}
+
 	/**
 	 * Check a blob for errors.
 	 *
+	 * <p>This may not be called from PackParser in some cases. Use {@link
+	 * #newBlobObjectChecker} instead.
+	 *
 	 * @param raw
 	 *            the blob data. The array is never modified.
 	 * @throws CorruptObjectException

org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java

@@ -66,6 +66,7 @@ import org.eclipse.jgit.internal.storage.file.PackLock;
 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.BatchingProgressMonitor;
+import org.eclipse.jgit.lib.BlobObjectChecker;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.MutableObjectId;
@@ -1043,16 +1044,25 @@ public abstract class PackParser {
 		if (type == Constants.OBJ_BLOB) {
 			byte[] readBuffer = buffer();
 			InputStream inf = inflate(Source.INPUT, sz);
+			BlobObjectChecker checker = null;
+			if (objCheck != null) {
+				checker = objCheck.newBlobObjectChecker();
+			}
+			if (checker == null) {
+				checker = BlobObjectChecker.NULL_CHECKER;
+			}
 			long cnt = 0;
 			while (cnt < sz) {
 				int r = inf.read(readBuffer);
 				if (r <= 0)
 					break;
 				objectDigest.update(readBuffer, 0, r);
+				checker.update(readBuffer, 0, r);
 				cnt += r;
 			}
 			inf.close();
 			objectDigest.digest(tempObjectId);
+			checker.endBlob(tempObjectId);
 			data = null;
 		} else {
 			data = inflateAndReturn(Source.INPUT, sz);