]> source.dussan.org Git - jgit.git/commitdiff
Add BlobObjectChecker 30/102830/4
authorMasaya Suzuki <masayasuzuki@google.com>
Thu, 10 Aug 2017 06:30:03 +0000 (23:30 -0700)
committerMasaya Suzuki <masayasuzuki@google.com>
Mon, 28 Aug 2017 15:42:27 +0000 (08:42 -0700)
Some repositories can have a policy that do not accept certain blobs. To
check if the incoming pack file contains such blobs, ObjectChecker can
be used. However, this ObjectChecker is not called by PackParser if the
blob is stored as a whole. This is because the object can be so large
that it doesn't fit in memory.

This change introduces BlobObjectChecker. This interface takes chunks of
a blob instead of the entire object. ObjectChecker can optionally return
a BlobObjectChecker. This won't change existing ObjectChecker
implementation; existing implementation continues to receive deltified
blob objects only.

Change-Id: Ic33a92c2de42bd7a89786a4da26b7a648b25218d
Signed-off-by: Masaya Suzuki <masayasuzuki@google.com>
org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java
org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java [new file with mode: 0644]
org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java

index c8729d9443e44020f6b255cf04dcfc4c375259a8..7f783536a1fe8ac4bcb215d60a4506ad7d6dd6d4 100644 (file)
@@ -63,6 +63,7 @@ import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.HAS_DOTGIT;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.NULL_SHA1;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.TREE_NOT_SORTED;
 import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.ZERO_PADDED_FILEMODE;
+import static org.eclipse.jgit.util.RawParseUtils.decode;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.fail;
@@ -73,11 +74,16 @@ import java.text.MessageFormat;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 public class ObjectCheckerTest {
        private ObjectChecker checker;
 
+       @Rule
+       public final ExpectedException thrown = ExpectedException.none();
+
        @Before
        public void setUp() throws Exception {
                checker = new ObjectChecker();
@@ -101,6 +107,98 @@ public class ObjectCheckerTest {
                checker.check(OBJ_BLOB, new byte[1]);
        }
 
+       @Test
+       public void testCheckBlobNotCorrupt() throws CorruptObjectException {
+               checker = new ObjectChecker() {
+                       @Override
+                       public void checkBlob(byte[] raw) throws CorruptObjectException {
+                               String in = decode(raw);
+                               if (in.contains("secret_key")) {
+                                       throw new CorruptObjectException("don't add a secret key");
+                               }
+                       }
+               };
+               checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\""));
+       }
+
+       @Test
+       public void testCheckBlobCorrupt() throws CorruptObjectException {
+               checker = new ObjectChecker() {
+                       @Override
+                       public void checkBlob(byte[] raw) throws CorruptObjectException {
+                               String in = decode(raw);
+                               if (in.contains("secret_key")) {
+                                       throw new CorruptObjectException("don't add a secret key");
+                               }
+                       }
+               };
+               thrown.expect(CorruptObjectException.class);
+               checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\""));
+       }
+
+       @Test
+       public void testCheckBlobWithBlobObjectCheckerNotCorrupt()
+                       throws CorruptObjectException {
+               checker = new ObjectChecker() {
+                       @Override
+                       public BlobObjectChecker newBlobObjectChecker() {
+                               return new BlobObjectChecker() {
+                                       private boolean containSecretKey;
+
+                                       @Override
+                                       public void update(byte[] in, int offset, int len) {
+                                               String str = decode(in, offset, offset + len);
+                                               if (str.contains("secret_key")) {
+                                                       containSecretKey = true;
+                                               }
+                                       }
+
+                                       @Override
+                                       public void endBlob(AnyObjectId id)
+                                               throws CorruptObjectException {
+                                               if (containSecretKey) {
+                                                       throw new CorruptObjectException(
+                                                                       "don't add a secret key");
+                                               }
+                                       }
+                               };
+                       }
+               };
+               checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\""));
+       }
+
+       @Test
+       public void testCheckBlobWithBlobObjectCheckerCorrupt()
+                       throws CorruptObjectException {
+               checker = new ObjectChecker() {
+                       @Override
+                       public BlobObjectChecker newBlobObjectChecker() {
+                               return new BlobObjectChecker() {
+                                       private boolean containSecretKey;
+
+                                       @Override
+                                       public void update(byte[] in, int offset, int len) {
+                                               String str = decode(in, offset, offset + len);
+                                               if (str.contains("secret_key")) {
+                                                       containSecretKey = true;
+                                               }
+                                       }
+
+                                       @Override
+                                       public void endBlob(AnyObjectId id)
+                                               throws CorruptObjectException {
+                                               if (containSecretKey) {
+                                                       throw new CorruptObjectException(
+                                                                       "don't add a secret key");
+                                               }
+                                       }
+                               };
+                       }
+               };
+               thrown.expect(CorruptObjectException.class);
+               checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\""));
+       }
+
        @Test
        public void testValidCommitNoParent() throws CorruptObjectException {
                StringBuilder b = new StringBuilder();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java
new file mode 100644 (file)
index 0000000..0fe63ae
--- /dev/null
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+import org.eclipse.jgit.errors.CorruptObjectException;
+
+/**
+ * Verifies that a blob object is a valid object.
+ * <p>
+ * Unlike trees, commits and tags, there's no validity of blobs. Implementers
+ * can optionally implement this blob checker to reject certain blobs.
+ *
+ * @since 4.9
+ */
+public interface BlobObjectChecker {
+       /** No-op implementation of {@link BlobObjectChecker}. */
+       public static final BlobObjectChecker NULL_CHECKER =
+                       new BlobObjectChecker() {
+                               @Override
+                               public void update(byte[] in, int p, int len) {
+                                       // Empty implementation.
+                               }
+
+                               @Override
+                               public void endBlob(AnyObjectId id) {
+                                       // Empty implementation.
+                               }
+                       };
+
+       /**
+        * Check a new fragment of the blob.
+        *
+        * @param in
+        *            input array of bytes.
+        * @param offset
+        *            offset to start at from {@code in}.
+        * @param len
+        *            length of the fragment to check.
+        */
+       void update(byte[] in, int offset, int len);
+
+       /**
+        * Finalize the blob checking.
+        *
+        * @param id
+        *            identity of the object being checked.
+        * @throws CorruptObjectException
+        *             if any error was detected.
+        */
+       void endBlob(AnyObjectId id) throws CorruptObjectException;
+}
index 9d3aee1508d0cd51fa023dd4b4a6de82445ec9cd..19c5c7eb4530e6134a51018908e18bf6d7477e5f 100644 (file)
@@ -359,7 +359,13 @@ public class ObjectChecker {
                        checkTree(id, raw);
                        break;
                case OBJ_BLOB:
-                       checkBlob(raw);
+                       BlobObjectChecker checker = newBlobObjectChecker();
+                       if (checker == null) {
+                               checkBlob(raw);
+                       } else {
+                               checker.update(raw, 0, raw.length);
+                               checker.endBlob(id);
+                       }
                        break;
                default:
                        report(UNKNOWN_TYPE, id, MessageFormat.format(
@@ -1066,9 +1072,23 @@ public class ObjectChecker {
                return '1' <= b && b <= '9';
        }
 
+       /**
+        * Create a new {@link BlobObjectChecker}.
+        *
+        * @return new BlobObjectChecker or null if it's not provided.
+        * @since 4.9
+        */
+       @Nullable
+       public BlobObjectChecker newBlobObjectChecker() {
+               return null;
+       }
+
        /**
         * Check a blob for errors.
         *
+        * <p>This may not be called from PackParser in some cases. Use {@link
+        * #newBlobObjectChecker} instead.
+        *
         * @param raw
         *            the blob data. The array is never modified.
         * @throws CorruptObjectException
index 2f6b271d877d5ad1304e9b4ca5f2af42a17b022c..833d2114cfb5dcfad59a2bd7672b44fea37db5fb 100644 (file)
@@ -66,6 +66,7 @@ import org.eclipse.jgit.internal.storage.file.PackLock;
 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.BatchingProgressMonitor;
+import org.eclipse.jgit.lib.BlobObjectChecker;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.MutableObjectId;
@@ -1043,16 +1044,25 @@ public abstract class PackParser {
                if (type == Constants.OBJ_BLOB) {
                        byte[] readBuffer = buffer();
                        InputStream inf = inflate(Source.INPUT, sz);
+                       BlobObjectChecker checker = null;
+                       if (objCheck != null) {
+                               checker = objCheck.newBlobObjectChecker();
+                       }
+                       if (checker == null) {
+                               checker = BlobObjectChecker.NULL_CHECKER;
+                       }
                        long cnt = 0;
                        while (cnt < sz) {
                                int r = inf.read(readBuffer);
                                if (r <= 0)
                                        break;
                                objectDigest.update(readBuffer, 0, r);
+                               checker.update(readBuffer, 0, r);
                                cnt += r;
                        }
                        inf.close();
                        objectDigest.digest(tempObjectId);
+                       checker.endBlob(tempObjectId);
                        data = null;
                } else {
                        data = inflateAndReturn(Source.INPUT, sz);