From: Shawn O. Pearce Date: Fri, 9 Jul 2010 00:08:55 +0000 (-0700) Subject: Add getObjectSize to ObjectReader X-Git-Tag: v0.9.1~157^2~16 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b584cb8754796bb96526b37b26b517741b9c9d1c;p=jgit.git Add getObjectSize to ObjectReader This is an informational function used by PackWriter to help it better organize objects for delta compression. Storage systems can implement it to provide up more detailed size information, or they can simply rely on the default behavior that uses the ObjectLoader obtained from open. For local file storage, we can obtain this information faster through specialized routines that parse a pack object header. Change-Id: I13a09b4effb71ea5151b51547f7d091564531e58 Signed-off-by: Shawn O. Pearce --- diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index 1af3cb2dea..d0351f8670 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -135,6 +135,34 @@ public abstract class ObjectReader { throws MissingObjectException, IncorrectObjectTypeException, IOException; + /** + * Get only the size of an object. + *

+ * The default implementation of this method opens an ObjectLoader. + * Databases are encouraged to override this if a faster access method is + * available to them. + * + * @param objectId + * identity of the object to open. + * @param typeHint + * hint about the type of object being requested; + * {@link #OBJ_ANY} if the object type is not known, or does not + * matter to the caller. + * @return size of object in bytes. + * @throws MissingObjectException + * the object does not exist. + * @throws IncorrectObjectTypeException + * typeHint was not OBJ_ANY, and the object's actual type does + * not match typeHint. + * @throws IOException + * the object store cannot be accessed. + */ + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + return open(objectId, typeHint).getSize(); + } + /** * Release any resources used by this reader. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java index 505850c427..8ea0b854c0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java @@ -186,6 +186,20 @@ class CachedObjectDirectory extends FileObjectDatabase { throw new UnsupportedOperationException(); } + @Override + long getObjectSize1(WindowCursor curs, AnyObjectId objectId) throws IOException { + if (unpackedObjects.contains(objectId)) + return wrapped.getObjectSize2(curs, objectId.name(), objectId); + return wrapped.getObjectSize1(curs, objectId); + } + + @Override + long getObjectSize2(WindowCursor curs, String objectName, AnyObjectId objectId) + throws IOException { + // This method should never be invoked. + throw new UnsupportedOperationException(); + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java index 444fd809b6..250c7cac0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java @@ -166,6 +166,55 @@ abstract class FileObjectDatabase extends ObjectDatabase { return null; } + long getObjectSize(WindowCursor curs, AnyObjectId objectId) + throws IOException { + long sz = getObjectSizeImpl1(curs, objectId); + if (0 <= sz) + return sz; + return getObjectSizeImpl2(curs, objectId.name(), objectId); + } + + final long getObjectSizeImpl1(final WindowCursor curs, + final AnyObjectId objectId) throws IOException { + long sz; + + sz = getObjectSize1(curs, objectId); + if (0 <= sz) + return sz; + + for (final AlternateHandle alt : myAlternates()) { + sz = alt.db.getObjectSizeImpl1(curs, objectId); + if (0 <= sz) + return sz; + } + + if (tryAgain1()) { + sz = getObjectSize1(curs, objectId); + if (0 <= sz) + return sz; + } + + return -1; + } + + final long getObjectSizeImpl2(final WindowCursor curs, + final String objectName, final AnyObjectId objectId) + throws IOException { + long sz; + + sz = getObjectSize2(curs, objectName, objectId); + if (0 <= sz) + return sz; + + for (final AlternateHandle alt : myAlternates()) { + sz = alt.db.getObjectSizeImpl2(curs, objectName, objectId); + if (0 <= sz) + return sz; + } + + return -1; + } + abstract void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException; @@ -185,6 +234,12 @@ abstract class FileObjectDatabase extends ObjectDatabase { abstract ObjectLoader openObject2(WindowCursor curs, String objectName, AnyObjectId objectId) throws IOException; + abstract long getObjectSize1(WindowCursor curs, AnyObjectId objectId) + throws IOException; + + abstract long getObjectSize2(WindowCursor curs, String objectName, + AnyObjectId objectId) throws IOException; + abstract FileObjectDatabase newCachedFileObjectDatabase(); abstract int getStreamFileThreshold(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java index 8177155f42..6fe4fd754e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java @@ -304,6 +304,46 @@ public class ObjectDirectory extends FileObjectDatabase implements } } + long getObjectSize1(final WindowCursor curs, final AnyObjectId objectId) + throws IOException { + PackList pList = packList.get(); + SEARCH: for (;;) { + for (final PackFile p : pList.packs) { + try { + long sz = p.getObjectSize(curs, objectId); + if (0 <= sz) + return sz; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + // + pList = scanPacks(pList); + continue SEARCH; + } catch (IOException e) { + // Assume the pack is corrupted. + // + removePack(p); + } + } + return -1; + } + } + + @Override + long getObjectSize2(WindowCursor curs, String objectName, + AnyObjectId objectId) throws IOException { + try { + File path = fileFor(objectName); + FileInputStream in = new FileInputStream(path); + try { + return UnpackedObject.getSize(in, objectId, curs); + } finally { + in.close(); + } + } catch (FileNotFoundException noFile) { + return -1; + } + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java index 40bb07100c..e74a7c0142 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java @@ -779,6 +779,59 @@ public class PackFile implements Iterable { } } + long getObjectSize(final WindowCursor curs, final AnyObjectId id) + throws IOException { + final long offset = idx().findOffset(id); + return 0 < offset ? getObjectSize(curs, offset) : -1; + } + + long getObjectSize(final WindowCursor curs, final long pos) + throws IOException { + final byte[] ib = curs.tempId; + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + final int type = (c >> 4) & 7; + long sz = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = ib[p++] & 0xff; + sz += (c & 0x7f) << shift; + shift += 7; + } + + long deltaAt; + switch (type) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + return sz; + + case Constants.OBJ_OFS_DELTA: + c = ib[p++] & 0xff; + while ((c & 128) != 0) + c = ib[p++] & 0xff; + deltaAt = pos + p; + break; + + case Constants.OBJ_REF_DELTA: + deltaAt = pos + p + 20; + break; + + default: + throw new IOException(MessageFormat.format( + JGitText.get().unknownObjectType, type)); + } + + try { + return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt)); + } catch (DataFormatException e) { + throw new CorruptObjectException(MessageFormat.format(JGitText + .get().objectAtHasBadZlibStream, pos, getPackFile())); + } + } + LocalObjectRepresentation representation(final WindowCursor curs, final AnyObjectId objectId) throws IOException { final long pos = idx().findOffset(objectId); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java index 0325c57118..59f9c82670 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java @@ -109,7 +109,7 @@ public class UnpackedObject { if (isStandardFormat(hdr)) { in.reset(); - Inflater inf = wc.inflater(); + Inflater inf = wc.inflater(); InputStream zIn = inflate(in, inf); int avail = readSome(zIn, hdr, 0, 64); if (avail < 5) @@ -183,6 +183,50 @@ public class UnpackedObject { } } + static long getSize(InputStream in, AnyObjectId id, WindowCursor wc) + throws IOException { + try { + in = buffer(in); + in.mark(20); + final byte[] hdr = new byte[64]; + IO.readFully(in, hdr, 0, 2); + + if (isStandardFormat(hdr)) { + in.reset(); + Inflater inf = wc.inflater(); + InputStream zIn = inflate(in, inf); + int avail = readSome(zIn, hdr, 0, 64); + if (avail < 5) + throw new CorruptObjectException(id, + JGitText.get().corruptObjectNoHeader); + + final MutableInteger p = new MutableInteger(); + Constants.decodeTypeString(id, hdr, (byte) ' ', p); + long size = RawParseUtils.parseLongBase10(hdr, p.value, p); + if (size < 0) + throw new CorruptObjectException(id, + JGitText.get().corruptObjectNegativeSize); + return size; + + } else { + readSome(in, hdr, 2, 18); + int c = hdr[0] & 0xff; + long size = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = hdr[p++] & 0xff; + size += (c & 0x7f) << shift; + shift += 7; + } + return size; + } + } catch (ZipException badStream) { + throw new CorruptObjectException(id, + JGitText.get().corruptObjectBadStream); + } + } + private static void checkValidEndOfStream(InputStream in, Inflater inf, AnyObjectId id, final byte[] buf) throws IOException, CorruptObjectException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java index 6f4e72a82f..04ee8b2c49 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java @@ -95,6 +95,18 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { return ldr; } + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + long sz = db.getObjectSize(this, objectId); + if (sz < 0) { + if (typeHint == OBJ_ANY) + throw new MissingObjectException(objectId.copy(), "unknown"); + throw new MissingObjectException(objectId.copy(), typeHint); + } + return sz; + } + public LocalObjectToPack newObjectToPack(RevObject obj) { return new LocalObjectToPack(obj); }