]> source.dussan.org Git - jgit.git/commitdiff
Tighten up local packed object representation during packing 71/971/1
authorShawn O. Pearce <spearce@spearce.org>
Sun, 27 Jun 2010 00:37:16 +0000 (17:37 -0700)
committerShawn O. Pearce <spearce@spearce.org>
Sun, 27 Jun 2010 01:50:38 +0000 (18:50 -0700)
Rather than making a loader, and then using that to fill the object
representation, parse the header and set up our data directly.
This saves some time, as we don't waste cycles on information we
won't use right now.

The weight computed for a representation is now its actual stored
size in the pack file, rather than its inflated size.  This accounts
for changes made when the compression level is modified on the
repository.  It is however more costly to determine the weight of
the object, since we have to find its length in the pack.  To try and
recover that cost we now cache the length as part of our ObjectToPack
record, so it doesn't have to be found during the output phase.

A LocalObjectToPack now costs us (assuming 32 bit pointers):

                   (32 bit)     (64 bit)
  vm header:         8 bytes      8 bytes
  ObjectId:         20 bytes     20 bytes
  PackedObjectInfo: 12 bytes     12 bytes
  ObjectToPack:      8 bytes     12 bytes
  LocalOTP:         20 bytes     24 bytes
                 -----------    ---------
                    68 bytes     74 bytes

Change-Id: I923d2736186eb2ac8ab498d3eb137e17930fcb50
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalObjectRepresentation.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalObjectToPack.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java

index 4949b507eb7166cd8f12070215ba5fc52f963ae4..6ddd66e6d32c8d79a8d0ca716d932ccd15a42690 100644 (file)
@@ -49,35 +49,70 @@ import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.StoredObjectRepresentation;
 
 class LocalObjectRepresentation extends StoredObjectRepresentation {
-       final PackedObjectLoader ldr;
+       static LocalObjectRepresentation newWhole(PackFile f, long p, long length) {
+               LocalObjectRepresentation r = new LocalObjectRepresentation() {
+                       @Override
+                       public int getFormat() {
+                               return PACK_WHOLE;
+                       }
+               };
+               r.pack = f;
+               r.offset = p;
+               r.length = length;
+               return r;
+       }
 
-       LocalObjectRepresentation(PackedObjectLoader ldr) {
-               this.ldr = ldr;
+       static LocalObjectRepresentation newDelta(PackFile f, long p, long n,
+                       ObjectId base) {
+               LocalObjectRepresentation r = new Delta();
+               r.pack = f;
+               r.offset = p;
+               r.length = n;
+               r.baseId = base;
+               return r;
        }
 
-       @Override
-       public int getFormat() {
-               if (ldr instanceof DeltaPackedObjectLoader)
-                       return PACK_DELTA;
-               if (ldr instanceof WholePackedObjectLoader)
-                       return PACK_WHOLE;
-               return FORMAT_OTHER;
+       static LocalObjectRepresentation newDelta(PackFile f, long p, long n,
+                       long base) {
+               LocalObjectRepresentation r = new Delta();
+               r.pack = f;
+               r.offset = p;
+               r.length = n;
+               r.baseOffset = base;
+               return r;
        }
 
+       PackFile pack;
+
+       long offset;
+
+       long length;
+
+       private long baseOffset;
+
+       private ObjectId baseId;
+
        @Override
        public int getWeight() {
-               long sz = ldr.getRawSize();
-               if (Integer.MAX_VALUE < sz)
-                       return WEIGHT_UNKNOWN;
-               return (int) sz;
+               return (int) Math.min(length, Integer.MAX_VALUE);
        }
 
        @Override
        public ObjectId getDeltaBase() {
-               try {
-                       return ldr.getDeltaBase();
-               } catch (IOException e) {
-                       return null;
+               if (baseId == null && getFormat() == PACK_DELTA) {
+                       try {
+                               baseId = pack.findObjectForOffset(baseOffset);
+                       } catch (IOException error) {
+                               return null;
+                       }
+               }
+               return baseId;
+       }
+
+       private static final class Delta extends LocalObjectRepresentation {
+               @Override
+               public int getFormat() {
+                       return PACK_DELTA;
                }
        }
 }
index 9ee43f398d4551923141439fd9fd0142215d7d60..e1b25426778539e179d952eff83b04a5e6f02245 100644 (file)
@@ -50,10 +50,13 @@ import org.eclipse.jgit.revwalk.RevObject;
 /** {@link ObjectToPack} for {@link ObjectDirectory}. */
 class LocalObjectToPack extends ObjectToPack {
        /** Pack to reuse compressed data from, otherwise null. */
-       PackFile copyFromPack;
+       PackFile pack;
 
-       /** Offset of the object's header in {@link #copyFromPack}. */
-       long copyOffset;
+       /** Offset of the object's header in {@link #pack}. */
+       long offset;
+
+       /** Length of the data section of the object. */
+       long length;
 
        LocalObjectToPack(RevObject obj) {
                super(obj);
@@ -61,8 +64,9 @@ class LocalObjectToPack extends ObjectToPack {
 
        @Override
        public void select(StoredObjectRepresentation ref) {
-               LocalObjectRepresentation ptr = (LocalObjectRepresentation)ref;
-               this.copyFromPack = ptr.ldr.pack;
-               this.copyOffset = ptr.ldr.objectOffset;
+               LocalObjectRepresentation ptr = (LocalObjectRepresentation) ref;
+               this.pack = ptr.pack;
+               this.offset = ptr.offset;
+               this.length = ptr.length;
        }
 }
index 8a8055605a648d8bd3c481a5cc77391495ff6a1b..8db258bb4351f7a728995abfeaef1bd29fa46f93 100644 (file)
@@ -299,9 +299,9 @@ public class ObjectDirectory extends FileObjectDatabase {
                SEARCH: for (;;) {
                        for (final PackFile p : pList.packs) {
                                try {
-                                       PackedObjectLoader ldr = p.get(curs, otp);
-                                       if (ldr != null)
-                                               packer.select(otp, new LocalObjectRepresentation(ldr));
+                                       LocalObjectRepresentation rep = p.representation(curs, otp);
+                                       if (rep != null)
+                                               packer.select(otp, rep);
                                } catch (PackMismatchException e) {
                                        // Pack was modified; refresh the entire pack list.
                                        //
index e5f6f03f409188991685b8e4983d038c6bd289bb..01db79dcd45e028d027e40f0d6bbaaa7c86e98e5 100644 (file)
@@ -309,7 +309,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
                // Rip apart the header so we can discover the size.
                //
-               readFully(src.copyOffset, buf, 0, 20, curs);
+               readFully(src.offset, buf, 0, 20, curs);
                int c = buf[0] & 0xff;
                final int typeCode = (c >> 4) & 7;
                long inflatedLength = c & 15;
@@ -331,7 +331,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                        crc1.update(buf, 0, headerCnt);
                        crc2.update(buf, 0, headerCnt);
 
-                       readFully(src.copyOffset + headerCnt, buf, 0, 20, curs);
+                       readFully(src.offset + headerCnt, buf, 0, 20, curs);
                        crc1.update(buf, 0, 20);
                        crc2.update(buf, 0, headerCnt);
                        headerCnt += 20;
@@ -340,8 +340,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                        crc2.update(buf, 0, headerCnt);
                }
 
-               final long dataOffset = src.copyOffset + headerCnt;
-               final long dataLength;
+               final long dataOffset = src.offset + headerCnt;
+               final long dataLength = src.length;
                final long expectedCRC;
                final ByteArrayWindow quickCopy;
 
@@ -349,7 +349,6 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                // we report it missing instead.
                //
                try {
-                       dataLength = findEndOffset(src.copyOffset) - dataOffset;
                        quickCopy = curs.quickCopy(this, dataOffset, dataLength);
 
                        if (idx().hasCRC32Support()) {
@@ -370,10 +369,10 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                                        }
                                }
                                if (crc1.getValue() != expectedCRC) {
-                                       setCorrupt(src.copyOffset);
+                                       setCorrupt(src.offset);
                                        throw new CorruptObjectException(MessageFormat.format(
                                                        JGitText.get().objectAtHasBadZlibStream,
-                                                       src.copyOffset, getPackFile()));
+                                                       src.offset, getPackFile()));
                                }
                        } else {
                                // We don't have a CRC32 code in the index, so compute it
@@ -399,20 +398,20 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                                        }
                                }
                                if (!inf.finished() || inf.getBytesRead() != dataLength) {
-                                       setCorrupt(src.copyOffset);
+                                       setCorrupt(src.offset);
                                        throw new EOFException(MessageFormat.format(
                                                        JGitText.get().shortCompressedStreamAt,
-                                                       src.copyOffset));
+                                                       src.offset));
                                }
                                expectedCRC = crc1.getValue();
                        }
                } catch (DataFormatException dataFormat) {
-                       setCorrupt(src.copyOffset);
+                       setCorrupt(src.offset);
 
                        CorruptObjectException corruptObject = new CorruptObjectException(
                                        MessageFormat.format(
                                                        JGitText.get().objectAtHasBadZlibStream,
-                                                       src.copyOffset, getPackFile()));
+                                                       src.offset, getPackFile()));
                        corruptObject.initCause(dataFormat);
 
                        StoredObjectRepresentationNotAvailableException gone;
@@ -458,7 +457,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                        }
                        if (crc2.getValue() != expectedCRC) {
                                throw new CorruptObjectException(MessageFormat.format(JGitText
-                                               .get().objectAtHasBadZlibStream, src.copyOffset,
+                                               .get().objectAtHasBadZlibStream, src.offset,
                                                getPackFile()));
                        }
                }
@@ -661,6 +660,55 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
                }
        }
 
+       LocalObjectRepresentation representation(final WindowCursor curs,
+                       final AnyObjectId objectId) throws IOException {
+               final long pos = idx().findOffset(objectId);
+               if (pos < 0)
+                       return null;
+
+               final byte[] ib = curs.tempId;
+               readFully(pos, ib, 0, 20, curs);
+               int c = ib[0] & 0xff;
+               int p = 1;
+               final int typeCode = (c >> 4) & 7;
+               while ((c & 0x80) != 0)
+                       c = ib[p++] & 0xff;
+
+               long len = (findEndOffset(pos) - pos);
+               switch (typeCode) {
+               case Constants.OBJ_COMMIT:
+               case Constants.OBJ_TREE:
+               case Constants.OBJ_BLOB:
+               case Constants.OBJ_TAG:
+                       return LocalObjectRepresentation.newWhole(this, pos, len - p);
+
+               case Constants.OBJ_OFS_DELTA: {
+                       c = ib[p++] & 0xff;
+                       long ofs = c & 127;
+                       while ((c & 128) != 0) {
+                               ofs += 1;
+                               c = ib[p++] & 0xff;
+                               ofs <<= 7;
+                               ofs += (c & 127);
+                       }
+                       ofs = pos - ofs;
+                       return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
+               }
+
+               case Constants.OBJ_REF_DELTA: {
+                       len -= p;
+                       len -= Constants.OBJECT_ID_LENGTH;
+                       readFully(pos + p, ib, 0, 20, curs);
+                       ObjectId id = ObjectId.fromRaw(ib);
+                       return LocalObjectRepresentation.newDelta(this, pos, len, id);
+               }
+
+               default:
+                       throw new IOException(MessageFormat.format(
+                                       JGitText.get().unknownObjectType, typeCode));
+               }
+       }
+
        private long findEndOffset(final long startOffset)
                        throws IOException, CorruptObjectException {
                final long maxOffset = length - 20;
index a88261162d275258af61ac863197fca725e9a29f..d359de07e9bd608910f874293dde89edbc31623f 100644 (file)
@@ -103,7 +103,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs {
        public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp)
                        throws IOException, StoredObjectRepresentationNotAvailableException {
                LocalObjectToPack src = (LocalObjectToPack) otp;
-               src.copyFromPack.copyAsIs(out, src, this);
+               src.pack.copyAsIs(out, src, this);
        }
 
        /**