diff options
12 files changed, 512 insertions, 294 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/StoredObjectRepresentationNotAvailableException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/StoredObjectRepresentationNotAvailableException.java new file mode 100644 index 0000000000..cff4499277 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/StoredObjectRepresentationNotAvailableException.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2010, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.errors; + +import org.eclipse.jgit.lib.ObjectToPack; + +/** A previously selected representation is no longer available. */ +public class StoredObjectRepresentationNotAvailableException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Construct an error for an object. + * + * @param otp + * the object whose current representation is no longer present. + */ + public StoredObjectRepresentationNotAvailableException(ObjectToPack otp) { + // Do nothing. + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteArrayWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteArrayWindow.java index 8042610310..4f2373d3d2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteArrayWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteArrayWindow.java @@ -80,18 +80,4 @@ final class ByteArrayWindow extends ByteWindow { o += inf.inflate(b, o, b.length - o); return o; } - - @Override - protected void inflateVerify(final int pos, final Inflater inf) - throws DataFormatException { - while (!inf.finished()) { - if (inf.needsInput()) { - inf.setInput(array, pos, array.length - pos); - break; - } - inf.inflate(verifyGarbageBuffer, 0, verifyGarbageBuffer.length); - } - while (!inf.finished() && !inf.needsInput()) - inf.inflate(verifyGarbageBuffer, 0, verifyGarbageBuffer.length); - } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteBufferWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteBufferWindow.java index 1b29934d28..794d7428e6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteBufferWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteBufferWindow.java @@ -89,22 +89,4 @@ final class ByteBufferWindow extends ByteWindow { o += inf.inflate(b, o, b.length - o); return o; } - - @Override - protected void inflateVerify(final int pos, final Inflater inf) - throws DataFormatException { - final byte[] tmp = new byte[512]; - final ByteBuffer s = buffer.slice(); - s.position(pos); - while (s.remaining() > 0 && !inf.finished()) { - if (inf.needsInput()) { - final int n = Math.min(s.remaining(), tmp.length); - s.get(tmp, 0, n); - inf.setInput(tmp, 0, n); - } - inf.inflate(verifyGarbageBuffer, 0, verifyGarbageBuffer.length); - } - while (!inf.finished() && !inf.needsInput()) - inf.inflate(verifyGarbageBuffer, 0, verifyGarbageBuffer.length); - } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteWindow.java index cbef4218af..69d255c781 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ByteWindow.java @@ -172,14 +172,4 @@ abstract class ByteWindow { */ protected abstract int inflate(int pos, byte[] dstbuf, int dstoff, Inflater inf) throws DataFormatException; - - protected static final byte[] verifyGarbageBuffer = new byte[2048]; - - final void inflateVerify(final long pos, final Inflater inf) - throws DataFormatException { - inflateVerify((int) (pos - start), inf); - } - - protected abstract void inflateVerify(int pos, Inflater inf) - throws DataFormatException; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/LocalObjectToPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/LocalObjectToPack.java index 516cf631c2..8db58707e8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/LocalObjectToPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/LocalObjectToPack.java @@ -43,26 +43,20 @@ package org.eclipse.jgit.lib; -import java.io.IOException; - import org.eclipse.jgit.revwalk.RevObject; /** {@link ObjectToPack} for {@link ObjectDirectory}. */ class LocalObjectToPack extends ObjectToPack { /** Pack to reuse compressed data from, otherwise null. */ - private PackFile copyFromPack; + PackFile copyFromPack; /** Offset of the object's header in {@link #copyFromPack}. */ - private long copyOffset; + long copyOffset; LocalObjectToPack(RevObject obj) { super(obj); } - PackedObjectLoader getCopyLoader(WindowCursor curs) throws IOException { - return copyFromPack.resolveBase(curs, copyOffset); - } - @Override public void select(StoredObjectRepresentation ref) { LocalObjectRepresentation ptr = (LocalObjectRepresentation)ref; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReuseAsIs.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReuseAsIs.java index f7aebf1241..f87b8301c1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReuseAsIs.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReuseAsIs.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.lib; import java.io.IOException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.revwalk.RevObject; /** @@ -95,4 +96,40 @@ public interface ObjectReuseAsIs { */ public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp) throws IOException, MissingObjectException; + + /** + * Output a previously selected representation. + * <p> + * {@code PackWriter} invokes this method only if a representation + * previously given to it by {@code selectObjectRepresentation} was chosen + * for reuse into the output stream. The {@code otp} argument is an instance + * created by this reader's own {@code newObjectToPack}, and the + * representation data saved within it also originated from this reader. + * <p> + * Implementors must write the object header before copying the raw data to + * the output stream. The typical implementation is like: + * + * <pre> + * MyToPack mtp = (MyToPack) otp; + * byte[] raw = validate(mtp); // throw SORNAE here, if at all + * out.writeHeader(mtp, mtp.inflatedSize); + * out.write(raw); + * </pre> + * + * @param out + * stream the object should be written to. + * @param otp + * the object's saved representation information. + * @throws StoredObjectRepresentationNotAvailableException + * the previously selected representation is no longer + * available. If thrown before {@code out.writeHeader} the pack + * writer will try to find another representation, and write + * that one instead. If throw after {@code out.writeHeader}, + * packing will abort. + * @throws IOException + * the stream's write method threw an exception. Packing will + * abort. + */ + public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp) + throws IOException, StoredObjectRepresentationNotAvailableException; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackFile.java index 829832e6a5..25835e2ff2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackFile.java @@ -48,7 +48,6 @@ package org.eclipse.jgit.lib; import java.io.EOFException; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel.MapMode; @@ -58,13 +57,15 @@ import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.zip.CRC32; -import java.util.zip.CheckedOutputStream; import java.util.zip.DataFormatException; +import java.util.zip.Inflater; import org.eclipse.jgit.JGitText; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; +import org.eclipse.jgit.util.LongList; import org.eclipse.jgit.util.NB; import org.eclipse.jgit.util.RawParseUtils; @@ -109,6 +110,15 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { private PackReverseIndex reverseIdx; /** + * Objects we have tried to read, and discovered to be corrupt. + * <p> + * The list is allocated after the first corruption is found, and filled in + * as more entries are discovered. Typically this list is never used, as + * pack files do not usually contain corrupt objects. + */ + private volatile LongList corruptObjects; + + /** * Construct a reader for an existing, pre-indexed packfile. * * @param idxFile @@ -152,6 +162,10 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { final PackedObjectLoader resolveBase(final WindowCursor curs, final long ofs) throws IOException { + if (isCorrupt(ofs)) { + throw new CorruptObjectException(MessageFormat.format(JGitText + .get().objectAtHasBadZlibStream, ofs, getPackFile())); + } return reader(curs, ofs); } @@ -174,7 +188,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { * the index file cannot be loaded into memory. */ public boolean hasObject(final AnyObjectId id) throws IOException { - return idx().hasObject(id); + final long offset = idx().findOffset(id); + return 0 < offset && !isCorrupt(offset); } /** @@ -192,7 +207,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id) throws IOException { final long offset = idx().findOffset(id); - return 0 < offset ? reader(curs, offset) : null; + return 0 < offset && !isCorrupt(offset) ? reader(curs, offset) : null; } /** @@ -269,48 +284,163 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { return dstbuf; } - final void copyRawData(final PackedObjectLoader loader, - final OutputStream out, final byte buf[], final WindowCursor curs) - throws IOException { - final long objectOffset = loader.objectOffset; - final long dataOffset = objectOffset + loader.headerSize; - final long sz = findEndOffset(objectOffset) - dataOffset; - final PackIndex idx = idx(); + final void copyAsIs(PackOutputStream out, LocalObjectToPack src, + WindowCursor curs) throws IOException, + StoredObjectRepresentationNotAvailableException { + beginCopyAsIs(src); + try { + copyAsIs2(out, src, curs); + } finally { + endCopyAsIs(); + } + } - if (idx.hasCRC32Support()) { - final CRC32 crc = new CRC32(); - int headerCnt = loader.headerSize; - while (headerCnt > 0) { - final int toRead = Math.min(headerCnt, buf.length); - readFully(objectOffset, buf, 0, toRead, curs); - crc.update(buf, 0, toRead); - headerCnt -= toRead; - } - final CheckedOutputStream crcOut = new CheckedOutputStream(out, crc); - copyToStream(dataOffset, buf, sz, crcOut, curs); - final long computed = crc.getValue(); - - final ObjectId id = findObjectForOffset(objectOffset); - final long expected = idx.findCRC32(id); - if (computed != expected) - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, objectOffset, getPackFile())); + private void copyAsIs2(PackOutputStream out, LocalObjectToPack src, + WindowCursor curs) throws IOException, + StoredObjectRepresentationNotAvailableException { + final CRC32 crc1 = new CRC32(); + final CRC32 crc2 = new CRC32(); + final byte[] buf = out.getCopyBuffer(); + + // Rip apart the header so we can discover the size. + // + readFully(src.copyOffset, buf, 0, 20, curs); + int c = buf[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long inflatedLength = c & 15; + int shift = 4; + int headerCnt = 1; + while ((c & 0x80) != 0) { + c = buf[headerCnt++] & 0xff; + inflatedLength += (c & 0x7f) << shift; + shift += 7; + } + + if (typeCode == Constants.OBJ_OFS_DELTA) { + do { + c = buf[headerCnt++] & 0xff; + } while ((c & 128) != 0); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } else if (typeCode == Constants.OBJ_REF_DELTA) { + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + + readFully(src.copyOffset + headerCnt, buf, 0, 20, curs); + crc1.update(buf, 0, 20); + crc2.update(buf, 0, headerCnt); + headerCnt += 20; } else { - try { - curs.inflateVerify(this, dataOffset); - } catch (DataFormatException dfe) { - final CorruptObjectException coe; - coe = new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, objectOffset, getPackFile())); - coe.initCause(dfe); - throw coe; + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + + final long dataOffset = src.copyOffset + headerCnt; + final long dataLength; + final long expectedCRC; + + // Verify the object isn't corrupt before sending. If it is, + // we report it missing instead. + // + try { + dataLength = findEndOffset(src.copyOffset) - dataOffset; + + if (idx().hasCRC32Support()) { + // Index has the CRC32 code cached, validate the object. + // + expectedCRC = idx().findCRC32(src); + + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + pos += n; + cnt -= n; + } + if (crc1.getValue() != expectedCRC) { + setCorrupt(src.copyOffset); + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + src.copyOffset, getPackFile())); + } + } else { + // We don't have a CRC32 code in the index, so compute it + // now while inflating the raw data to get zlib to tell us + // whether or not the data is safe. + // + long pos = dataOffset; + long cnt = dataLength; + Inflater inf = curs.inflater(); + byte[] tmp = new byte[1024]; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + inf.setInput(buf, 0, n); + while (inf.inflate(tmp, 0, tmp.length) > 0) + continue; + pos += n; + cnt -= n; + } + if (!inf.finished()) { + setCorrupt(src.copyOffset); + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + src.copyOffset)); + } + expectedCRC = crc1.getValue(); } - copyToStream(dataOffset, buf, sz, out, curs); + } catch (DataFormatException dataFormat) { + setCorrupt(src.copyOffset); + + CorruptObjectException corruptObject = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + src.copyOffset, getPackFile())); + corruptObject.initCause(dataFormat); + + StoredObjectRepresentationNotAvailableException gone; + gone = new StoredObjectRepresentationNotAvailableException(src); + gone.initCause(corruptObject); + throw gone; + + } catch (IOException ioError) { + StoredObjectRepresentationNotAvailableException gone; + gone = new StoredObjectRepresentationNotAvailableException(src); + gone.initCause(ioError); + throw gone; } - } - boolean supportsFastCopyRawData() throws IOException { - return idx().hasCRC32Support(); + if (dataLength <= buf.length) { + // Tiny optimization: Lots of objects are very small deltas or + // deflated commits that are likely to fit in the copy buffer. + // + out.writeHeader(src, inflatedLength); + out.write(buf, 0, (int) dataLength); + } else { + // Now we are committed to sending the object. As we spool it out, + // check its CRC32 code to make sure there wasn't corruption between + // the verification we did above, and us actually outputting it. + // + out.writeHeader(src, inflatedLength); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc2.update(buf, 0, n); + out.write(buf, 0, n); + pos += n; + cnt -= n; + } + if (crc2.getValue() != expectedCRC) { + throw new CorruptObjectException(MessageFormat.format(JGitText + .get().objectAtHasBadZlibStream, src.copyOffset, + getPackFile())); + } + } } boolean invalid() { @@ -324,24 +454,22 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { throw new EOFException(); } - private void copyToStream(long position, final byte[] buf, long cnt, - final OutputStream out, final WindowCursor curs) - throws IOException, EOFException { - while (cnt > 0) { - final int toRead = (int) Math.min(cnt, buf.length); - readFully(position, buf, 0, toRead, curs); - position += toRead; - cnt -= toRead; - out.write(buf, 0, toRead); - } - } + private synchronized void beginCopyAsIs(ObjectToPack otp) + throws StoredObjectRepresentationNotAvailableException { + if (++activeCopyRawData == 1 && activeWindows == 0) { + try { + doOpen(); + } catch (IOException thisPackNotValid) { + StoredObjectRepresentationNotAvailableException gone; - synchronized void beginCopyRawData() throws IOException { - if (++activeCopyRawData == 1 && activeWindows == 0) - doOpen(); + gone = new StoredObjectRepresentationNotAvailableException(otp); + gone.initCause(thisPackNotValid); + throw gone; + } + } } - synchronized void endCopyRawData() { + private synchronized void endCopyAsIs() { if (--activeCopyRawData == 0 && activeWindows == 0) doClose(); } @@ -523,4 +651,29 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { reverseIdx = new PackReverseIndex(idx()); return reverseIdx; } + + private boolean isCorrupt(long offset) { + LongList list = corruptObjects; + if (list == null) + return false; + synchronized (list) { + return list.contains(offset); + } + } + + private void setCorrupt(long offset) { + LongList list = corruptObjects; + if (list == null) { + synchronized (readLock) { + list = corruptObjects; + if (list == null) { + list = new LongList(); + corruptObjects = list; + } + } + } + synchronized (list) { + list.add(offset); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackOutputStream.java index a348f1e547..48ec2b5a17 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackOutputStream.java @@ -49,35 +49,44 @@ import java.io.OutputStream; import java.security.MessageDigest; import java.util.zip.CRC32; +import org.eclipse.jgit.util.NB; + /** Custom output stream to support {@link PackWriter}. */ -final class PackOutputStream extends OutputStream { +public final class PackOutputStream extends OutputStream { private final OutputStream out; + private final boolean ofsDelta; + private final CRC32 crc = new CRC32(); private final MessageDigest md = Constants.newMessageDigest(); private long count; - PackOutputStream(final OutputStream out) { + private byte[] headerBuffer = new byte[32]; + + private byte[] copyBuffer; + + PackOutputStream(final OutputStream out, final boolean ofsDelta) { this.out = out; + this.ofsDelta = ofsDelta; } @Override public void write(final int b) throws IOException { + count++; out.write(b); crc.update(b); md.update((byte) b); - count++; } @Override public void write(final byte[] b, final int off, final int len) throws IOException { + count += len; out.write(b, off, len); crc.update(b, off, len); md.update(b, off, len); - count += len; } @Override @@ -85,6 +94,79 @@ final class PackOutputStream extends OutputStream { out.flush(); } + void writeFileHeader(int version, int objectCount) throws IOException { + System.arraycopy(Constants.PACK_SIGNATURE, 0, headerBuffer, 0, 4); + NB.encodeInt32(headerBuffer, 4, version); + NB.encodeInt32(headerBuffer, 8, objectCount); + write(headerBuffer, 0, 12); + } + + /** + * Commits the object header onto the stream. + * <p> + * Once the header has been written, the object representation must be fully + * output, or packing must abort abnormally. + * + * @param otp + * the object to pack. Header information is obtained. + * @param rawLength + * number of bytes of the inflated content. For an object that is + * in whole object format, this is the same as the object size. + * For an object that is in a delta format, this is the size of + * the inflated delta instruction stream. + * @throws IOException + * the underlying stream refused to accept the header. + */ + public void writeHeader(ObjectToPack otp, long rawLength) + throws IOException { + if (otp.isDeltaRepresentation()) { + if (ofsDelta) { + ObjectToPack baseInPack = otp.getDeltaBase(); + if (baseInPack != null && baseInPack.isWritten()) { + final long start = count; + int n = encodeTypeSize(Constants.OBJ_OFS_DELTA, rawLength); + write(headerBuffer, 0, n); + + long offsetDiff = start - baseInPack.getOffset(); + n = headerBuffer.length - 1; + headerBuffer[n] = (byte) (offsetDiff & 0x7F); + while ((offsetDiff >>= 7) > 0) + headerBuffer[--n] = (byte) (0x80 | (--offsetDiff & 0x7F)); + write(headerBuffer, n, headerBuffer.length - n); + return; + } + } + + int n = encodeTypeSize(Constants.OBJ_REF_DELTA, rawLength); + otp.getDeltaBaseId().copyRawTo(headerBuffer, n); + write(headerBuffer, 0, n + Constants.OBJECT_ID_LENGTH); + } else { + int n = encodeTypeSize(otp.getType(), rawLength); + write(headerBuffer, 0, n); + } + } + + private int encodeTypeSize(int type, long rawLength) { + long nextLength = rawLength >>> 4; + headerBuffer[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) + | (type << 4) | (rawLength & 0x0F)); + rawLength = nextLength; + int n = 1; + while (rawLength > 0) { + nextLength >>>= 7; + headerBuffer[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F)); + rawLength = nextLength; + } + return n; + } + + /** @return a temporary buffer writers can use to copy data with. */ + public byte[] getCopyBuffer() { + if (copyBuffer == null) + copyBuffer = new byte[16 * 1024]; + return copyBuffer; + } + /** @return total number of bytes written since stream start. */ long length() { return count; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackWriter.java index 462b12f39d..80d8fff536 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackWriter.java @@ -58,13 +58,14 @@ import java.util.List; import java.util.zip.Deflater; import org.eclipse.jgit.JGitText; +import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevFlag; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevSort; -import org.eclipse.jgit.util.NB; /** * <p> @@ -155,13 +156,13 @@ public class PackWriter { private static final int PACK_VERSION_GENERATED = 2; @SuppressWarnings("unchecked") - private final List<LocalObjectToPack> objectsLists[] = new List[Constants.OBJ_TAG + 1]; + private final List<ObjectToPack> objectsLists[] = new List[Constants.OBJ_TAG + 1]; { - objectsLists[0] = Collections.<LocalObjectToPack> emptyList(); - objectsLists[Constants.OBJ_COMMIT] = new ArrayList<LocalObjectToPack>(); - objectsLists[Constants.OBJ_TREE] = new ArrayList<LocalObjectToPack>(); - objectsLists[Constants.OBJ_BLOB] = new ArrayList<LocalObjectToPack>(); - objectsLists[Constants.OBJ_TAG] = new ArrayList<LocalObjectToPack>(); + objectsLists[0] = Collections.<ObjectToPack> emptyList(); + objectsLists[Constants.OBJ_COMMIT] = new ArrayList<ObjectToPack>(); + objectsLists[Constants.OBJ_TREE] = new ArrayList<ObjectToPack>(); + objectsLists[Constants.OBJ_BLOB] = new ArrayList<ObjectToPack>(); + objectsLists[Constants.OBJ_TAG] = new ArrayList<ObjectToPack>(); } private final ObjectIdSubclassMap<ObjectToPack> objectsMap = new ObjectIdSubclassMap<ObjectToPack>(); @@ -179,9 +180,10 @@ public class PackWriter { private ProgressMonitor writeMonitor; - private final byte[] buf = new byte[16384]; // 16 KB + private final ObjectReader reader; - private final WindowCursor windowCursor; + /** {@link #reader} recast to the reuse interface, if it supports it. */ + private final ObjectReuseAsIs reuseSupport; private List<ObjectToPack> sortedByName; @@ -238,8 +240,12 @@ public class PackWriter { public PackWriter(final Repository repo, final ProgressMonitor imonitor, final ProgressMonitor wmonitor) { this.db = repo; - windowCursor = new WindowCursor((ObjectDirectory) repo - .getObjectDatabase()); + + reader = db.newObjectReader(); + if (reader instanceof ObjectReuseAsIs) + reuseSupport = ((ObjectReuseAsIs) reader); + else + reuseSupport = null; initMonitor = imonitor == null ? NullProgressMonitor.INSTANCE : imonitor; writeMonitor = wmonitor == null ? NullProgressMonitor.INSTANCE : wmonitor; @@ -525,6 +531,7 @@ public class PackWriter { * @return ObjectId representing SHA-1 name of a pack that was created. */ public ObjectId computeName() { + final byte[] buf = new byte[Constants.OBJECT_ID_LENGTH]; final MessageDigest md = Constants.newMessageDigest(); for (ObjectToPack otp : sortByName()) { otp.copyRawTo(buf, 0); @@ -560,8 +567,8 @@ public class PackWriter { private List<ObjectToPack> sortByName() { if (sortedByName == null) { sortedByName = new ArrayList<ObjectToPack>(objectsMap.size()); - for (List<LocalObjectToPack> list : objectsLists) { - for (LocalObjectToPack otp : list) + for (List<ObjectToPack> list : objectsLists) { + for (ObjectToPack otp : list) sortedByName.add(otp); } Collections.sort(sortedByName); @@ -592,44 +599,38 @@ public class PackWriter { * stream. */ public void writePack(OutputStream packStream) throws IOException { - if (reuseDeltas || reuseObjects) + if ((reuseDeltas || reuseObjects) && reuseSupport != null) searchForReuse(); - out = new PackOutputStream(packStream); + out = new PackOutputStream(packStream, isDeltaBaseAsOffset()); writeMonitor.beginTask(WRITING_OBJECTS_PROGRESS, getObjectsNumber()); - writeHeader(); + out.writeFileHeader(PACK_VERSION_GENERATED, getObjectsNumber()); writeObjects(); writeChecksum(); - windowCursor.release(); + out = null; + reader.release(); writeMonitor.endTask(); } private void searchForReuse() throws IOException { initMonitor.beginTask(SEARCHING_REUSE_PROGRESS, getObjectsNumber()); - for (List<LocalObjectToPack> list : objectsLists) { + for (List<ObjectToPack> list : objectsLists) { for (ObjectToPack otp : list) { if (initMonitor.isCancelled()) throw new IOException( JGitText.get().packingCancelledDuringObjectsWriting); - windowCursor.selectObjectRepresentation(this, otp); + reuseSupport.selectObjectRepresentation(this, otp); initMonitor.update(1); } } initMonitor.endTask(); } - private void writeHeader() throws IOException { - System.arraycopy(Constants.PACK_SIGNATURE, 0, buf, 0, 4); - NB.encodeInt32(buf, 4, PACK_VERSION_GENERATED); - NB.encodeInt32(buf, 8, getObjectsNumber()); - out.write(buf, 0, 12); - } - private void writeObjects() throws IOException { - for (List<LocalObjectToPack> list : objectsLists) { - for (LocalObjectToPack otp : list) { + for (List<ObjectToPack> list : objectsLists) { + for (ObjectToPack otp : list) { if (writeMonitor.isCancelled()) throw new IOException( JGitText.get().packingCancelledDuringObjectsWriting); @@ -639,74 +640,88 @@ public class PackWriter { } } - private void writeObject(final LocalObjectToPack otp) throws IOException { - otp.markWantWrite(); - if (otp.isDeltaRepresentation()) { - LocalObjectToPack deltaBase = (LocalObjectToPack)otp.getDeltaBase(); - assert deltaBase != null || thin; - if (deltaBase != null && !deltaBase.isWritten()) { - if (deltaBase.wantWrite()) { - otp.clearDeltaBase(); // cycle detected - otp.clearReuseAsIs(); - } else { - writeObject(deltaBase); - } - } - } + private void writeObject(final ObjectToPack otp) throws IOException { + if (otp.isWritten()) + return; // We shouldn't be here. - assert !otp.isWritten(); + otp.markWantWrite(); + if (otp.isDeltaRepresentation()) + writeBaseFirst(otp); out.resetCRC32(); otp.setOffset(out.length()); - final PackedObjectLoader reuse = open(otp); - if (reuse != null) { + while (otp.isReuseAsIs()) { try { - if (otp.isDeltaRepresentation()) - writeDeltaObjectHeader(otp, reuse); - else - writeObjectHeader(otp.getType(), reuse.getSize()); - reuse.copyRawData(out, buf, windowCursor); - } finally { - reuse.endCopyRawData(); + reuseSupport.copyObjectAsIs(out, otp); + otp.setCRC(out.getCRC32()); + writeMonitor.update(1); + return; + } catch (StoredObjectRepresentationNotAvailableException gone) { + if (otp.getOffset() == out.length()) { + redoSearchForReuse(otp); + continue; + } else { + // Object writing already started, we cannot recover. + // + CorruptObjectException coe; + coe = new CorruptObjectException(otp, ""); + coe.initCause(gone); + throw coe; + } } - } else if (otp.isDeltaRepresentation()) { - throw new IOException(JGitText.get().creatingDeltasIsNotImplemented); - } else { - writeWholeObjectDeflate(otp); } - otp.setCRC(out.getCRC32()); + // If we reached here, reuse wasn't possible. + // + writeWholeObjectDeflate(otp); + otp.setCRC(out.getCRC32()); writeMonitor.update(1); } - private PackedObjectLoader open(final LocalObjectToPack otp) throws IOException { - while (otp.isReuseAsIs()) { - try { - PackedObjectLoader reuse = otp.getCopyLoader(windowCursor); - reuse.beginCopyRawData(); - return reuse; - } catch (IOException err) { - // The pack we found the object in originally is gone, or - // it has been overwritten with a different layout. - // - otp.clearDeltaBase(); - otp.clearReuseAsIs(); - windowCursor.selectObjectRepresentation(this, otp); - continue; + private void writeBaseFirst(final ObjectToPack otp) throws IOException { + ObjectToPack baseInPack = otp.getDeltaBase(); + if (baseInPack != null) { + if (!baseInPack.isWritten()) { + if (baseInPack.wantWrite()) { + // There is a cycle. Our caller is trying to write the + // object we want as a base, and called us. Turn off + // delta reuse so we can find another form. + // + reuseDeltas = false; + redoSearchForReuse(otp); + reuseDeltas = true; + } else { + writeObject(baseInPack); + } } + } else if (!thin) { + // This should never occur, the base isn't in the pack and + // the pack isn't allowed to reference base outside objects. + // Write the object as a whole form, even if that is slow. + // + otp.clearDeltaBase(); + otp.clearReuseAsIs(); } - return null; + } + + private void redoSearchForReuse(final ObjectToPack otp) throws IOException, + MissingObjectException { + otp.clearDeltaBase(); + otp.clearReuseAsIs(); + reuseSupport.selectObjectRepresentation(this, otp); } private void writeWholeObjectDeflate(final ObjectToPack otp) throws IOException { - final ObjectLoader loader = db.openObject(windowCursor, otp); + final ObjectLoader loader = db.openObject(reader, otp); final byte[] data = loader.getCachedBytes(); - writeObjectHeader(otp.getType(), data.length); + out.writeHeader(otp, data.length); deflater.reset(); deflater.setInput(data, 0, data.length); deflater.finish(); + + byte[] buf = out.getCopyBuffer(); do { final int n = deflater.deflate(buf, 0, buf.length); if (n > 0) @@ -714,42 +729,6 @@ public class PackWriter { } while (!deflater.finished()); } - private void writeDeltaObjectHeader(final ObjectToPack otp, - final PackedObjectLoader reuse) throws IOException { - if (deltaBaseAsOffset && otp.getDeltaBase() != null) { - writeObjectHeader(Constants.OBJ_OFS_DELTA, reuse.getRawSize()); - - final ObjectToPack deltaBase = otp.getDeltaBase(); - long offsetDiff = otp.getOffset() - deltaBase.getOffset(); - int pos = buf.length - 1; - buf[pos] = (byte) (offsetDiff & 0x7F); - while ((offsetDiff >>= 7) > 0) { - buf[--pos] = (byte) (0x80 | (--offsetDiff & 0x7F)); - } - - out.write(buf, pos, buf.length - pos); - } else { - writeObjectHeader(Constants.OBJ_REF_DELTA, reuse.getRawSize()); - otp.getDeltaBaseId().copyRawTo(buf, 0); - out.write(buf, 0, Constants.OBJECT_ID_LENGTH); - } - } - - private void writeObjectHeader(final int objectType, long dataLength) - throws IOException { - long nextLength = dataLength >>> 4; - int size = 0; - buf[size++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) - | (objectType << 4) | (dataLength & 0x0F)); - dataLength = nextLength; - while (dataLength > 0) { - nextLength >>>= 7; - buf[size++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (dataLength & 0x7F)); - dataLength = nextLength; - } - out.write(buf, 0, size); - } - private void writeChecksum() throws IOException { packcsum = out.getDigest(); out.write(packcsum); @@ -824,7 +803,11 @@ public class PackWriter { return; } - final LocalObjectToPack otp = windowCursor.newObjectToPack(object); + final ObjectToPack otp; + if (reuseSupport != null) + otp = reuseSupport.newObjectToPack(object); + else + otp = new ObjectToPack(object); try { objectsLists[object.getType()].add(otp); } catch (ArrayIndexOutOfBoundsException x) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackedObjectLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackedObjectLoader.java index 026b008f1a..47f5e67a73 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackedObjectLoader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PackedObjectLoader.java @@ -47,7 +47,6 @@ package org.eclipse.jgit.lib; import java.io.IOException; -import java.io.OutputStream; /** * Base class for a set of object loader classes for packed objects. @@ -116,69 +115,6 @@ abstract class PackedObjectLoader extends ObjectLoader { } /** - * Peg the pack file open to support data copying. - * <p> - * Applications trying to copy raw pack data should ensure the pack stays - * open and available throughout the entire copy. To do that use: - * - * <pre> - * loader.beginCopyRawData(); - * try { - * loader.copyRawData(out, tmpbuf, curs); - * } finally { - * loader.endCopyRawData(); - * } - * </pre> - * - * @throws IOException - * this loader contains stale information and cannot be used. - * The most likely cause is the underlying pack file has been - * deleted, and the object has moved to another pack file. - */ - public void beginCopyRawData() throws IOException { - pack.beginCopyRawData(); - } - - /** - * Copy raw object representation from storage to provided output stream. - * <p> - * Copied data doesn't include object header. User must provide temporary - * buffer used during copying by underlying I/O layer. - * </p> - * - * @param out - * output stream when data is copied. No buffering is guaranteed. - * @param buf - * temporary buffer used during copying. Recommended size is at - * least few kB. - * @param curs - * temporary thread storage during data access. - * @throws IOException - * when the object cannot be read. - * @see #beginCopyRawData() - */ - public void copyRawData(OutputStream out, byte buf[], WindowCursor curs) - throws IOException { - pack.copyRawData(this, out, buf, curs); - } - - /** Release resources after {@link #beginCopyRawData()}. */ - public void endCopyRawData() { - pack.endCopyRawData(); - } - - /** - * @return true if this loader is capable of fast raw-data copying basing on - * compressed data checksum; false if raw-data copying needs - * uncompressing and compressing data - * @throws IOException - * the index file format cannot be determined. - */ - public boolean supportsFastCopyRawData() throws IOException { - return pack.supportsFastCopyRawData(); - } - - /** * @return id of delta base object for this object representation. null if * object is not stored as delta. * @throws IOException diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/WindowCursor.java index e167354928..36095ed5e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/WindowCursor.java @@ -49,6 +49,7 @@ import java.util.zip.DataFormatException; import java.util.zip.Inflater; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.revwalk.RevObject; /** Active handle to a ByteWindow. */ @@ -90,6 +91,12 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { db.selectObjectRepresentation(packer, otp, this); } + public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp) + throws IOException, StoredObjectRepresentationNotAvailableException { + LocalObjectToPack src = (LocalObjectToPack) otp; + src.copyFromPack.copyAsIs(out, src, this); + } + /** * Copy bytes from the window to a caller supplied buffer. * @@ -159,16 +166,9 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } } - void inflateVerify(final PackFile pack, long position) throws IOException, - DataFormatException { + Inflater inflater() { prepareInflater(); - for (;;) { - pin(pack, position); - window.inflateVerify(position, inf); - if (inf.finished()) - return; - position = window.end; - } + return inf; } private void prepareInflater() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/LongList.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/LongList.java index 26608bb2a1..96b311dfbf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/LongList.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/LongList.java @@ -83,6 +83,20 @@ public class LongList { return entries[i]; } + /** + * Determine if an entry appears in this collection. + * + * @param value + * the value to search for. + * @return true of {@code value} appears in this list. + */ + public boolean contains(final long value) { + for (int i = 0; i < count; i++) + if (entries[i] == value) + return true; + return false; + } + /** Empty this list */ public void clear() { count = 0; |