123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526 |
- /*
- * Copyright (C) 2008-2009, Google Inc.
- * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org>
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.lib;
-
- import java.io.EOFException;
- import java.io.File;
- import java.io.IOException;
- import java.io.OutputStream;
- import java.io.RandomAccessFile;
- import java.nio.MappedByteBuffer;
- import java.nio.channels.FileChannel.MapMode;
- import java.text.MessageFormat;
- import java.util.Arrays;
- import java.util.Collections;
- import java.util.Comparator;
- import java.util.Iterator;
- import java.util.zip.CRC32;
- import java.util.zip.CheckedOutputStream;
- import java.util.zip.DataFormatException;
-
- import org.eclipse.jgit.JGitText;
- import org.eclipse.jgit.errors.CorruptObjectException;
- import org.eclipse.jgit.errors.PackInvalidException;
- import org.eclipse.jgit.errors.PackMismatchException;
- import org.eclipse.jgit.util.NB;
- import org.eclipse.jgit.util.RawParseUtils;
-
- /**
- * A Git version 2 pack file representation. A pack file contains Git objects in
- * delta packed format yielding high compression of lots of object where some
- * objects are similar.
- */
- public class PackFile implements Iterable<PackIndex.MutableEntry> {
- /** Sorts PackFiles to be most recently created to least recently created. */
- public static Comparator<PackFile> SORT = new Comparator<PackFile>() {
- public int compare(final PackFile a, final PackFile b) {
- return b.packLastModified - a.packLastModified;
- }
- };
-
- private final File idxFile;
-
- private final File packFile;
-
- final int hash;
-
- private RandomAccessFile fd;
-
- /** Serializes reads performed against {@link #fd}. */
- private final Object readLock = new Object();
-
- long length;
-
- private int activeWindows;
-
- private int activeCopyRawData;
-
- private int packLastModified;
-
- private volatile boolean invalid;
-
- private byte[] packChecksum;
-
- private PackIndex loadedIdx;
-
- private PackReverseIndex reverseIdx;
-
- /**
- * Construct a reader for an existing, pre-indexed packfile.
- *
- * @param idxFile
- * path of the <code>.idx</code> file listing the contents.
- * @param packFile
- * path of the <code>.pack</code> file holding the data.
- */
- public PackFile(final File idxFile, final File packFile) {
- this.idxFile = idxFile;
- this.packFile = packFile;
- this.packLastModified = (int) (packFile.lastModified() >> 10);
-
- // Multiply by 31 here so we can more directly combine with another
- // value in WindowCache.hash(), without doing the multiply there.
- //
- hash = System.identityHashCode(this) * 31;
- length = Long.MAX_VALUE;
- }
-
- private synchronized PackIndex idx() throws IOException {
- if (loadedIdx == null) {
- if (invalid)
- throw new PackInvalidException(packFile);
-
- try {
- final PackIndex idx = PackIndex.open(idxFile);
-
- if (packChecksum == null)
- packChecksum = idx.packChecksum;
- else if (!Arrays.equals(packChecksum, idx.packChecksum))
- throw new PackMismatchException(JGitText.get().packChecksumMismatch);
-
- loadedIdx = idx;
- } catch (IOException e) {
- invalid = true;
- throw e;
- }
- }
- return loadedIdx;
- }
-
- final PackedObjectLoader resolveBase(final WindowCursor curs, final long ofs)
- throws IOException {
- return reader(curs, ofs);
- }
-
- /** @return the File object which locates this pack on disk. */
- public File getPackFile() {
- return packFile;
- }
-
- /**
- * Determine if an object is contained within the pack file.
- * <p>
- * For performance reasons only the index file is searched; the main pack
- * content is ignored entirely.
- * </p>
- *
- * @param id
- * the object to look for. Must not be null.
- * @return true if the object is in this pack; false otherwise.
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- public boolean hasObject(final AnyObjectId id) throws IOException {
- return idx().hasObject(id);
- }
-
- /**
- * Get an object from this pack.
- *
- * @param curs
- * temporary working space associated with the calling thread.
- * @param id
- * the object to obtain from the pack. Must not be null.
- * @return the object loader for the requested object if it is contained in
- * this pack; null if the object was not found.
- * @throws IOException
- * the pack file or the index could not be read.
- */
- public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
- throws IOException {
- final long offset = idx().findOffset(id);
- return 0 < offset ? reader(curs, offset) : null;
- }
-
- /**
- * Close the resources utilized by this repository
- */
- public void close() {
- UnpackedObjectCache.purge(this);
- WindowCache.purge(this);
- synchronized (this) {
- loadedIdx = null;
- reverseIdx = null;
- }
- }
-
- /**
- * Provide iterator over entries in associated pack index, that should also
- * exist in this pack file. Objects returned by such iterator are mutable
- * during iteration.
- * <p>
- * Iterator returns objects in SHA-1 lexicographical order.
- * </p>
- *
- * @return iterator over entries of associated pack index
- *
- * @see PackIndex#iterator()
- */
- public Iterator<PackIndex.MutableEntry> iterator() {
- try {
- return idx().iterator();
- } catch (IOException e) {
- return Collections.<PackIndex.MutableEntry> emptyList().iterator();
- }
- }
-
- /**
- * Obtain the total number of objects available in this pack. This method
- * relies on pack index, giving number of effectively available objects.
- *
- * @return number of objects in index of this pack, likewise in this pack
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- long getObjectCount() throws IOException {
- return idx().getObjectCount();
- }
-
- /**
- * Search for object id with the specified start offset in associated pack
- * (reverse) index.
- *
- * @param offset
- * start offset of object to find
- * @return object id for this offset, or null if no object was found
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- ObjectId findObjectForOffset(final long offset) throws IOException {
- return getReverseIdx().findObject(offset);
- }
-
- final UnpackedObjectCache.Entry readCache(final long position) {
- return UnpackedObjectCache.get(this, position);
- }
-
- final void saveCache(final long position, final byte[] data, final int type) {
- UnpackedObjectCache.store(this, position, data, type);
- }
-
- final byte[] decompress(final long position, final int totalSize,
- final WindowCursor curs) throws DataFormatException, IOException {
- final byte[] dstbuf = new byte[totalSize];
- if (curs.inflate(this, position, dstbuf, 0) != totalSize)
- throw new EOFException(MessageFormat.format(JGitText.get().shortCompressedStreamAt, position));
- return dstbuf;
- }
-
- final void copyRawData(final PackedObjectLoader loader,
- final OutputStream out, final byte buf[], final WindowCursor curs)
- throws IOException {
- final long objectOffset = loader.objectOffset;
- final long dataOffset = objectOffset + loader.headerSize;
- final long sz = findEndOffset(objectOffset) - dataOffset;
- final PackIndex idx = idx();
-
- if (idx.hasCRC32Support()) {
- final CRC32 crc = new CRC32();
- int headerCnt = loader.headerSize;
- while (headerCnt > 0) {
- final int toRead = Math.min(headerCnt, buf.length);
- readFully(objectOffset, buf, 0, toRead, curs);
- crc.update(buf, 0, toRead);
- headerCnt -= toRead;
- }
- final CheckedOutputStream crcOut = new CheckedOutputStream(out, crc);
- copyToStream(dataOffset, buf, sz, crcOut, curs);
- final long computed = crc.getValue();
-
- final ObjectId id = findObjectForOffset(objectOffset);
- final long expected = idx.findCRC32(id);
- if (computed != expected)
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream, objectOffset, getPackFile()));
- } else {
- try {
- curs.inflateVerify(this, dataOffset);
- } catch (DataFormatException dfe) {
- final CorruptObjectException coe;
- coe = new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream, objectOffset, getPackFile()));
- coe.initCause(dfe);
- throw coe;
- }
- copyToStream(dataOffset, buf, sz, out, curs);
- }
- }
-
- boolean supportsFastCopyRawData() throws IOException {
- return idx().hasCRC32Support();
- }
-
- boolean invalid() {
- return invalid;
- }
-
- private void readFully(final long position, final byte[] dstbuf,
- int dstoff, final int cnt, final WindowCursor curs)
- throws IOException {
- if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt)
- throw new EOFException();
- }
-
- private void copyToStream(long position, final byte[] buf, long cnt,
- final OutputStream out, final WindowCursor curs)
- throws IOException, EOFException {
- while (cnt > 0) {
- final int toRead = (int) Math.min(cnt, buf.length);
- readFully(position, buf, 0, toRead, curs);
- position += toRead;
- cnt -= toRead;
- out.write(buf, 0, toRead);
- }
- }
-
- synchronized void beginCopyRawData() throws IOException {
- if (++activeCopyRawData == 1 && activeWindows == 0)
- doOpen();
- }
-
- synchronized void endCopyRawData() {
- if (--activeCopyRawData == 0 && activeWindows == 0)
- doClose();
- }
-
- synchronized boolean beginWindowCache() throws IOException {
- if (++activeWindows == 1) {
- if (activeCopyRawData == 0)
- doOpen();
- return true;
- }
- return false;
- }
-
- synchronized boolean endWindowCache() {
- final boolean r = --activeWindows == 0;
- if (r && activeCopyRawData == 0)
- doClose();
- return r;
- }
-
- private void doOpen() throws IOException {
- try {
- if (invalid)
- throw new PackInvalidException(packFile);
- synchronized (readLock) {
- fd = new RandomAccessFile(packFile, "r");
- length = fd.length();
- onOpenPack();
- }
- } catch (IOException ioe) {
- openFail();
- throw ioe;
- } catch (RuntimeException re) {
- openFail();
- throw re;
- } catch (Error re) {
- openFail();
- throw re;
- }
- }
-
- private void openFail() {
- activeWindows = 0;
- activeCopyRawData = 0;
- invalid = true;
- doClose();
- }
-
- private void doClose() {
- synchronized (readLock) {
- if (fd != null) {
- try {
- fd.close();
- } catch (IOException err) {
- // Ignore a close event. We had it open only for reading.
- // There should not be errors related to network buffers
- // not flushed, etc.
- }
- fd = null;
- }
- }
- }
-
- ByteArrayWindow read(final long pos, int size) throws IOException {
- synchronized (readLock) {
- if (length < pos + size)
- size = (int) (length - pos);
- final byte[] buf = new byte[size];
- fd.seek(pos);
- fd.readFully(buf, 0, size);
- return new ByteArrayWindow(this, pos, buf);
- }
- }
-
- ByteWindow mmap(final long pos, int size) throws IOException {
- synchronized (readLock) {
- if (length < pos + size)
- size = (int) (length - pos);
-
- MappedByteBuffer map;
- try {
- map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
- } catch (IOException ioe1) {
- // The most likely reason this failed is the JVM has run out
- // of virtual memory. We need to discard quickly, and try to
- // force the GC to finalize and release any existing mappings.
- //
- System.gc();
- System.runFinalization();
- map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
- }
-
- if (map.hasArray())
- return new ByteArrayWindow(this, pos, map.array());
- return new ByteBufferWindow(this, pos, map);
- }
- }
-
- private void onOpenPack() throws IOException {
- final PackIndex idx = idx();
- final byte[] buf = new byte[20];
-
- fd.seek(0);
- fd.readFully(buf, 0, 12);
- if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4)
- throw new IOException(JGitText.get().notAPACKFile);
- final long vers = NB.decodeUInt32(buf, 4);
- final long packCnt = NB.decodeUInt32(buf, 8);
- if (vers != 2 && vers != 3)
- throw new IOException(MessageFormat.format(JGitText.get().unsupportedPackVersion, vers));
-
- if (packCnt != idx.getObjectCount())
- throw new PackMismatchException(MessageFormat.format(
- JGitText.get().packObjectCountMismatch, packCnt, idx.getObjectCount(), getPackFile()));
-
- fd.seek(length - 20);
- fd.read(buf, 0, 20);
- if (!Arrays.equals(buf, packChecksum))
- throw new PackMismatchException(MessageFormat.format(
- JGitText.get().packObjectCountMismatch
- , ObjectId.fromRaw(buf).name()
- , ObjectId.fromRaw(idx.packChecksum).name()
- , getPackFile()));
- }
-
- private PackedObjectLoader reader(final WindowCursor curs,
- final long objOffset) throws IOException {
- int p = 0;
- final byte[] ib = curs.tempId;
- readFully(objOffset, ib, 0, 20, curs);
- int c = ib[p++] & 0xff;
- final int typeCode = (c >> 4) & 7;
- long dataSize = c & 15;
- int shift = 4;
- while ((c & 0x80) != 0) {
- c = ib[p++] & 0xff;
- dataSize += (c & 0x7f) << shift;
- shift += 7;
- }
-
- switch (typeCode) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return new WholePackedObjectLoader(this, objOffset, p, typeCode,
- (int) dataSize);
-
- case Constants.OBJ_OFS_DELTA: {
- c = ib[p++] & 0xff;
- long ofs = c & 127;
- while ((c & 128) != 0) {
- ofs += 1;
- c = ib[p++] & 0xff;
- ofs <<= 7;
- ofs += (c & 127);
- }
- return new DeltaOfsPackedObjectLoader(this, objOffset, p,
- (int) dataSize, objOffset - ofs);
- }
- case Constants.OBJ_REF_DELTA: {
- readFully(objOffset + p, ib, 0, 20, curs);
- return new DeltaRefPackedObjectLoader(this, objOffset, p + 20,
- (int) dataSize, ObjectId.fromRaw(ib));
- }
- default:
- throw new IOException(MessageFormat.format(JGitText.get().unknownObjectType, typeCode));
- }
- }
-
- private long findEndOffset(final long startOffset)
- throws IOException, CorruptObjectException {
- final long maxOffset = length - 20;
- return getReverseIdx().findNextOffset(startOffset, maxOffset);
- }
-
- private synchronized PackReverseIndex getReverseIdx() throws IOException {
- if (reverseIdx == null)
- reverseIdx = new PackReverseIndex(idx());
- return reverseIdx;
- }
- }
|