123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200 |
- /*
- * Copyright (C) 2008-2009, Google Inc.
- * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org>
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.internal.storage.file;
-
- import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
- import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
- import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
-
- import java.io.EOFException;
- import java.io.File;
- import java.io.FileNotFoundException;
- import java.io.IOException;
- import java.io.InterruptedIOException;
- import java.io.RandomAccessFile;
- import java.nio.MappedByteBuffer;
- import java.nio.channels.FileChannel.MapMode;
- import java.nio.file.AccessDeniedException;
- import java.nio.file.NoSuchFileException;
- import java.text.MessageFormat;
- import java.util.Arrays;
- import java.util.Collections;
- import java.util.Comparator;
- import java.util.Iterator;
- import java.util.Set;
- import java.util.concurrent.atomic.AtomicInteger;
- import java.util.zip.CRC32;
- import java.util.zip.DataFormatException;
- import java.util.zip.Inflater;
-
- import org.eclipse.jgit.errors.CorruptObjectException;
- import org.eclipse.jgit.errors.LargeObjectException;
- import org.eclipse.jgit.errors.MissingObjectException;
- import org.eclipse.jgit.errors.NoPackSignatureException;
- import org.eclipse.jgit.errors.PackInvalidException;
- import org.eclipse.jgit.errors.PackMismatchException;
- import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
- import org.eclipse.jgit.errors.UnpackException;
- import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException;
- import org.eclipse.jgit.errors.UnsupportedPackVersionException;
- import org.eclipse.jgit.internal.JGitText;
- import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
- import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
- import org.eclipse.jgit.internal.storage.pack.PackExt;
- import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
- import org.eclipse.jgit.lib.AbbreviatedObjectId;
- import org.eclipse.jgit.lib.AnyObjectId;
- import org.eclipse.jgit.lib.Constants;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.lib.ObjectLoader;
- import org.eclipse.jgit.util.LongList;
- import org.eclipse.jgit.util.NB;
- import org.eclipse.jgit.util.RawParseUtils;
-
- /**
- * A Git version 2 pack file representation. A pack file contains Git objects in
- * delta packed format yielding high compression of lots of object where some
- * objects are similar.
- */
- public class PackFile implements Iterable<PackIndex.MutableEntry> {
- /** Sorts PackFiles to be most recently created to least recently created. */
- public static final Comparator<PackFile> SORT = new Comparator<PackFile>() {
- @Override
- public int compare(final PackFile a, final PackFile b) {
- return b.packLastModified - a.packLastModified;
- }
- };
-
- private final File packFile;
-
- private final int extensions;
-
- private File keepFile;
-
- private volatile String packName;
-
- final int hash;
-
- private RandomAccessFile fd;
-
- /** Serializes reads performed against {@link #fd}. */
- private final Object readLock = new Object();
-
- long length;
-
- private int activeWindows;
-
- private int activeCopyRawData;
-
- int packLastModified;
-
- private FileSnapshot fileSnapshot;
-
- private volatile boolean invalid;
-
- private volatile Exception invalidatingCause;
-
- private boolean invalidBitmap;
-
- private AtomicInteger transientErrorCount = new AtomicInteger();
-
- private byte[] packChecksum;
-
- private PackIndex loadedIdx;
-
- private PackReverseIndex reverseIdx;
-
- private PackBitmapIndex bitmapIdx;
-
- /**
- * Objects we have tried to read, and discovered to be corrupt.
- * <p>
- * The list is allocated after the first corruption is found, and filled in
- * as more entries are discovered. Typically this list is never used, as
- * pack files do not usually contain corrupt objects.
- */
- private volatile LongList corruptObjects;
-
- /**
- * Construct a reader for an existing, pre-indexed packfile.
- *
- * @param packFile
- * path of the <code>.pack</code> file holding the data.
- * @param extensions
- * additional pack file extensions with the same base as the pack
- */
- public PackFile(final File packFile, int extensions) {
- this.packFile = packFile;
- this.fileSnapshot = FileSnapshot.save(packFile);
- this.packLastModified = (int) (fileSnapshot.lastModified() >> 10);
- this.extensions = extensions;
-
- // Multiply by 31 here so we can more directly combine with another
- // value in WindowCache.hash(), without doing the multiply there.
- //
- hash = System.identityHashCode(this) * 31;
- length = Long.MAX_VALUE;
- }
-
- private synchronized PackIndex idx() throws IOException {
- if (loadedIdx == null) {
- if (invalid) {
- throw new PackInvalidException(packFile, invalidatingCause);
- }
-
- try {
- final PackIndex idx = PackIndex.open(extFile(INDEX));
-
- if (packChecksum == null) {
- packChecksum = idx.packChecksum;
- } else if (!Arrays.equals(packChecksum, idx.packChecksum)) {
- throw new PackMismatchException(MessageFormat.format(
- JGitText.get().packChecksumMismatch,
- packFile.getPath()));
- }
- loadedIdx = idx;
- } catch (InterruptedIOException e) {
- // don't invalidate the pack, we are interrupted from another thread
- throw e;
- } catch (IOException e) {
- invalid = true;
- invalidatingCause = e;
- throw e;
- }
- }
- return loadedIdx;
- }
-
- /** @return the File object which locates this pack on disk. */
- public File getPackFile() {
- return packFile;
- }
-
- /**
- * @return the index for this pack file.
- * @throws IOException
- */
- public PackIndex getIndex() throws IOException {
- return idx();
- }
-
- /** @return name extracted from {@code pack-*.pack} pattern. */
- public String getPackName() {
- String name = packName;
- if (name == null) {
- name = getPackFile().getName();
- if (name.startsWith("pack-")) //$NON-NLS-1$
- name = name.substring("pack-".length()); //$NON-NLS-1$
- if (name.endsWith(".pack")) //$NON-NLS-1$
- name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$
- packName = name;
- }
- return name;
- }
-
- /**
- * Determine if an object is contained within the pack file.
- * <p>
- * For performance reasons only the index file is searched; the main pack
- * content is ignored entirely.
- * </p>
- *
- * @param id
- * the object to look for. Must not be null.
- * @return true if the object is in this pack; false otherwise.
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- public boolean hasObject(final AnyObjectId id) throws IOException {
- final long offset = idx().findOffset(id);
- return 0 < offset && !isCorrupt(offset);
- }
-
- /**
- * Determines whether a .keep file exists for this pack file.
- *
- * @return true if a .keep file exist.
- */
- public boolean shouldBeKept() {
- if (keepFile == null)
- keepFile = extFile(KEEP);
- return keepFile.exists();
- }
-
- /**
- * Get an object from this pack.
- *
- * @param curs
- * temporary working space associated with the calling thread.
- * @param id
- * the object to obtain from the pack. Must not be null.
- * @return the object loader for the requested object if it is contained in
- * this pack; null if the object was not found.
- * @throws IOException
- * the pack file or the index could not be read.
- */
- ObjectLoader get(final WindowCursor curs, final AnyObjectId id)
- throws IOException {
- final long offset = idx().findOffset(id);
- return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null;
- }
-
- void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit)
- throws IOException {
- idx().resolve(matches, id, matchLimit);
- }
-
- /**
- * Close the resources utilized by this repository
- */
- public void close() {
- WindowCache.purge(this);
- synchronized (this) {
- loadedIdx = null;
- reverseIdx = null;
- }
- }
-
- /**
- * Provide iterator over entries in associated pack index, that should also
- * exist in this pack file. Objects returned by such iterator are mutable
- * during iteration.
- * <p>
- * Iterator returns objects in SHA-1 lexicographical order.
- * </p>
- *
- * @return iterator over entries of associated pack index
- *
- * @see PackIndex#iterator()
- */
- @Override
- public Iterator<PackIndex.MutableEntry> iterator() {
- try {
- return idx().iterator();
- } catch (IOException e) {
- return Collections.<PackIndex.MutableEntry> emptyList().iterator();
- }
- }
-
- /**
- * Obtain the total number of objects available in this pack. This method
- * relies on pack index, giving number of effectively available objects.
- *
- * @return number of objects in index of this pack, likewise in this pack
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- long getObjectCount() throws IOException {
- return idx().getObjectCount();
- }
-
- /**
- * Search for object id with the specified start offset in associated pack
- * (reverse) index.
- *
- * @param offset
- * start offset of object to find
- * @return object id for this offset, or null if no object was found
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- ObjectId findObjectForOffset(final long offset) throws IOException {
- return getReverseIdx().findObject(offset);
- }
-
- /**
- * Return the @{@link FileSnapshot} associated to the underlying packfile
- * that has been used when the object was created.
- *
- * @return the packfile @{@link FileSnapshot} that the object is loaded from.
- */
- FileSnapshot getFileSnapshot() {
- return fileSnapshot;
- }
-
- private final byte[] decompress(final long position, final int sz,
- final WindowCursor curs) throws IOException, DataFormatException {
- byte[] dstbuf;
- try {
- dstbuf = new byte[sz];
- } catch (OutOfMemoryError noMemory) {
- // The size may be larger than our heap allows, return null to
- // let the caller know allocation isn't possible and it should
- // use the large object streaming approach instead.
- //
- // For example, this can occur when sz is 640 MB, and JRE
- // maximum heap size is only 256 MB. Even if the JRE has
- // 200 MB free, it cannot allocate a 640 MB byte array.
- return null;
- }
-
- if (curs.inflate(this, position, dstbuf, false) != sz)
- throw new EOFException(MessageFormat.format(
- JGitText.get().shortCompressedStreamAt,
- Long.valueOf(position)));
- return dstbuf;
- }
-
- void copyPackAsIs(PackOutputStream out, WindowCursor curs)
- throws IOException {
- // Pin the first window, this ensures the length is accurate.
- curs.pin(this, 0);
- curs.copyPackAsIs(this, length, out);
- }
-
- final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
- boolean validate, WindowCursor curs) throws IOException,
- StoredObjectRepresentationNotAvailableException {
- beginCopyAsIs(src);
- try {
- copyAsIs2(out, src, validate, curs);
- } finally {
- endCopyAsIs();
- }
- }
-
- private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
- boolean validate, WindowCursor curs) throws IOException,
- StoredObjectRepresentationNotAvailableException {
- final CRC32 crc1 = validate ? new CRC32() : null;
- final CRC32 crc2 = validate ? new CRC32() : null;
- final byte[] buf = out.getCopyBuffer();
-
- // Rip apart the header so we can discover the size.
- //
- readFully(src.offset, buf, 0, 20, curs);
- int c = buf[0] & 0xff;
- final int typeCode = (c >> 4) & 7;
- long inflatedLength = c & 15;
- int shift = 4;
- int headerCnt = 1;
- while ((c & 0x80) != 0) {
- c = buf[headerCnt++] & 0xff;
- inflatedLength += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- if (typeCode == Constants.OBJ_OFS_DELTA) {
- do {
- c = buf[headerCnt++] & 0xff;
- } while ((c & 128) != 0);
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
- } else if (typeCode == Constants.OBJ_REF_DELTA) {
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
-
- readFully(src.offset + headerCnt, buf, 0, 20, curs);
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, 20);
- crc2.update(buf, 0, 20);
- }
- headerCnt += 20;
- } else if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
-
- final long dataOffset = src.offset + headerCnt;
- final long dataLength = src.length;
- final long expectedCRC;
- final ByteArrayWindow quickCopy;
-
- // Verify the object isn't corrupt before sending. If it is,
- // we report it missing instead.
- //
- try {
- quickCopy = curs.quickCopy(this, dataOffset, dataLength);
-
- if (validate && idx().hasCRC32Support()) {
- assert(crc1 != null);
- // Index has the CRC32 code cached, validate the object.
- //
- expectedCRC = idx().findCRC32(src);
- if (quickCopy != null) {
- quickCopy.crc32(crc1, dataOffset, (int) dataLength);
- } else {
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, curs);
- crc1.update(buf, 0, n);
- pos += n;
- cnt -= n;
- }
- }
- if (crc1.getValue() != expectedCRC) {
- setCorrupt(src.offset);
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackFile()));
- }
- } else if (validate) {
- // We don't have a CRC32 code in the index, so compute it
- // now while inflating the raw data to get zlib to tell us
- // whether or not the data is safe.
- //
- Inflater inf = curs.inflater();
- byte[] tmp = new byte[1024];
- if (quickCopy != null) {
- quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
- } else {
- assert(crc1 != null);
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, curs);
- crc1.update(buf, 0, n);
- inf.setInput(buf, 0, n);
- while (inf.inflate(tmp, 0, tmp.length) > 0)
- continue;
- pos += n;
- cnt -= n;
- }
- }
- if (!inf.finished() || inf.getBytesRead() != dataLength) {
- setCorrupt(src.offset);
- throw new EOFException(MessageFormat.format(
- JGitText.get().shortCompressedStreamAt,
- Long.valueOf(src.offset)));
- }
- assert(crc1 != null);
- expectedCRC = crc1.getValue();
- } else {
- expectedCRC = -1;
- }
- } catch (DataFormatException dataFormat) {
- setCorrupt(src.offset);
-
- CorruptObjectException corruptObject = new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackFile()));
- corruptObject.initCause(dataFormat);
-
- StoredObjectRepresentationNotAvailableException gone;
- gone = new StoredObjectRepresentationNotAvailableException(src);
- gone.initCause(corruptObject);
- throw gone;
-
- } catch (IOException ioError) {
- StoredObjectRepresentationNotAvailableException gone;
- gone = new StoredObjectRepresentationNotAvailableException(src);
- gone.initCause(ioError);
- throw gone;
- }
-
- if (quickCopy != null) {
- // The entire object fits into a single byte array window slice,
- // and we have it pinned. Write this out without copying.
- //
- out.writeHeader(src, inflatedLength);
- quickCopy.write(out, dataOffset, (int) dataLength);
-
- } else if (dataLength <= buf.length) {
- // Tiny optimization: Lots of objects are very small deltas or
- // deflated commits that are likely to fit in the copy buffer.
- //
- if (!validate) {
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, curs);
- pos += n;
- cnt -= n;
- }
- }
- out.writeHeader(src, inflatedLength);
- out.write(buf, 0, (int) dataLength);
- } else {
- // Now we are committed to sending the object. As we spool it out,
- // check its CRC32 code to make sure there wasn't corruption between
- // the verification we did above, and us actually outputting it.
- //
- out.writeHeader(src, inflatedLength);
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, curs);
- if (validate) {
- assert(crc2 != null);
- crc2.update(buf, 0, n);
- }
- out.write(buf, 0, n);
- pos += n;
- cnt -= n;
- }
- if (validate) {
- assert(crc2 != null);
- if (crc2.getValue() != expectedCRC) {
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackFile()));
- }
- }
- }
- }
-
- boolean invalid() {
- return invalid;
- }
-
- void setInvalid() {
- invalid = true;
- }
-
- int incrementTransientErrorCount() {
- return transientErrorCount.incrementAndGet();
- }
-
- void resetTransientErrorCount() {
- transientErrorCount.set(0);
- }
-
- private void readFully(final long position, final byte[] dstbuf,
- int dstoff, final int cnt, final WindowCursor curs)
- throws IOException {
- if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt)
- throw new EOFException();
- }
-
- private synchronized void beginCopyAsIs(ObjectToPack otp)
- throws StoredObjectRepresentationNotAvailableException {
- if (++activeCopyRawData == 1 && activeWindows == 0) {
- try {
- doOpen();
- } catch (IOException thisPackNotValid) {
- StoredObjectRepresentationNotAvailableException gone;
-
- gone = new StoredObjectRepresentationNotAvailableException(otp);
- gone.initCause(thisPackNotValid);
- throw gone;
- }
- }
- }
-
- private synchronized void endCopyAsIs() {
- if (--activeCopyRawData == 0 && activeWindows == 0)
- doClose();
- }
-
- synchronized boolean beginWindowCache() throws IOException {
- if (++activeWindows == 1) {
- if (activeCopyRawData == 0)
- doOpen();
- return true;
- }
- return false;
- }
-
- synchronized boolean endWindowCache() {
- final boolean r = --activeWindows == 0;
- if (r && activeCopyRawData == 0)
- doClose();
- return r;
- }
-
- private void doOpen() throws IOException {
- if (invalid) {
- throw new PackInvalidException(packFile, invalidatingCause);
- }
- try {
- synchronized (readLock) {
- fd = new RandomAccessFile(packFile, "r"); //$NON-NLS-1$
- length = fd.length();
- onOpenPack();
- }
- } catch (InterruptedIOException e) {
- // don't invalidate the pack, we are interrupted from another thread
- openFail(false, e);
- throw e;
- } catch (FileNotFoundException fn) {
- // don't invalidate the pack if opening an existing file failed
- // since it may be related to a temporary lack of resources (e.g.
- // max open files)
- openFail(!packFile.exists(), fn);
- throw fn;
- } catch (EOFException | AccessDeniedException | NoSuchFileException
- | CorruptObjectException | NoPackSignatureException
- | PackMismatchException | UnpackException
- | UnsupportedPackIndexVersionException
- | UnsupportedPackVersionException pe) {
- // exceptions signaling permanent problems with a pack
- openFail(true, pe);
- throw pe;
- } catch (IOException | RuntimeException ge) {
- // generic exceptions could be transient so we should not mark the
- // pack invalid to avoid false MissingObjectExceptions
- openFail(false, ge);
- throw ge;
- }
- }
-
- private void openFail(boolean invalidate, Exception cause) {
- activeWindows = 0;
- activeCopyRawData = 0;
- invalid = invalidate;
- invalidatingCause = cause;
- doClose();
- }
-
- private void doClose() {
- synchronized (readLock) {
- if (fd != null) {
- try {
- fd.close();
- } catch (IOException err) {
- // Ignore a close event. We had it open only for reading.
- // There should not be errors related to network buffers
- // not flushed, etc.
- }
- fd = null;
- }
- }
- }
-
- ByteArrayWindow read(final long pos, int size) throws IOException {
- synchronized (readLock) {
- if (invalid || fd == null) {
- // Due to concurrency between a read and another packfile invalidation thread
- // one thread could come up to this point and then fail with NPE.
- // Detect the situation and throw a proper exception so that can be properly
- // managed by the main packfile search loop and the Git client won't receive
- // any failures.
- throw new PackInvalidException(packFile, invalidatingCause);
- }
- if (length < pos + size)
- size = (int) (length - pos);
- final byte[] buf = new byte[size];
- fd.seek(pos);
- fd.readFully(buf, 0, size);
- return new ByteArrayWindow(this, pos, buf);
- }
- }
-
- ByteWindow mmap(final long pos, int size) throws IOException {
- synchronized (readLock) {
- if (length < pos + size)
- size = (int) (length - pos);
-
- MappedByteBuffer map;
- try {
- map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
- } catch (IOException ioe1) {
- // The most likely reason this failed is the JVM has run out
- // of virtual memory. We need to discard quickly, and try to
- // force the GC to finalize and release any existing mappings.
- //
- System.gc();
- System.runFinalization();
- map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
- }
-
- if (map.hasArray())
- return new ByteArrayWindow(this, pos, map.array());
- return new ByteBufferWindow(this, pos, map);
- }
- }
-
- private void onOpenPack() throws IOException {
- final PackIndex idx = idx();
- final byte[] buf = new byte[20];
-
- fd.seek(0);
- fd.readFully(buf, 0, 12);
- if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) {
- throw new NoPackSignatureException(JGitText.get().notAPACKFile);
- }
- final long vers = NB.decodeUInt32(buf, 4);
- final long packCnt = NB.decodeUInt32(buf, 8);
- if (vers != 2 && vers != 3) {
- throw new UnsupportedPackVersionException(vers);
- }
-
- if (packCnt != idx.getObjectCount()) {
- throw new PackMismatchException(MessageFormat.format(
- JGitText.get().packObjectCountMismatch,
- Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()),
- getPackFile()));
- }
-
- fd.seek(length - 20);
- fd.readFully(buf, 0, 20);
- if (!Arrays.equals(buf, packChecksum)) {
- throw new PackMismatchException(MessageFormat.format(
- JGitText.get().packObjectCountMismatch
- , ObjectId.fromRaw(buf).name()
- , ObjectId.fromRaw(idx.packChecksum).name()
- , getPackFile()));
- }
- }
-
- ObjectLoader load(final WindowCursor curs, long pos)
- throws IOException, LargeObjectException {
- try {
- final byte[] ib = curs.tempId;
- Delta delta = null;
- byte[] data = null;
- int type = Constants.OBJ_BAD;
- boolean cached = false;
-
- SEARCH: for (;;) {
- readFully(pos, ib, 0, 20, curs);
- int c = ib[0] & 0xff;
- final int typeCode = (c >> 4) & 7;
- long sz = c & 15;
- int shift = 4;
- int p = 1;
- while ((c & 0x80) != 0) {
- c = ib[p++] & 0xff;
- sz += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- switch (typeCode) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG: {
- if (delta != null || sz < curs.getStreamFileThreshold())
- data = decompress(pos + p, (int) sz, curs);
-
- if (delta != null) {
- type = typeCode;
- break SEARCH;
- }
-
- if (data != null)
- return new ObjectLoader.SmallObject(typeCode, data);
- else
- return new LargePackedWholeObject(typeCode, sz, pos, p,
- this, curs.db);
- }
-
- case Constants.OBJ_OFS_DELTA: {
- c = ib[p++] & 0xff;
- long base = c & 127;
- while ((c & 128) != 0) {
- base += 1;
- c = ib[p++] & 0xff;
- base <<= 7;
- base += (c & 127);
- }
- base = pos - base;
- delta = new Delta(delta, pos, (int) sz, p, base);
- if (sz != delta.deltaSize)
- break SEARCH;
-
- DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
- if (e != null) {
- type = e.type;
- data = e.data;
- cached = true;
- break SEARCH;
- }
- pos = base;
- continue SEARCH;
- }
-
- case Constants.OBJ_REF_DELTA: {
- readFully(pos + p, ib, 0, 20, curs);
- long base = findDeltaBase(ObjectId.fromRaw(ib));
- delta = new Delta(delta, pos, (int) sz, p + 20, base);
- if (sz != delta.deltaSize)
- break SEARCH;
-
- DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
- if (e != null) {
- type = e.type;
- data = e.data;
- cached = true;
- break SEARCH;
- }
- pos = base;
- continue SEARCH;
- }
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType,
- Integer.valueOf(typeCode)));
- }
- }
-
- // At this point there is at least one delta to apply to data.
- // (Whole objects with no deltas to apply return early above.)
-
- if (data == null)
- throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
-
- assert(delta != null);
- do {
- // Cache only the base immediately before desired object.
- if (cached)
- cached = false;
- else if (delta.next == null)
- curs.getDeltaBaseCache().store(this, delta.basePos, data, type);
-
- pos = delta.deltaPos;
-
- final byte[] cmds = decompress(pos + delta.hdrLen,
- delta.deltaSize, curs);
- if (cmds == null) {
- data = null; // Discard base in case of OutOfMemoryError
- throw new LargeObjectException.OutOfMemory(new OutOfMemoryError());
- }
-
- final long sz = BinaryDelta.getResultSize(cmds);
- if (Integer.MAX_VALUE <= sz)
- throw new LargeObjectException.ExceedsByteArrayLimit();
-
- final byte[] result;
- try {
- result = new byte[(int) sz];
- } catch (OutOfMemoryError tooBig) {
- data = null; // Discard base in case of OutOfMemoryError
- throw new LargeObjectException.OutOfMemory(tooBig);
- }
-
- BinaryDelta.apply(data, cmds, result);
- data = result;
- delta = delta.next;
- } while (delta != null);
-
- return new ObjectLoader.SmallObject(type, data);
-
- } catch (DataFormatException dfe) {
- CorruptObjectException coe = new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(pos), getPackFile()));
- coe.initCause(dfe);
- throw coe;
- }
- }
-
- private long findDeltaBase(ObjectId baseId) throws IOException,
- MissingObjectException {
- long ofs = idx().findOffset(baseId);
- if (ofs < 0)
- throw new MissingObjectException(baseId,
- JGitText.get().missingDeltaBase);
- return ofs;
- }
-
- private static class Delta {
- /** Child that applies onto this object. */
- final Delta next;
-
- /** Offset of the delta object. */
- final long deltaPos;
-
- /** Size of the inflated delta stream. */
- final int deltaSize;
-
- /** Total size of the delta's pack entry header (including base). */
- final int hdrLen;
-
- /** Offset of the base object this delta applies onto. */
- final long basePos;
-
- Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
- this.next = next;
- this.deltaPos = ofs;
- this.deltaSize = sz;
- this.hdrLen = hdrLen;
- this.basePos = baseOffset;
- }
- }
-
- byte[] getDeltaHeader(WindowCursor wc, long pos)
- throws IOException, DataFormatException {
- // The delta stream starts as two variable length integers. If we
- // assume they are 64 bits each, we need 16 bytes to encode them,
- // plus 2 extra bytes for the variable length overhead. So 18 is
- // the longest delta instruction header.
- //
- final byte[] hdr = new byte[18];
- wc.inflate(this, pos, hdr, true /* headerOnly */);
- return hdr;
- }
-
- int getObjectType(final WindowCursor curs, long pos) throws IOException {
- final byte[] ib = curs.tempId;
- for (;;) {
- readFully(pos, ib, 0, 20, curs);
- int c = ib[0] & 0xff;
- final int type = (c >> 4) & 7;
-
- switch (type) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return type;
-
- case Constants.OBJ_OFS_DELTA: {
- int p = 1;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
- c = ib[p++] & 0xff;
- long ofs = c & 127;
- while ((c & 128) != 0) {
- ofs += 1;
- c = ib[p++] & 0xff;
- ofs <<= 7;
- ofs += (c & 127);
- }
- pos = pos - ofs;
- continue;
- }
-
- case Constants.OBJ_REF_DELTA: {
- int p = 1;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
- readFully(pos + p, ib, 0, 20, curs);
- pos = findDeltaBase(ObjectId.fromRaw(ib));
- continue;
- }
-
- default:
- throw new IOException(
- MessageFormat.format(JGitText.get().unknownObjectType,
- Integer.valueOf(type)));
- }
- }
- }
-
- long getObjectSize(final WindowCursor curs, final AnyObjectId id)
- throws IOException {
- final long offset = idx().findOffset(id);
- return 0 < offset ? getObjectSize(curs, offset) : -1;
- }
-
- long getObjectSize(final WindowCursor curs, final long pos)
- throws IOException {
- final byte[] ib = curs.tempId;
- readFully(pos, ib, 0, 20, curs);
- int c = ib[0] & 0xff;
- final int type = (c >> 4) & 7;
- long sz = c & 15;
- int shift = 4;
- int p = 1;
- while ((c & 0x80) != 0) {
- c = ib[p++] & 0xff;
- sz += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- long deltaAt;
- switch (type) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return sz;
-
- case Constants.OBJ_OFS_DELTA:
- c = ib[p++] & 0xff;
- while ((c & 128) != 0)
- c = ib[p++] & 0xff;
- deltaAt = pos + p;
- break;
-
- case Constants.OBJ_REF_DELTA:
- deltaAt = pos + p + 20;
- break;
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType, Integer.valueOf(type)));
- }
-
- try {
- return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt));
- } catch (DataFormatException e) {
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
- getPackFile()));
- }
- }
-
- LocalObjectRepresentation representation(final WindowCursor curs,
- final AnyObjectId objectId) throws IOException {
- final long pos = idx().findOffset(objectId);
- if (pos < 0)
- return null;
-
- final byte[] ib = curs.tempId;
- readFully(pos, ib, 0, 20, curs);
- int c = ib[0] & 0xff;
- int p = 1;
- final int typeCode = (c >> 4) & 7;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
-
- long len = (findEndOffset(pos) - pos);
- switch (typeCode) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return LocalObjectRepresentation.newWhole(this, pos, len - p);
-
- case Constants.OBJ_OFS_DELTA: {
- c = ib[p++] & 0xff;
- long ofs = c & 127;
- while ((c & 128) != 0) {
- ofs += 1;
- c = ib[p++] & 0xff;
- ofs <<= 7;
- ofs += (c & 127);
- }
- ofs = pos - ofs;
- return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
- }
-
- case Constants.OBJ_REF_DELTA: {
- len -= p;
- len -= Constants.OBJECT_ID_LENGTH;
- readFully(pos + p, ib, 0, 20, curs);
- ObjectId id = ObjectId.fromRaw(ib);
- return LocalObjectRepresentation.newDelta(this, pos, len, id);
- }
-
- default:
- throw new IOException(
- MessageFormat.format(JGitText.get().unknownObjectType,
- Integer.valueOf(typeCode)));
- }
- }
-
- private long findEndOffset(final long startOffset)
- throws IOException, CorruptObjectException {
- final long maxOffset = length - 20;
- return getReverseIdx().findNextOffset(startOffset, maxOffset);
- }
-
- synchronized PackBitmapIndex getBitmapIndex() throws IOException {
- if (invalid || invalidBitmap)
- return null;
- if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
- final PackBitmapIndex idx;
- try {
- idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(),
- getReverseIdx());
- } catch (FileNotFoundException e) {
- // Once upon a time this bitmap file existed. Now it
- // has been removed. Most likely an external gc has
- // removed this packfile and the bitmap
- invalidBitmap = true;
- return null;
- }
-
- // At this point, idx() will have set packChecksum.
- if (Arrays.equals(packChecksum, idx.packChecksum))
- bitmapIdx = idx;
- else
- invalidBitmap = true;
- }
- return bitmapIdx;
- }
-
- private synchronized PackReverseIndex getReverseIdx() throws IOException {
- if (reverseIdx == null)
- reverseIdx = new PackReverseIndex(idx());
- return reverseIdx;
- }
-
- private boolean isCorrupt(long offset) {
- LongList list = corruptObjects;
- if (list == null)
- return false;
- synchronized (list) {
- return list.contains(offset);
- }
- }
-
- private void setCorrupt(long offset) {
- LongList list = corruptObjects;
- if (list == null) {
- synchronized (readLock) {
- list = corruptObjects;
- if (list == null) {
- list = new LongList();
- corruptObjects = list;
- }
- }
- }
- synchronized (list) {
- list.add(offset);
- }
- }
-
- private File extFile(PackExt ext) {
- String p = packFile.getName();
- int dot = p.lastIndexOf('.');
- String b = (dot < 0) ? p : p.substring(0, dot);
- return new File(packFile.getParentFile(), b + '.' + ext.getExtension());
- }
-
- private boolean hasExt(PackExt ext) {
- return (extensions & ext.getBit()) != 0;
- }
- }
|