1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216 |
- /*
- * Copyright (C) 2008-2011, Google Inc.
- * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org>
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.internal.storage.dfs;
-
- import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
- import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
- import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
- import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
-
- import java.io.BufferedInputStream;
- import java.io.EOFException;
- import java.io.IOException;
- import java.io.InputStream;
- import java.nio.ByteBuffer;
- import java.nio.channels.Channels;
- import java.text.MessageFormat;
- import java.util.Set;
- import java.util.zip.CRC32;
- import java.util.zip.DataFormatException;
- import java.util.zip.Inflater;
-
- import org.eclipse.jgit.errors.CorruptObjectException;
- import org.eclipse.jgit.errors.LargeObjectException;
- import org.eclipse.jgit.errors.MissingObjectException;
- import org.eclipse.jgit.errors.PackInvalidException;
- import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
- import org.eclipse.jgit.internal.JGitText;
- import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
- import org.eclipse.jgit.internal.storage.file.PackIndex;
- import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
- import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
- import org.eclipse.jgit.internal.storage.pack.PackExt;
- import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
- import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
- import org.eclipse.jgit.lib.AbbreviatedObjectId;
- import org.eclipse.jgit.lib.AnyObjectId;
- import org.eclipse.jgit.lib.Constants;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.lib.ObjectLoader;
- import org.eclipse.jgit.lib.Repository;
- import org.eclipse.jgit.util.LongList;
-
- /**
- * A Git version 2 pack file representation. A pack file contains Git objects in
- * delta packed format yielding high compression of lots of object where some
- * objects are similar.
- */
- public final class DfsPackFile {
- /**
- * File offset used to cache {@link #index} in {@link DfsBlockCache}.
- * <p>
- * To better manage memory, the forward index is stored as a single block in
- * the block cache under this file position. A negative value is used
- * because it cannot occur in a normal pack file, and it is less likely to
- * collide with a valid data block from the file as the high bits will all
- * be set when treated as an unsigned long by the cache code.
- */
- private static final long POS_INDEX = -1;
-
- /** Offset used to cache {@link #reverseIndex}. See {@link #POS_INDEX}. */
- private static final long POS_REVERSE_INDEX = -2;
-
- /** Offset used to cache {@link #bitmapIndex}. See {@link #POS_INDEX}. */
- private static final long POS_BITMAP_INDEX = -3;
-
- /** Cache that owns this pack file and its data. */
- private final DfsBlockCache cache;
-
- /** Description of the pack file's storage. */
- private final DfsPackDescription packDesc;
-
- /** Unique identity of this pack while in-memory. */
- final DfsPackKey key;
-
- /**
- * Total number of bytes in this pack file.
- * <p>
- * This field initializes to -1 and gets populated when a block is loaded.
- */
- volatile long length;
-
- /**
- * Preferred alignment for loading blocks from the backing file.
- * <p>
- * It is initialized to 0 and filled in on the first read made from the
- * file. Block sizes may be odd, e.g. 4091, caused by the underling DFS
- * storing 4091 user bytes and 5 bytes block metadata into a lower level
- * 4096 byte block on disk.
- */
- private volatile int blockSize;
-
- /** True once corruption has been detected that cannot be worked around. */
- private volatile boolean invalid;
-
- /**
- * Lock for initialization of {@link #index} and {@link #corruptObjects}.
- * <p>
- * This lock ensures only one thread can perform the initialization work.
- */
- private final Object initLock = new Object();
-
- /** Index mapping {@link ObjectId} to position within the pack stream. */
- private volatile DfsBlockCache.Ref<PackIndex> index;
-
- /** Reverse version of {@link #index} mapping position to {@link ObjectId}. */
- private volatile DfsBlockCache.Ref<PackReverseIndex> reverseIndex;
-
- /** Index of compressed bitmap mapping entire object graph. */
- private volatile DfsBlockCache.Ref<PackBitmapIndex> bitmapIndex;
-
- /**
- * Objects we have tried to read, and discovered to be corrupt.
- * <p>
- * The list is allocated after the first corruption is found, and filled in
- * as more entries are discovered. Typically this list is never used, as
- * pack files do not usually contain corrupt objects.
- */
- private volatile LongList corruptObjects;
-
- /**
- * Construct a reader for an existing, packfile.
- *
- * @param cache
- * cache that owns the pack data.
- * @param desc
- * description of the pack within the DFS.
- * @param key
- * interned key used to identify blocks in the block cache.
- */
- DfsPackFile(DfsBlockCache cache, DfsPackDescription desc, DfsPackKey key) {
- this.cache = cache;
- this.packDesc = desc;
- this.key = key;
-
- length = desc.getFileSize(PACK);
- if (length <= 0)
- length = -1;
- }
-
- /** @return description that was originally used to configure this pack file. */
- public DfsPackDescription getPackDescription() {
- return packDesc;
- }
-
- /**
- * @return whether the pack index file is loaded and cached in memory.
- * @since 2.2
- */
- public boolean isIndexLoaded() {
- DfsBlockCache.Ref<PackIndex> idxref = index;
- return idxref != null && idxref.has();
- }
-
- /** @return bytes cached in memory for this pack, excluding the index. */
- public long getCachedSize() {
- return key.cachedSize.get();
- }
-
- String getPackName() {
- return packDesc.getFileName(PACK);
- }
-
- void setBlockSize(int newSize) {
- blockSize = newSize;
- }
-
- void setPackIndex(PackIndex idx) {
- long objCnt = idx.getObjectCount();
- int recSize = Constants.OBJECT_ID_LENGTH + 8;
- int sz = (int) Math.min(objCnt * recSize, Integer.MAX_VALUE);
- index = cache.put(key, POS_INDEX, sz, idx);
- }
-
- /**
- * Get the PackIndex for this PackFile.
- *
- * @param ctx
- * reader context to support reading from the backing store if
- * the index is not already loaded in memory.
- * @return the PackIndex.
- * @throws IOException
- * the pack index is not available, or is corrupt.
- */
- public PackIndex getPackIndex(DfsReader ctx) throws IOException {
- return idx(ctx);
- }
-
- private PackIndex idx(DfsReader ctx) throws IOException {
- DfsBlockCache.Ref<PackIndex> idxref = index;
- if (idxref != null) {
- PackIndex idx = idxref.get();
- if (idx != null)
- return idx;
- }
-
- if (invalid)
- throw new PackInvalidException(getPackName());
-
- Repository.getGlobalListenerList()
- .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
-
- synchronized (initLock) {
- idxref = index;
- if (idxref != null) {
- PackIndex idx = idxref.get();
- if (idx != null)
- return idx;
- }
-
- PackIndex idx;
- try {
- ReadableChannel rc = ctx.db.openFile(packDesc, INDEX);
- try {
- InputStream in = Channels.newInputStream(rc);
- int wantSize = 8192;
- int bs = rc.blockSize();
- if (0 < bs && bs < wantSize)
- bs = (wantSize / bs) * bs;
- else if (bs <= 0)
- bs = wantSize;
- in = new BufferedInputStream(in, bs);
- idx = PackIndex.read(in);
- } finally {
- rc.close();
- }
- } catch (EOFException e) {
- invalid = true;
- IOException e2 = new IOException(MessageFormat.format(
- DfsText.get().shortReadOfIndex,
- packDesc.getFileName(INDEX)));
- e2.initCause(e);
- throw e2;
- } catch (IOException e) {
- invalid = true;
- IOException e2 = new IOException(MessageFormat.format(
- DfsText.get().cannotReadIndex,
- packDesc.getFileName(INDEX)));
- e2.initCause(e);
- throw e2;
- }
-
- setPackIndex(idx);
- return idx;
- }
- }
-
- final boolean isGarbage() {
- return packDesc.getPackSource() == UNREACHABLE_GARBAGE;
- }
-
- PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
- if (invalid || isGarbage())
- return null;
- DfsBlockCache.Ref<PackBitmapIndex> idxref = bitmapIndex;
- if (idxref != null) {
- PackBitmapIndex idx = idxref.get();
- if (idx != null)
- return idx;
- }
-
- if (!packDesc.hasFileExt(PackExt.BITMAP_INDEX))
- return null;
-
- synchronized (initLock) {
- idxref = bitmapIndex;
- if (idxref != null) {
- PackBitmapIndex idx = idxref.get();
- if (idx != null)
- return idx;
- }
-
- long size;
- PackBitmapIndex idx;
- try {
- ReadableChannel rc = ctx.db.openFile(packDesc, BITMAP_INDEX);
- try {
- InputStream in = Channels.newInputStream(rc);
- int wantSize = 8192;
- int bs = rc.blockSize();
- if (0 < bs && bs < wantSize)
- bs = (wantSize / bs) * bs;
- else if (bs <= 0)
- bs = wantSize;
- in = new BufferedInputStream(in, bs);
- idx = PackBitmapIndex.read(
- in, idx(ctx), getReverseIdx(ctx));
- } finally {
- size = rc.position();
- rc.close();
- }
- } catch (EOFException e) {
- IOException e2 = new IOException(MessageFormat.format(
- DfsText.get().shortReadOfIndex,
- packDesc.getFileName(BITMAP_INDEX)));
- e2.initCause(e);
- throw e2;
- } catch (IOException e) {
- IOException e2 = new IOException(MessageFormat.format(
- DfsText.get().cannotReadIndex,
- packDesc.getFileName(BITMAP_INDEX)));
- e2.initCause(e);
- throw e2;
- }
-
- bitmapIndex = cache.put(key, POS_BITMAP_INDEX,
- (int) Math.min(size, Integer.MAX_VALUE), idx);
- return idx;
- }
- }
-
- PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
- DfsBlockCache.Ref<PackReverseIndex> revref = reverseIndex;
- if (revref != null) {
- PackReverseIndex revidx = revref.get();
- if (revidx != null)
- return revidx;
- }
-
- synchronized (initLock) {
- revref = reverseIndex;
- if (revref != null) {
- PackReverseIndex revidx = revref.get();
- if (revidx != null)
- return revidx;
- }
-
- PackIndex idx = idx(ctx);
- PackReverseIndex revidx = new PackReverseIndex(idx);
- int sz = (int) Math.min(
- idx.getObjectCount() * 8, Integer.MAX_VALUE);
- reverseIndex = cache.put(key, POS_REVERSE_INDEX, sz, revidx);
- return revidx;
- }
- }
-
- /**
- * Check if an object is stored within this pack.
- *
- * @param ctx
- * reader context to support reading from the backing store if
- * the index is not already loaded in memory.
- * @param id
- * object to be located.
- * @return true if the object exists in this pack; false if it does not.
- * @throws IOException
- * the pack index is not available, or is corrupt.
- */
- public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
- final long offset = idx(ctx).findOffset(id);
- return 0 < offset && !isCorrupt(offset);
- }
-
- /**
- * Get an object from this pack.
- *
- * @param ctx
- * temporary working space associated with the calling thread.
- * @param id
- * the object to obtain from the pack. Must not be null.
- * @return the object loader for the requested object if it is contained in
- * this pack; null if the object was not found.
- * @throws IOException
- * the pack file or the index could not be read.
- */
- ObjectLoader get(DfsReader ctx, AnyObjectId id)
- throws IOException {
- long offset = idx(ctx).findOffset(id);
- return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
- }
-
- long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
- return idx(ctx).findOffset(id);
- }
-
- void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
- int matchLimit) throws IOException {
- idx(ctx).resolve(matches, id, matchLimit);
- }
-
- /** Release all memory used by this DfsPackFile instance. */
- public void close() {
- cache.remove(this);
- index = null;
- reverseIndex = null;
- }
-
- /**
- * Obtain the total number of objects available in this pack. This method
- * relies on pack index, giving number of effectively available objects.
- *
- * @param ctx
- * current reader for the calling thread.
- * @return number of objects in index of this pack, likewise in this pack
- * @throws IOException
- * the index file cannot be loaded into memory.
- */
- long getObjectCount(DfsReader ctx) throws IOException {
- return idx(ctx).getObjectCount();
- }
-
- private byte[] decompress(long position, int sz, DfsReader ctx)
- throws IOException, DataFormatException {
- byte[] dstbuf;
- try {
- dstbuf = new byte[sz];
- } catch (OutOfMemoryError noMemory) {
- // The size may be larger than our heap allows, return null to
- // let the caller know allocation isn't possible and it should
- // use the large object streaming approach instead.
- //
- // For example, this can occur when sz is 640 MB, and JRE
- // maximum heap size is only 256 MB. Even if the JRE has
- // 200 MB free, it cannot allocate a 640 MB byte array.
- return null;
- }
-
- if (ctx.inflate(this, position, dstbuf, false) != sz)
- throw new EOFException(MessageFormat.format(
- JGitText.get().shortCompressedStreamAt,
- Long.valueOf(position)));
- return dstbuf;
- }
-
- void copyPackAsIs(PackOutputStream out, DfsReader ctx)
- throws IOException {
- // If the length hasn't been determined yet, pin to set it.
- if (length == -1) {
- ctx.pin(this, 0);
- ctx.unpin();
- }
- if (cache.shouldCopyThroughCache(length))
- copyPackThroughCache(out, ctx);
- else
- copyPackBypassCache(out, ctx);
- }
-
- private void copyPackThroughCache(PackOutputStream out, DfsReader ctx)
- throws IOException {
- long position = 12;
- long remaining = length - (12 + 20);
- while (0 < remaining) {
- DfsBlock b = cache.getOrLoad(this, position, ctx);
- int ptr = (int) (position - b.start);
- int n = (int) Math.min(b.size() - ptr, remaining);
- b.write(out, position, n);
- position += n;
- remaining -= n;
- }
- }
-
- private long copyPackBypassCache(PackOutputStream out, DfsReader ctx)
- throws IOException {
- try (ReadableChannel rc = ctx.db.openFile(packDesc, PACK)) {
- ByteBuffer buf = newCopyBuffer(out, rc);
- if (ctx.getOptions().getStreamPackBufferSize() > 0)
- rc.setReadAheadBytes(ctx.getOptions().getStreamPackBufferSize());
- long position = 12;
- long remaining = length - (12 + 20);
- while (0 < remaining) {
- DfsBlock b = cache.get(key, alignToBlock(position));
- if (b != null) {
- int ptr = (int) (position - b.start);
- int n = (int) Math.min(b.size() - ptr, remaining);
- b.write(out, position, n);
- position += n;
- remaining -= n;
- rc.position(position);
- continue;
- }
-
- buf.position(0);
- int n = read(rc, buf);
- if (n <= 0)
- throw packfileIsTruncated();
- else if (n > remaining)
- n = (int) remaining;
- out.write(buf.array(), 0, n);
- position += n;
- remaining -= n;
- }
- return position;
- }
- }
-
- private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
- int bs = blockSize(rc);
- byte[] copyBuf = out.getCopyBuffer();
- if (bs > copyBuf.length)
- copyBuf = new byte[bs];
- return ByteBuffer.wrap(copyBuf, 0, bs);
- }
-
- void copyAsIs(PackOutputStream out, DfsObjectToPack src,
- boolean validate, DfsReader ctx) throws IOException,
- StoredObjectRepresentationNotAvailableException {
- final CRC32 crc1 = validate ? new CRC32() : null;
- final CRC32 crc2 = validate ? new CRC32() : null;
- final byte[] buf = out.getCopyBuffer();
-
- // Rip apart the header so we can discover the size.
- //
- try {
- readFully(src.offset, buf, 0, 20, ctx);
- } catch (IOException ioError) {
- StoredObjectRepresentationNotAvailableException gone;
- gone = new StoredObjectRepresentationNotAvailableException(src);
- gone.initCause(ioError);
- throw gone;
- }
- int c = buf[0] & 0xff;
- final int typeCode = (c >> 4) & 7;
- long inflatedLength = c & 15;
- int shift = 4;
- int headerCnt = 1;
- while ((c & 0x80) != 0) {
- c = buf[headerCnt++] & 0xff;
- inflatedLength += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- if (typeCode == Constants.OBJ_OFS_DELTA) {
- do {
- c = buf[headerCnt++] & 0xff;
- } while ((c & 128) != 0);
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
- } else if (typeCode == Constants.OBJ_REF_DELTA) {
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
-
- readFully(src.offset + headerCnt, buf, 0, 20, ctx);
- if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, 20);
- crc2.update(buf, 0, 20);
- }
- headerCnt += 20;
- } else if (validate) {
- assert(crc1 != null && crc2 != null);
- crc1.update(buf, 0, headerCnt);
- crc2.update(buf, 0, headerCnt);
- }
-
- final long dataOffset = src.offset + headerCnt;
- final long dataLength = src.length;
- final long expectedCRC;
- final DfsBlock quickCopy;
-
- // Verify the object isn't corrupt before sending. If it is,
- // we report it missing instead.
- //
- try {
- quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
-
- if (validate && idx(ctx).hasCRC32Support()) {
- assert(crc1 != null);
- // Index has the CRC32 code cached, validate the object.
- //
- expectedCRC = idx(ctx).findCRC32(src);
- if (quickCopy != null) {
- quickCopy.crc32(crc1, dataOffset, (int) dataLength);
- } else {
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, ctx);
- crc1.update(buf, 0, n);
- pos += n;
- cnt -= n;
- }
- }
- if (crc1.getValue() != expectedCRC) {
- setCorrupt(src.offset);
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackName()));
- }
- } else if (validate) {
- assert(crc1 != null);
- // We don't have a CRC32 code in the index, so compute it
- // now while inflating the raw data to get zlib to tell us
- // whether or not the data is safe.
- //
- Inflater inf = ctx.inflater();
- byte[] tmp = new byte[1024];
- if (quickCopy != null) {
- quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
- } else {
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, ctx);
- crc1.update(buf, 0, n);
- inf.setInput(buf, 0, n);
- while (inf.inflate(tmp, 0, tmp.length) > 0)
- continue;
- pos += n;
- cnt -= n;
- }
- }
- if (!inf.finished() || inf.getBytesRead() != dataLength) {
- setCorrupt(src.offset);
- throw new EOFException(MessageFormat.format(
- JGitText.get().shortCompressedStreamAt,
- Long.valueOf(src.offset)));
- }
- expectedCRC = crc1.getValue();
- } else {
- expectedCRC = -1;
- }
- } catch (DataFormatException dataFormat) {
- setCorrupt(src.offset);
-
- CorruptObjectException corruptObject = new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackName()));
- corruptObject.initCause(dataFormat);
-
- StoredObjectRepresentationNotAvailableException gone;
- gone = new StoredObjectRepresentationNotAvailableException(src);
- gone.initCause(corruptObject);
- throw gone;
-
- } catch (IOException ioError) {
- StoredObjectRepresentationNotAvailableException gone;
- gone = new StoredObjectRepresentationNotAvailableException(src);
- gone.initCause(ioError);
- throw gone;
- }
-
- if (quickCopy != null) {
- // The entire object fits into a single byte array window slice,
- // and we have it pinned. Write this out without copying.
- //
- out.writeHeader(src, inflatedLength);
- quickCopy.write(out, dataOffset, (int) dataLength);
-
- } else if (dataLength <= buf.length) {
- // Tiny optimization: Lots of objects are very small deltas or
- // deflated commits that are likely to fit in the copy buffer.
- //
- if (!validate) {
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, ctx);
- pos += n;
- cnt -= n;
- }
- }
- out.writeHeader(src, inflatedLength);
- out.write(buf, 0, (int) dataLength);
- } else {
- // Now we are committed to sending the object. As we spool it out,
- // check its CRC32 code to make sure there wasn't corruption between
- // the verification we did above, and us actually outputting it.
- //
- out.writeHeader(src, inflatedLength);
- long pos = dataOffset;
- long cnt = dataLength;
- while (cnt > 0) {
- final int n = (int) Math.min(cnt, buf.length);
- readFully(pos, buf, 0, n, ctx);
- if (validate) {
- assert(crc2 != null);
- crc2.update(buf, 0, n);
- }
- out.write(buf, 0, n);
- pos += n;
- cnt -= n;
- }
- if (validate) {
- assert(crc2 != null);
- if (crc2.getValue() != expectedCRC) {
- throw new CorruptObjectException(MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(src.offset), getPackName()));
- }
- }
- }
- }
-
- boolean invalid() {
- return invalid;
- }
-
- void setInvalid() {
- invalid = true;
- }
-
- private IOException packfileIsTruncated() {
- invalid = true;
- return new IOException(MessageFormat.format(
- JGitText.get().packfileIsTruncated, getPackName()));
- }
-
- private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
- DfsReader ctx) throws IOException {
- if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt)
- throw new EOFException();
- }
-
- long alignToBlock(long pos) {
- int size = blockSize;
- if (size == 0)
- size = cache.getBlockSize();
- return (pos / size) * size;
- }
-
- DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
- return cache.getOrLoad(this, pos, ctx);
- }
-
- DfsBlock readOneBlock(long pos, DfsReader ctx)
- throws IOException {
- if (invalid)
- throw new PackInvalidException(getPackName());
-
- ReadableChannel rc = ctx.db.openFile(packDesc, PACK);
- try {
- int size = blockSize(rc);
- pos = (pos / size) * size;
-
- // If the size of the file is not yet known, try to discover it.
- // Channels may choose to return -1 to indicate they don't
- // know the length yet, in this case read up to the size unit
- // given by the caller, then recheck the length.
- long len = length;
- if (len < 0) {
- len = rc.size();
- if (0 <= len)
- length = len;
- }
-
- if (0 <= len && len < pos + size)
- size = (int) (len - pos);
- if (size <= 0)
- throw new EOFException(MessageFormat.format(
- DfsText.get().shortReadOfBlock, Long.valueOf(pos),
- getPackName(), Long.valueOf(0), Long.valueOf(0)));
-
- byte[] buf = new byte[size];
- rc.position(pos);
- int cnt = read(rc, ByteBuffer.wrap(buf, 0, size));
- if (cnt != size) {
- if (0 <= len) {
- throw new EOFException(MessageFormat.format(
- DfsText.get().shortReadOfBlock,
- Long.valueOf(pos),
- getPackName(),
- Integer.valueOf(size),
- Integer.valueOf(cnt)));
- }
-
- // Assume the entire thing was read in a single shot, compact
- // the buffer to only the space required.
- byte[] n = new byte[cnt];
- System.arraycopy(buf, 0, n, 0, n.length);
- buf = n;
- } else if (len < 0) {
- // With no length at the start of the read, the channel should
- // have the length available at the end.
- length = len = rc.size();
- }
-
- DfsBlock v = new DfsBlock(key, pos, buf);
- return v;
- } finally {
- rc.close();
- }
- }
-
- private int blockSize(ReadableChannel rc) {
- // If the block alignment is not yet known, discover it. Prefer the
- // larger size from either the cache or the file itself.
- int size = blockSize;
- if (size == 0) {
- size = rc.blockSize();
- if (size <= 0)
- size = cache.getBlockSize();
- else if (size < cache.getBlockSize())
- size = (cache.getBlockSize() / size) * size;
- blockSize = size;
- }
- return size;
- }
-
- private static int read(ReadableChannel rc, ByteBuffer buf)
- throws IOException {
- int n;
- do {
- n = rc.read(buf);
- } while (0 < n && buf.hasRemaining());
- return buf.position();
- }
-
- ObjectLoader load(DfsReader ctx, long pos)
- throws IOException {
- try {
- final byte[] ib = ctx.tempId;
- Delta delta = null;
- byte[] data = null;
- int type = Constants.OBJ_BAD;
- boolean cached = false;
-
- SEARCH: for (;;) {
- readFully(pos, ib, 0, 20, ctx);
- int c = ib[0] & 0xff;
- final int typeCode = (c >> 4) & 7;
- long sz = c & 15;
- int shift = 4;
- int p = 1;
- while ((c & 0x80) != 0) {
- c = ib[p++] & 0xff;
- sz += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- switch (typeCode) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG: {
- if (delta != null) {
- data = decompress(pos + p, (int) sz, ctx);
- type = typeCode;
- break SEARCH;
- }
-
- if (sz < ctx.getStreamFileThreshold()) {
- data = decompress(pos + p, (int) sz, ctx);
- if (data != null)
- return new ObjectLoader.SmallObject(typeCode, data);
- }
- return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
- }
-
- case Constants.OBJ_OFS_DELTA: {
- c = ib[p++] & 0xff;
- long base = c & 127;
- while ((c & 128) != 0) {
- base += 1;
- c = ib[p++] & 0xff;
- base <<= 7;
- base += (c & 127);
- }
- base = pos - base;
- delta = new Delta(delta, pos, (int) sz, p, base);
- if (sz != delta.deltaSize)
- break SEARCH;
-
- DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
- if (e != null) {
- type = e.type;
- data = e.data;
- cached = true;
- break SEARCH;
- }
- pos = base;
- continue SEARCH;
- }
-
- case Constants.OBJ_REF_DELTA: {
- readFully(pos + p, ib, 0, 20, ctx);
- long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
- delta = new Delta(delta, pos, (int) sz, p + 20, base);
- if (sz != delta.deltaSize)
- break SEARCH;
-
- DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
- if (e != null) {
- type = e.type;
- data = e.data;
- cached = true;
- break SEARCH;
- }
- pos = base;
- continue SEARCH;
- }
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
- }
- }
-
- // At this point there is at least one delta to apply to data.
- // (Whole objects with no deltas to apply return early above.)
-
- if (data == null)
- throw new LargeObjectException();
-
- assert(delta != null);
- do {
- // Cache only the base immediately before desired object.
- if (cached)
- cached = false;
- else if (delta.next == null)
- ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
-
- pos = delta.deltaPos;
-
- byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
- if (cmds == null) {
- data = null; // Discard base in case of OutOfMemoryError
- throw new LargeObjectException();
- }
-
- final long sz = BinaryDelta.getResultSize(cmds);
- if (Integer.MAX_VALUE <= sz)
- throw new LargeObjectException.ExceedsByteArrayLimit();
-
- final byte[] result;
- try {
- result = new byte[(int) sz];
- } catch (OutOfMemoryError tooBig) {
- data = null; // Discard base in case of OutOfMemoryError
- cmds = null;
- throw new LargeObjectException.OutOfMemory(tooBig);
- }
-
- BinaryDelta.apply(data, cmds, result);
- data = result;
- delta = delta.next;
- } while (delta != null);
-
- return new ObjectLoader.SmallObject(type, data);
-
- } catch (DataFormatException dfe) {
- CorruptObjectException coe = new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
- getPackName()));
- coe.initCause(dfe);
- throw coe;
- }
- }
-
- private long findDeltaBase(DfsReader ctx, ObjectId baseId)
- throws IOException, MissingObjectException {
- long ofs = idx(ctx).findOffset(baseId);
- if (ofs < 0)
- throw new MissingObjectException(baseId,
- JGitText.get().missingDeltaBase);
- return ofs;
- }
-
- private static class Delta {
- /** Child that applies onto this object. */
- final Delta next;
-
- /** Offset of the delta object. */
- final long deltaPos;
-
- /** Size of the inflated delta stream. */
- final int deltaSize;
-
- /** Total size of the delta's pack entry header (including base). */
- final int hdrLen;
-
- /** Offset of the base object this delta applies onto. */
- final long basePos;
-
- Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
- this.next = next;
- this.deltaPos = ofs;
- this.deltaSize = sz;
- this.hdrLen = hdrLen;
- this.basePos = baseOffset;
- }
- }
-
- byte[] getDeltaHeader(DfsReader wc, long pos)
- throws IOException, DataFormatException {
- // The delta stream starts as two variable length integers. If we
- // assume they are 64 bits each, we need 16 bytes to encode them,
- // plus 2 extra bytes for the variable length overhead. So 18 is
- // the longest delta instruction header.
- //
- final byte[] hdr = new byte[32];
- wc.inflate(this, pos, hdr, true /* header only */);
- return hdr;
- }
-
- int getObjectType(DfsReader ctx, long pos) throws IOException {
- final byte[] ib = ctx.tempId;
- for (;;) {
- readFully(pos, ib, 0, 20, ctx);
- int c = ib[0] & 0xff;
- final int type = (c >> 4) & 7;
-
- switch (type) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return type;
-
- case Constants.OBJ_OFS_DELTA: {
- int p = 1;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
- c = ib[p++] & 0xff;
- long ofs = c & 127;
- while ((c & 128) != 0) {
- ofs += 1;
- c = ib[p++] & 0xff;
- ofs <<= 7;
- ofs += (c & 127);
- }
- pos = pos - ofs;
- continue;
- }
-
- case Constants.OBJ_REF_DELTA: {
- int p = 1;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
- readFully(pos + p, ib, 0, 20, ctx);
- pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
- continue;
- }
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType, Integer.valueOf(type)));
- }
- }
- }
-
- long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
- final long offset = idx(ctx).findOffset(id);
- return 0 < offset ? getObjectSize(ctx, offset) : -1;
- }
-
- long getObjectSize(DfsReader ctx, long pos)
- throws IOException {
- final byte[] ib = ctx.tempId;
- readFully(pos, ib, 0, 20, ctx);
- int c = ib[0] & 0xff;
- final int type = (c >> 4) & 7;
- long sz = c & 15;
- int shift = 4;
- int p = 1;
- while ((c & 0x80) != 0) {
- c = ib[p++] & 0xff;
- sz += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- long deltaAt;
- switch (type) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- return sz;
-
- case Constants.OBJ_OFS_DELTA:
- c = ib[p++] & 0xff;
- while ((c & 128) != 0)
- c = ib[p++] & 0xff;
- deltaAt = pos + p;
- break;
-
- case Constants.OBJ_REF_DELTA:
- deltaAt = pos + p + 20;
- break;
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType, Integer.valueOf(type)));
- }
-
- try {
- return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
- } catch (DataFormatException dfe) {
- CorruptObjectException coe = new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
- getPackName()));
- coe.initCause(dfe);
- throw coe;
- }
- }
-
- void representation(DfsObjectRepresentation r, final long pos,
- DfsReader ctx, PackReverseIndex rev)
- throws IOException {
- r.offset = pos;
- final byte[] ib = ctx.tempId;
- readFully(pos, ib, 0, 20, ctx);
- int c = ib[0] & 0xff;
- int p = 1;
- final int typeCode = (c >> 4) & 7;
- while ((c & 0x80) != 0)
- c = ib[p++] & 0xff;
-
- long len = rev.findNextOffset(pos, length - 20) - pos;
- switch (typeCode) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- r.format = StoredObjectRepresentation.PACK_WHOLE;
- r.baseId = null;
- r.length = len - p;
- return;
-
- case Constants.OBJ_OFS_DELTA: {
- c = ib[p++] & 0xff;
- long ofs = c & 127;
- while ((c & 128) != 0) {
- ofs += 1;
- c = ib[p++] & 0xff;
- ofs <<= 7;
- ofs += (c & 127);
- }
- r.format = StoredObjectRepresentation.PACK_DELTA;
- r.baseId = rev.findObject(pos - ofs);
- r.length = len - p;
- return;
- }
-
- case Constants.OBJ_REF_DELTA: {
- readFully(pos + p, ib, 0, 20, ctx);
- r.format = StoredObjectRepresentation.PACK_DELTA;
- r.baseId = ObjectId.fromRaw(ib);
- r.length = len - p - 20;
- return;
- }
-
- default:
- throw new IOException(MessageFormat.format(
- JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
- }
- }
-
- private boolean isCorrupt(long offset) {
- LongList list = corruptObjects;
- if (list == null)
- return false;
- synchronized (list) {
- return list.contains(offset);
- }
- }
-
- private void setCorrupt(long offset) {
- LongList list = corruptObjects;
- if (list == null) {
- synchronized (initLock) {
- list = corruptObjects;
- if (list == null) {
- list = new LongList();
- corruptObjects = list;
- }
- }
- }
- synchronized (list) {
- list.add(offset);
- }
- }
- }
|