diff options
Diffstat (limited to 'org.eclipse.jgit/src/org/eclipse/jgit')
10 files changed, 346 insertions, 130 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index efd666ff27..1a873d1204 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -52,16 +52,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { boolean dirty() { return true; } - - @Override - void clearDirty() { - // Always dirty. - } - - @Override - public void markDirty() { - // Always dirty. - } }; /** @@ -534,7 +524,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length]; packs[0] = newPack; System.arraycopy(o.packs, 0, packs, 1, o.packs.length); - n = new PackListImpl(packs, o.reftables); + n = new PackList(packs, o.reftables); } while (!packList.compareAndSet(o, n)); } @@ -559,7 +549,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } } tables.add(new DfsReftable(add)); - n = new PackListImpl(o.packs, tables.toArray(new DfsReftable[0])); + n = new PackList(o.packs, tables.toArray(new DfsReftable[0])); } while (!packList.compareAndSet(o, n)); } @@ -613,13 +603,12 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } if (newPacks.isEmpty() && newReftables.isEmpty()) - return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables); + return new PackList(NO_PACKS.packs, NO_PACKS.reftables); if (!foundNew) { - old.clearDirty(); return old; } Collections.sort(newReftables, reftableComparator()); - return new PackListImpl( + return new PackList( newPacks.toArray(new DfsPackFile[0]), newReftables.toArray(new DfsReftable[0])); } @@ -685,7 +674,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } /** Snapshot of packs scanned in a single pass. */ - public abstract static class PackList { + public static class PackList { /** All known packs, sorted. */ public final DfsPackFile[] packs; @@ -715,39 +704,8 @@ public abstract class DfsObjDatabase extends ObjectDatabase { return lastModified; } - abstract boolean dirty(); - abstract void clearDirty(); - - /** - * Mark pack list as dirty. - * <p> - * Used when the caller knows that new data might have been written to the - * repository that could invalidate open readers depending on this pack list, - * for example if refs are newly scanned. - */ - public abstract void markDirty(); - } - - private static final class PackListImpl extends PackList { - private volatile boolean dirty; - - PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) { - super(packs, reftables); - } - - @Override boolean dirty() { - return dirty; - } - - @Override - void clearDirty() { - dirty = false; - } - - @Override - public void markDirty() { - dirty = true; + return false; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 618969f62e..05b63eaca1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -442,6 +442,10 @@ public final class DfsPackFile extends BlockBasedFile { return 0 < offset && !isCorrupt(offset); } + int findIdxPosition(DfsReader ctx, AnyObjectId id) throws IOException { + return idx(ctx).findPosition(id); + } + /** * Get an object from this pack. * @@ -1164,31 +1168,29 @@ public final class DfsPackFile extends BlockBasedFile { /** * Return the size of the object from the object-size index. The object * should be a blob. Any other type is not indexed and returns -1. - * - * Caller MUST be sure that the object is in the pack (e.g. with - * {@link #hasObject(DfsReader, AnyObjectId)}) and the pack has object size - * index (e.g. with {@link #hasObjectSizeIndex(DfsReader)}) before asking - * the indexed size. + * <p> + * Caller MUST pass a valid index position, as returned by + * {@link #findIdxPosition(DfsReader, AnyObjectId)} and verify the pack has + * object size index (e.g. with {@link #hasObjectSizeIndex(DfsReader)}) + * before asking the indexed size. * * @param ctx * reader context to support reading from the backing store if * the object size index is not already loaded in memory. - * @param id - * object id of an object in the pack + * @param idxPosition + * position in the primary index of the object we are looking + * for, as returned by findIdxPosition * @return size of the object from the index. Negative if object is not in * the index (below threshold or not a blob) * @throws IOException * could not read the object size index. IO problem or the pack * doesn't have it. */ - long getIndexedObjectSize(DfsReader ctx, AnyObjectId id) + long getIndexedObjectSize(DfsReader ctx, int idxPosition) throws IOException { - int idxPosition = idx(ctx).findPosition(id); if (idxPosition < 0) { - throw new IllegalArgumentException( - "Cannot get size from index since object is not in pack"); //$NON-NLS-1$ + throw new IllegalArgumentException("Invalid index position"); //$NON-NLS-1$ } - PackObjectSizeIndex sizeIdx = getObjectSizeIndex(ctx); if (sizeIdx == null) { throw new IllegalStateException( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 04288bc6c6..f50cd597e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -187,31 +187,44 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { @Override public boolean has(AnyObjectId objectId) throws IOException { + return findPack(objectId) >= 0; + } + + private int findPack(AnyObjectId objectId) throws IOException { if (last != null - && !skipGarbagePack(last) - && last.hasObject(this, objectId)) - return true; + && !skipGarbagePack(last)) { + int idxPos = last.findIdxPosition(this, objectId); + if (idxPos >= 0) { + return idxPos; + } + } + PackList packList = db.getPackList(); - if (hasImpl(packList, objectId)) { - return true; + int idxPos = findInPackList(packList, objectId); + if (idxPos >= 0) { + return idxPos; } else if (packList.dirty()) { stats.scanPacks++; - return hasImpl(db.scanPacks(packList), objectId); + idxPos = findInPackList(db.scanPacks(packList), objectId); + return idxPos; } - return false; + return -1; } - private boolean hasImpl(PackList packList, AnyObjectId objectId) + // Leave "last" pointing to the pack and return the idx position of the + // object (-1 if not found) + private int findInPackList(PackList packList, AnyObjectId objectId) throws IOException { for (DfsPackFile pack : packList.packs) { if (pack == last || skipGarbagePack(pack)) continue; - if (pack.hasObject(this, objectId)) { + int idxPos = pack.findIdxPosition(this, objectId); + if (idxPos >= 0) { last = pack; - return true; + return idxPos; } } - return false; + return -1; } @Override @@ -499,8 +512,8 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { public long getObjectSize(AnyObjectId objectId, int typeHint) throws MissingObjectException, IncorrectObjectTypeException, IOException { - DfsPackFile pack = findPackWithObject(objectId); - if (pack == null) { + int idxPos = findPack(objectId); + if (idxPos < 0) { if (typeHint == OBJ_ANY) { throw new MissingObjectException(objectId.copy(), JGitText.get().unknownObjectType2); @@ -508,16 +521,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { throw new MissingObjectException(objectId.copy(), typeHint); } - if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { - return pack.getObjectSize(this, objectId); + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(last)) { + return last.getObjectSize(this, objectId); } - Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); - long sz = maybeSz.orElse(-1L); + long sz = safeGetIndexedObjectSize(last, idxPos); if (sz >= 0) { return sz; } - return pack.getObjectSize(this, objectId); + return last.getObjectSize(this, objectId); } @@ -525,8 +537,8 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { public boolean isNotLargerThan(AnyObjectId objectId, int typeHint, long limit) throws MissingObjectException, IncorrectObjectTypeException, IOException { - DfsPackFile pack = findPackWithObject(objectId); - if (pack == null) { + int idxPos = findPack(objectId); + if (idxPos < 0) { if (typeHint == OBJ_ANY) { throw new MissingObjectException(objectId.copy(), JGitText.get().unknownObjectType2); @@ -535,28 +547,22 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } stats.isNotLargerThanCallCount += 1; - if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { - return pack.getObjectSize(this, objectId) <= limit; + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(last)) { + return last.getObjectSize(this, objectId) <= limit; } - Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); - if (maybeSz.isEmpty()) { - // Exception in object size index - return pack.getObjectSize(this, objectId) <= limit; - } - - long sz = maybeSz.get(); + long sz = safeGetIndexedObjectSize(last, idxPos); if (sz >= 0) { return sz <= limit; } - if (isLimitInsideIndexThreshold(pack, limit)) { + if (isLimitInsideIndexThreshold(last, limit)) { // With threshold T, not-found means object < T // If limit L > T, then object < T < L return true; } - return pack.getObjectSize(this, objectId) <= limit; + return last.getObjectSize(this, objectId) <= limit; } private boolean safeHasObjectSizeIndex(DfsPackFile pack) { @@ -567,21 +573,22 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } - private Optional<Long> safeGetIndexedObjectSize(DfsPackFile pack, - AnyObjectId objectId) { + private long safeGetIndexedObjectSize(DfsPackFile pack, + int idxPos) { long sz; try { - sz = pack.getIndexedObjectSize(this, objectId); + sz = pack.getIndexedObjectSize(this, idxPos); } catch (IOException e) { - // Do not count the exception as an index miss - return Optional.empty(); + // If there is any error in the index, we should have seen it + // on hasObjectSizeIndex. + throw new IllegalStateException(e); } if (sz < 0) { stats.objectSizeIndexMiss += 1; } else { stats.objectSizeIndexHit += 1; } - return Optional.of(sz); + return sz; } private boolean isLimitInsideIndexThreshold(DfsPackFile pack, long limit) { @@ -592,25 +599,6 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } - private DfsPackFile findPackWithObject(AnyObjectId objectId) - throws IOException { - if (last != null && !skipGarbagePack(last) - && last.hasObject(this, objectId)) { - return last; - } - PackList packList = db.getPackList(); - // hasImpl doesn't check "last", but leaves "last" pointing to the pack - // with the object - if (hasImpl(packList, objectId)) { - return last; - } else if (packList.dirty()) { - if (hasImpl(db.getPackList(), objectId)) { - return last; - } - } - return null; - } - @Override public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) { return new DfsObjectToPack(objectId, type); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index c08a92e5a7..97473bba2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; @@ -131,7 +132,7 @@ public class GC { private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP); private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX, - REVERSE_INDEX); + REVERSE_INDEX, OBJECT_SIZE_INDEX); private static final int DEFAULT_AUTOPACKLIMIT = 50; @@ -412,6 +413,10 @@ public class GC { */ private void removeOldPack(PackFile packFile, int deleteOptions) throws IOException { + if (!packFile.exists()) { + return; + } + if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); @@ -1340,6 +1345,7 @@ public class GC { idxChannel.force(true); } + if (pw.isReverseIndexEnabled()) { File tmpReverseIndexFile = new File(packdir, tmpBase + REVERSE_INDEX.getTmpExtension()); @@ -1356,6 +1362,19 @@ public class GC { .newOutputStream(channel)) { pw.writeReverseIndex(stream); channel.force(true); + } + } + + // write the object size + if (pconfig.isWriteObjSizeIndex()) { + File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$ + tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx); + try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx); + FileChannel idxChannel = fos.getChannel(); + OutputStream idxStream = Channels + .newOutputStream(idxChannel)) { + pw.writeObjectSizeIndex(idxStream); + idxChannel.force(true); } } @@ -1525,6 +1544,11 @@ public class GC { */ public long numberOfBitmaps; + /** + * The number of objects in the size-index of the packs + */ + public long numberOfSizeIndexedObjects; + @Override public String toString() { final StringBuilder b = new StringBuilder(); @@ -1540,6 +1564,8 @@ public class GC { b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$ b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$ + b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$ + .append(numberOfSizeIndexedObjects); return b.toString(); } } @@ -1548,7 +1574,7 @@ public class GC { * Returns information about objects and pack files for a FileRepository. * * @return information about objects and pack files for a FileRepository - * @throws java.io.IOException + * @throws java.io.IOException * if an IO error occurred */ public RepoStatistics getStatistics() throws IOException { @@ -1560,16 +1586,16 @@ public class GC { ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); + ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); if (latestBitmapTime == 0L) { latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); } - } - else if (latestBitmapTime == 0L) { - ret.numberOfPackFilesSinceBitmap++; - ret.numberOfObjectsSinceBitmap += packedObjects; - } + } else if (latestBitmapTime == 0L) { + ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 746e124e1f..d97d5a7ccd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser { */ private File tmpIdx; + /** + * Path of the object-size index created for the pack, to filter quickly + * objects by size in partial clones + */ + private File tmpObjectSizeIndex; + /** Read/write handle to {@link #tmpPack} while it is being parsed. */ private RandomAccessFile out; @@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser { throws IOException { tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$ + try { out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$ @@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser { tmpPack.setReadOnly(); tmpIdx.setReadOnly(); + if (pconfig.isWriteObjSizeIndex()) { + tmpObjectSizeIndex = new File(db.getDirectory(), + baseName(tmpPack) + + PackExt.OBJECT_SIZE_INDEX.getExtension()); + writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex()); + tmpObjectSizeIndex.setReadOnly(); + } + return renameAndOpenPack(getLockMessage()); } finally { if (def != null) @@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser { tmpIdx.deleteOnExit(); if (tmpPack != null && !tmpPack.delete() && tmpPack.exists()) tmpPack.deleteOnExit(); + if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete() + && tmpObjectSizeIndex.exists()) + tmpPack.deleteOnExit(); } @Override @@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser { } } + private void writeObjectSizeIndex(int minSize) throws IOException { + try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) { + PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter + .createWriter(os, minSize); + iw.write(getSortedObjectList(null)); + os.getChannel().force(true); + } + } + private PackLock renameAndOpenPack(String lockMessage) throws IOException { if (!keepEmpty && getObjectCount() == 0) { @@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser { JGitText.get().cannotMoveIndexTo, finalIdx), e); } + if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) { + PackFile finalObjectSizeIndex = finalPack + .create(PackExt.OBJECT_SIZE_INDEX); + try { + FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + cleanupTemporaryFiles(); + keep.unlock(); + if (!finalPack.delete()) + finalPack.deleteOnExit(); + if (!finalIdx.delete()) { + finalIdx.deleteOnExit(); + } + throw new IOException(MessageFormat + .format(JGitText.get().cannotMoveIndexTo, + finalObjectSizeIndex), + e); + } + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(finalPack); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index bf574ec4c2..8988b41b55 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -126,8 +128,13 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private volatile Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); + private volatile PackObjectSizeIndex loadedObjSizeIdx; + + private volatile boolean attemptLoadObjSizeIdx; + private volatile Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty(); + /** * Objects we have tried to read, and discovered to be corrupt. * <p> @@ -218,6 +225,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } } + private PackObjectSizeIndex objectSizeIndex() throws IOException { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + if (attemptLoadObjSizeIdx) { + return null; + } + + synchronized (this) { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + PackObjectSizeIndex sizeIdx; + try { + long start = System.currentTimeMillis(); + PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX); + if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) { + attemptLoadObjSizeIdx = true; + return null; + } + sizeIdx = PackObjectSizeIndexLoader.load( + new FileInputStream(sizeIdxFile.getAbsoluteFile())); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + sizeIdxFile.getAbsolutePath(), + Float.valueOf( + sizeIdxFile.length() / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() - start))); + } + + loadedObjSizeIdx = sizeIdx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + return null; + } finally { + attemptLoadObjSizeIdx = true; + } + } + + return loadedObjSizeIdx; + } + /** * Get the File object which locates this pack on disk. * @@ -239,6 +292,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } /** + * Get the object size index for this pack file + * + * @return the object size index for this pack file if it exists (null + * otherwise) + * @throws IOException + * problem reading the index + */ + public boolean hasObjectSizeIndex() throws IOException { + return objectSizeIndex() != null; + } + + /** + * Number of objects in the object-size index of this pack + * + * @return number of objects in the index (0 if either the index is empty or + * it doesn't exist) + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getObjectSizeIndexCount() throws IOException { + if (!hasObjectSizeIndex()) { + return 0; + } + + return objectSizeIndex().getObjectCount(); + } + + /** + * Return the size of the object from the object-size index. + * + * Caller MUST check that the pack has object-size index + * ({@link #hasObjectSizeIndex()}) and that the pack contains the object. + * + * @param id + * object id of an object in the pack + * @return size of the object from the index. Negative if the object is not + * in the index. + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getIndexedObjectSize(AnyObjectId id) throws IOException { + int idxPos = idx().findPosition(id); + if (idxPos < 0) { + return -1; + } + + PackObjectSizeIndex sizeIdx = objectSizeIndex(); + if (sizeIdx == null) { + throw new IllegalStateException( + "Asking indexed size from a pack without object size index"); //$NON-NLS-1$ + } + + return sizeIdx.getSize(idxPos); + } + + /** * Get name extracted from {@code pack-*.pack} pattern. * * @return name extracted from {@code pack-*.pack} pattern. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 55e047bd43..97a854b8cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter { long offset = beginObject(type, len); packOut.compress.write(data, off, len); packOut.compress.finish(); - return endObject(id, offset); + return endObject(id, offset, len, type); } @Override @@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter { len -= n; } packOut.compress.finish(); - return endObject(md.toObjectId(), offset); + return endObject(md.toObjectId(), offset, len, type); } private long beginObject(int type, long len) throws IOException { @@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter { return offset; } - private ObjectId endObject(ObjectId id, long offset) { + private ObjectId endObject(ObjectId id, long offset, long len, int type) { PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setType(type); obj.setOffset(offset); obj.setCRC((int) packOut.crc32.getValue()); + obj.setFullSize(len); objectList.add(obj); objectMap.addIfAbsent(obj); return id; @@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter { p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ } + private static File getFileFor(File packFile, PackExt ext) { + String p = packFile.getName(); + return new File(packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ext.getExtension()); + } + private void beginPack() throws IOException { objectList = new BlockList<>(); objectMap = new ObjectIdOwnerMap<>(); @@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter { Collections.sort(objectList); File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - + File tmpObjSizeIdx = null; + if (pconfig.isWriteObjSizeIndex()) { + tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX); + writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig); + } PackFile realPack = new PackFile(db.getPackDirectory(), computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); @@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter { realIdx), e); } + if (tmpObjSizeIdx != null) { + PackFile realObjSizeIdx = realPack + .create(PackExt.OBJECT_SIZE_INDEX); + tmpObjSizeIdx.setReadOnly(); + FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE); + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(realPack); @@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter { } } + private static void writeObjectSizeIndex(File objIdx, + List<PackedObjectInfo> list, PackConfig cfg) throws IOException { + try (OutputStream os = new FileOutputStream(objIdx)) { + PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter + .createWriter(os, cfg.getMinBytesForObjSizeIndex()); + w.write(list); + } + } + private ObjectId computeName(List<PackedObjectInfo> list) { SHA1 md = digest().reset(); byte[] buf = buffer(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 11c45471e4..6612cfc838 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -55,6 +55,8 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { private DeltaBaseCache baseCache; + private Pack lastPack; + @Nullable private final ObjectInserter createdFromInserter; @@ -147,8 +149,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { return db.getShallowCommits(); } - @Override - public long getObjectSize(AnyObjectId objectId, int typeHint) + private long getObjectSizeStorage(AnyObjectId objectId, int typeHint) throws MissingObjectException, IncorrectObjectTypeException, IOException { long sz = db.getObjectSize(this, objectId); @@ -162,6 +163,61 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } @Override + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + // Async queue uses hint OBJ_ANY + if (typeHint != Constants.OBJ_BLOB) { + return getObjectSizeStorage(objectId, typeHint); + } + + Pack pack = findPack(objectId); + if (pack == null) { + // Non-packed object (e.g. loose or in alternates) + return getObjectSizeStorage(objectId, typeHint); + } + + long sz = pack.getIndexedObjectSize(objectId); + if (sz >= 0) { + return sz; + } + return getObjectSizeStorage(objectId, typeHint); + } + + @Override + public boolean isNotLargerThan(AnyObjectId objectId, int typeHint, + long size) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + if (typeHint != Constants.OBJ_BLOB) { + return getObjectSizeStorage(objectId, typeHint) <= size; + } + + Pack pack = findPack(objectId); + if (pack == null || !pack.hasObjectSizeIndex()) { + // Non-packed object (e.g. loose or in alternates) + return getObjectSizeStorage(objectId, typeHint) <= size; + } + + return pack.getIndexedObjectSize(objectId) <= size; + } + + private Pack findPack(AnyObjectId objectId) throws IOException { + if (lastPack != null && lastPack.hasObject(objectId)) { + return lastPack; + } + + for (Pack p : db.getPacks()) { + if (p.hasObject(objectId)) { + lastPack = p; + return p; + } + } + + return null; + } + + @Override public LocalObjectToPack newObjectToPack(AnyObjectId objectId, int type) { return new LocalObjectToPack(objectId, type); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java index 0e51e90815..61caddc221 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java @@ -257,7 +257,7 @@ public class MultiPackIndexLoader { MultiPackIndexBuilder addPackNames(byte[] buffer) throws MultiPackIndexFormatException { assertChunkNotSeenYet(packNames, MIDX_CHUNKID_PACKNAMES); - packNames = new String(buffer, UTF_8).split("\u0000"); + packNames = new String(buffer, UTF_8).split("\u0000"); //$NON-NLS-1$ return this; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java index 2400e12240..a74b9b617f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java @@ -125,6 +125,7 @@ public class ChangedPathTreeFilter extends TreeFilter { return paths; } + @SuppressWarnings("nls") @Override public String toString() { return "(CHANGED_PATH(" + pathFilter.toString() + ")" // |