diff options
Diffstat (limited to 'org.eclipse.jgit/src/org')
4 files changed, 221 insertions, 11 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index c08a92e5a7..97473bba2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; @@ -131,7 +132,7 @@ public class GC { private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP); private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX, - REVERSE_INDEX); + REVERSE_INDEX, OBJECT_SIZE_INDEX); private static final int DEFAULT_AUTOPACKLIMIT = 50; @@ -412,6 +413,10 @@ public class GC { */ private void removeOldPack(PackFile packFile, int deleteOptions) throws IOException { + if (!packFile.exists()) { + return; + } + if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); @@ -1340,6 +1345,7 @@ public class GC { idxChannel.force(true); } + if (pw.isReverseIndexEnabled()) { File tmpReverseIndexFile = new File(packdir, tmpBase + REVERSE_INDEX.getTmpExtension()); @@ -1356,6 +1362,19 @@ public class GC { .newOutputStream(channel)) { pw.writeReverseIndex(stream); channel.force(true); + } + } + + // write the object size + if (pconfig.isWriteObjSizeIndex()) { + File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$ + tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx); + try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx); + FileChannel idxChannel = fos.getChannel(); + OutputStream idxStream = Channels + .newOutputStream(idxChannel)) { + pw.writeObjectSizeIndex(idxStream); + idxChannel.force(true); } } @@ -1525,6 +1544,11 @@ public class GC { */ public long numberOfBitmaps; + /** + * The number of objects in the size-index of the packs + */ + public long numberOfSizeIndexedObjects; + @Override public String toString() { final StringBuilder b = new StringBuilder(); @@ -1540,6 +1564,8 @@ public class GC { b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$ b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$ + b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$ + .append(numberOfSizeIndexedObjects); return b.toString(); } } @@ -1548,7 +1574,7 @@ public class GC { * Returns information about objects and pack files for a FileRepository. * * @return information about objects and pack files for a FileRepository - * @throws java.io.IOException + * @throws java.io.IOException * if an IO error occurred */ public RepoStatistics getStatistics() throws IOException { @@ -1560,16 +1586,16 @@ public class GC { ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); + ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); if (latestBitmapTime == 0L) { latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); } - } - else if (latestBitmapTime == 0L) { - ret.numberOfPackFilesSinceBitmap++; - ret.numberOfObjectsSinceBitmap += packedObjects; - } + } else if (latestBitmapTime == 0L) { + ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 746e124e1f..d97d5a7ccd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser { */ private File tmpIdx; + /** + * Path of the object-size index created for the pack, to filter quickly + * objects by size in partial clones + */ + private File tmpObjectSizeIndex; + /** Read/write handle to {@link #tmpPack} while it is being parsed. */ private RandomAccessFile out; @@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser { throws IOException { tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$ + try { out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$ @@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser { tmpPack.setReadOnly(); tmpIdx.setReadOnly(); + if (pconfig.isWriteObjSizeIndex()) { + tmpObjectSizeIndex = new File(db.getDirectory(), + baseName(tmpPack) + + PackExt.OBJECT_SIZE_INDEX.getExtension()); + writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex()); + tmpObjectSizeIndex.setReadOnly(); + } + return renameAndOpenPack(getLockMessage()); } finally { if (def != null) @@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser { tmpIdx.deleteOnExit(); if (tmpPack != null && !tmpPack.delete() && tmpPack.exists()) tmpPack.deleteOnExit(); + if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete() + && tmpObjectSizeIndex.exists()) + tmpPack.deleteOnExit(); } @Override @@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser { } } + private void writeObjectSizeIndex(int minSize) throws IOException { + try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) { + PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter + .createWriter(os, minSize); + iw.write(getSortedObjectList(null)); + os.getChannel().force(true); + } + } + private PackLock renameAndOpenPack(String lockMessage) throws IOException { if (!keepEmpty && getObjectCount() == 0) { @@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser { JGitText.get().cannotMoveIndexTo, finalIdx), e); } + if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) { + PackFile finalObjectSizeIndex = finalPack + .create(PackExt.OBJECT_SIZE_INDEX); + try { + FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + cleanupTemporaryFiles(); + keep.unlock(); + if (!finalPack.delete()) + finalPack.deleteOnExit(); + if (!finalIdx.delete()) { + finalIdx.deleteOnExit(); + } + throw new IOException(MessageFormat + .format(JGitText.get().cannotMoveIndexTo, + finalObjectSizeIndex), + e); + } + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(finalPack); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 5813d39e9a..f2f54947af 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -126,6 +128,10 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); + private volatile PackObjectSizeIndex loadedObjSizeIdx; + + private volatile boolean attemptLoadObjSizeIdx; + private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty(); /** @@ -210,6 +216,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } } + private PackObjectSizeIndex objectSizeIndex() throws IOException { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + if (attemptLoadObjSizeIdx) { + return null; + } + + synchronized (this) { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + PackObjectSizeIndex sizeIdx; + try { + long start = System.currentTimeMillis(); + PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX); + if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) { + attemptLoadObjSizeIdx = true; + return null; + } + sizeIdx = PackObjectSizeIndexLoader.load( + new FileInputStream(sizeIdxFile.getAbsoluteFile())); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + sizeIdxFile.getAbsolutePath(), + Float.valueOf( + sizeIdxFile.length() / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() - start))); + } + + loadedObjSizeIdx = sizeIdx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + return null; + } finally { + attemptLoadObjSizeIdx = true; + } + } + + return loadedObjSizeIdx; + } + /** * Get the File object which locates this pack on disk. * @@ -231,6 +283,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } /** + * Get the object size index for this pack file + * + * @return the object size index for this pack file if it exists (null + * otherwise) + * @throws IOException + * problem reading the index + */ + public boolean hasObjectSizeIndex() throws IOException { + return objectSizeIndex() != null; + } + + /** + * Number of objects in the object-size index of this pack + * + * @return number of objects in the index (0 if either the index is empty or + * it doesn't exist) + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getObjectSizeIndexCount() throws IOException { + if (!hasObjectSizeIndex()) { + return 0; + } + + return objectSizeIndex().getObjectCount(); + } + + /** + * Return the size of the object from the object-size index. + * + * Caller MUST check that the pack has object-size index + * ({@link #hasObjectSizeIndex()}) and that the pack contains the object. + * + * @param id + * object id of an object in the pack + * @return size of the object from the index. Negative if the object is not + * in the index. + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getIndexedObjectSize(AnyObjectId id) throws IOException { + int idxPos = idx().findPosition(id); + if (idxPos < 0) { + return -1; + } + + PackObjectSizeIndex sizeIdx = objectSizeIndex(); + if (sizeIdx == null) { + throw new IllegalStateException( + "Asking indexed size from a pack without object size index"); //$NON-NLS-1$ + } + + return sizeIdx.getSize(idxPos); + } + + /** * Get name extracted from {@code pack-*.pack} pattern. * * @return name extracted from {@code pack-*.pack} pattern. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 55e047bd43..97a854b8cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter { long offset = beginObject(type, len); packOut.compress.write(data, off, len); packOut.compress.finish(); - return endObject(id, offset); + return endObject(id, offset, len, type); } @Override @@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter { len -= n; } packOut.compress.finish(); - return endObject(md.toObjectId(), offset); + return endObject(md.toObjectId(), offset, len, type); } private long beginObject(int type, long len) throws IOException { @@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter { return offset; } - private ObjectId endObject(ObjectId id, long offset) { + private ObjectId endObject(ObjectId id, long offset, long len, int type) { PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setType(type); obj.setOffset(offset); obj.setCRC((int) packOut.crc32.getValue()); + obj.setFullSize(len); objectList.add(obj); objectMap.addIfAbsent(obj); return id; @@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter { p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ } + private static File getFileFor(File packFile, PackExt ext) { + String p = packFile.getName(); + return new File(packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ext.getExtension()); + } + private void beginPack() throws IOException { objectList = new BlockList<>(); objectMap = new ObjectIdOwnerMap<>(); @@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter { Collections.sort(objectList); File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - + File tmpObjSizeIdx = null; + if (pconfig.isWriteObjSizeIndex()) { + tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX); + writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig); + } PackFile realPack = new PackFile(db.getPackDirectory(), computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); @@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter { realIdx), e); } + if (tmpObjSizeIdx != null) { + PackFile realObjSizeIdx = realPack + .create(PackExt.OBJECT_SIZE_INDEX); + tmpObjSizeIdx.setReadOnly(); + FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE); + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(realPack); @@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter { } } + private static void writeObjectSizeIndex(File objIdx, + List<PackedObjectInfo> list, PackConfig cfg) throws IOException { + try (OutputStream os = new FileOutputStream(objIdx)) { + PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter + .createWriter(os, cfg.getMinBytesForObjSizeIndex()); + w.write(list); + } + } + private ObjectId computeName(List<PackedObjectInfo> list) { SHA1 md = digest().reset(); byte[] buf = buffer(); |