diff options
6 files changed, 613 insertions, 7 deletions
diff --git a/org.eclipse.jgit.packaging/pom.xml b/org.eclipse.jgit.packaging/pom.xml index 69b57de3d9..74d39560f1 100644 --- a/org.eclipse.jgit.packaging/pom.xml +++ b/org.eclipse.jgit.packaging/pom.xml @@ -189,6 +189,19 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>17</version> + </requireJavaVersion> + </rules> + </configuration> + </execution> </executions> </plugin> <plugin> diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java new file mode 100644 index 0000000000..1a05d88583 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2025, Google LLC. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GcObjectSizeIndexTest extends GcTestCase { + + @Test + public void gc_2commits_noSizeLimit_blobsInIndex() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(4, stats.numberOfSizeIndexedObjects); + + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + Pack pack = getOnlyPack(repo); + assertEquals(7, pack.getIndexedObjectSize(blobA1)); + assertEquals(11, pack.getIndexedObjectSize(blobA2)); + assertEquals(1, pack.getIndexedObjectSize(blobB1)); + assertEquals(2, pack.getIndexedObjectSize(blobB2)); + } + + @Test + public void gc_2commits_sizeLimit_biggerBlobsInIndex() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 5); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(2, stats.numberOfSizeIndexedObjects); + + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + Pack pack = getOnlyPack(repo); + assertEquals(7, pack.getIndexedObjectSize(blobA1)); + assertEquals(11, pack.getIndexedObjectSize(blobA2)); + assertEquals(-1, pack.getIndexedObjectSize(blobB1)); + assertEquals(-1, pack.getIndexedObjectSize(blobB2)); + } + + @Test + public void gc_2commits_disableSizeIdx_noIdx() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, -1); + gc.gc().get(); + + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(0, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_alreadyPacked_noChanges() + throws Exception { + tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B") + .create(); + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + assertEquals(2, stats.numberOfSizeIndexedObjects); + + // Do the gc again and check that it hasn't changed anything + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + assertEquals(2, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_twoReachableCommits_oneUnreachable_twoPacks() + throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevCommit first = bb.commit().add("A", "A").add("B", "B").create(); + bb.commit().add("A", "A2").add("B", "B2").create(); + tr.update("refs/heads/master", first); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(8, stats.numberOfPackedObjects); + assertEquals(2, stats.numberOfPackFiles); + assertEquals(4, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_preserved_objSizeIdxIsPreserved() throws Exception { + Collection<Pack> oldPacks = preserveOldPacks(); + assertEquals(1, oldPacks.size()); + PackFile preserved = oldPacks.iterator().next().getPackFile() + .create(PackExt.OBJECT_SIZE_INDEX) + .createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preserved.exists()); + } + + @Test + public void gc_preserved_prune_noPreserves() throws Exception { + preserveOldPacks(); + configureGc(gc, 0).setPrunePreserved(true); + gc.gc().get(); + + assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists()); + } + + private Collection<Pack> preserveOldPacks() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + bb.commit().message("P").add("P", "P").create(); + + // pack loose object into packfile + configureGc(gc, 0); + gc.setExpireAgeMillis(0); + gc.gc().get(); + Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase() + .getPacks(); + PackFile oldPackfile = oldPacks.iterator().next().getPackFile(); + assertTrue(oldPackfile.exists()); + + fsTick(); + bb.commit().message("B").add("B", "Q").create(); + + // repack again but now without a grace period for packfiles. We should + // end up with a new packfile and the old one should be placed in the + // preserved directory + gc.setPackExpireAgeMillis(0); + configureGc(gc, 0).setPreserveOldPacks(true); + gc.gc().get(); + + File preservedPackFile = oldPackfile.createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preservedPackFile.exists()); + return oldPacks; + } + + @Ignore + public void testPruneAndRestoreOldPacks() throws Exception { + String tempRef = "refs/heads/soon-to-be-unreferenced"; + TestRepository<FileRepository>.BranchBuilder bb = tr.branch(tempRef); + bb.commit().add("A", "A").add("B", "B").create(); + + // Verify setup conditions + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + + // Force all referenced objects into packs (to avoid having loose objects) + configureGc(gc, 0); + gc.setExpireAgeMillis(0); + gc.setPackExpireAgeMillis(0); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + + // Delete the temp ref, orphaning its commit + RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setForceUpdate(true); + ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it! + RefUpdate.Result result = update.delete(); + assertEquals(RefUpdate.Result.FORCED, result); + + fsTick(); + + // Repack with only orphaned commit, so packfile will be pruned + configureGc(gc, 0).setPreserveOldPacks(true); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + assertEquals(0, stats.numberOfPackFiles); + + // Restore the temp ref to the deleted commit, should restore old-packs! + update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setNewObjectId(objectId); + update.setExpectedOldObjectId(null); + result = update.update(); + assertEquals(RefUpdate.Result.NEW, result); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + } + + private PackConfig configureGc(GC myGc, int minSize) { + PackConfig pconfig = new PackConfig(repo); + pconfig.setMinBytesForObjSizeIndex(minSize); + myGc.setPackConfig(pconfig); + return pconfig; + } + + private Pack getOnlyPack(FileRepository fileRepo) + throws IOException { + Collection<Pack> packs = fileRepo.getObjectDatabase().getPacks(); + if (packs.size() != 1) { + throw new IOException("More than one pack"); + } + + return packs.iterator().next(); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java new file mode 100644 index 0000000000..b17c577087 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2021, Google LLC. and others + * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com> + * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.util.zip.Deflater; + +import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; +import org.eclipse.jgit.internal.storage.file.Pack; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Pack parsing is covered in {@link PackParserTest}. + * + * Here we test ObjectDirectoryPackParser specific parts. e.g. that is creates + * the object-size index. + */ +public class ObjectDirectoryPackParserTest extends RepositoryTestCase { + + @Before + public void setup() throws IOException { + FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig(); + jGitConfig.setInt(ConfigConstants.CONFIG_PACK_SECTION, null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 7); + jGitConfig.save(); + } + + /** + * Test indexing one of the test packs in the egit repo. It has deltas. + * + * @throws IOException + */ + @Test + public void testGitPack() throws IOException { + File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); + try (InputStream is = new FileInputStream(packFile)) { + ObjectDirectoryPackParser p = index(is); + p.parse(NullProgressMonitor.INSTANCE); + + Pack pack = p.getPack(); + assertTrue(pack.hasObjectSizeIndex()); + + // Only blobs in the pack + ObjectId blob1 = ObjectId + .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); + ObjectId blob2 = ObjectId + .fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"); + assertEquals(18787, pack.getIndexedObjectSize(blob1)); + assertEquals(18009, pack.getIndexedObjectSize(blob2)); + + // Indexed sizes match object db sizes + assertEquals(db.getObjectDatabase().open(blob1).getSize(), + pack.getIndexedObjectSize(blob1)); + assertEquals(db.getObjectDatabase().open(blob2).getSize(), + pack.getIndexedObjectSize(blob2)); + + } + } + + /** + * This is just another pack. It so happens that we have two convenient pack to + * test with in the repository. + * + * @throws IOException + */ + @Test + public void testAnotherGitPack() throws IOException { + File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack"); + try (InputStream is = new FileInputStream(packFile)) { + ObjectDirectoryPackParser p = index(is); + p.parse(NullProgressMonitor.INSTANCE); + Pack pack = p.getPack(); + + // Blob smaller than threshold: + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); + + // Blob bigger than threshold + assertEquals(10, pack.getIndexedObjectSize(ObjectId + .fromString("8230f48330e0055d9e0bc5a2a77718f6dd9324b8"))); + + // A commit (not indexed) + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("d0114ab8ac326bab30e3a657a0397578c5a1af88"))); + + // Object not in pack + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))); + } + } + + @Test + public void testTinyThinPack() throws Exception { + // less than 16 bytes, so its length fits in a single byte later + String base = "abcdefghijklmn"; + RevBlob a; + try (TestRepository d = new TestRepository<Repository>(db)) { + a = d.blob(base); + } + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + + packHeader(pack, 1); + + pack.write(Constants.OBJ_REF_DELTA << 4 | 4); + a.copyRawTo(pack); + deflate(pack, new byte[] { (byte) base.length(), // size of the base + (byte) (base.length() + 1), // size after reconstruction + 0x1, 'b' }); // append one byte + + digest(pack); + + ObjectDirectoryPackParser p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.parse(NullProgressMonitor.INSTANCE); + + Pack writtenPack = p.getPack(); + // base + assertEquals(base.length(), writtenPack.getIndexedObjectSize(a)); + // undeltified blob + assertEquals(base.length() + 1, + writtenPack.getIndexedObjectSize(ObjectId.fromString( + "f177875498138143c9657cc52b049ad4d20d5223"))); + } + + @Test + public void testPackWithDuplicateBlob() throws Exception { + final byte[] data = Constants.encode("0123456789abcdefg"); + RevBlob blob; + try (TestRepository<Repository> d = new TestRepository<>(db)) { + blob = d.blob(data); + assertTrue(db.getObjectDatabase().has(blob)); + } + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + packHeader(pack, 1); + pack.write(Constants.OBJ_BLOB << 4 | 0x80 | 1); + pack.write(1); + deflate(pack, data); + digest(pack); + + ObjectDirectoryPackParser p = index( + new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(false); + p.parse(NullProgressMonitor.INSTANCE); + + assertEquals(data.length, p.getPack().getIndexedObjectSize(blob)); + } + + private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt) + throws IOException { + final byte[] hdr = new byte[8]; + NB.encodeInt32(hdr, 0, 2); + NB.encodeInt32(hdr, 4, cnt); + + tinyPack.write(Constants.PACK_SIGNATURE); + tinyPack.write(hdr, 0, 8); + } + + private static void deflate(TemporaryBuffer.Heap tinyPack, + final byte[] content) + throws IOException { + final Deflater deflater = new Deflater(); + final byte[] buf = new byte[128]; + deflater.setInput(content, 0, content.length); + deflater.finish(); + do { + final int n = deflater.deflate(buf, 0, buf.length); + if (n > 0) + tinyPack.write(buf, 0, n); + } while (!deflater.finished()); + } + + private static void digest(TemporaryBuffer.Heap buf) throws IOException { + MessageDigest md = Constants.newMessageDigest(); + md.update(buf.toByteArray()); + buf.write(md.digest()); + } + + private ObjectInserter inserter; + + @After + public void release() { + if (inserter != null) { + inserter.close(); + } + } + + private ObjectDirectoryPackParser index(InputStream in) throws IOException { + if (inserter == null) + inserter = db.newObjectInserter(); + return (ObjectDirectoryPackParser) inserter.newPackParser(in); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index c08a92e5a7..97473bba2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; @@ -131,7 +132,7 @@ public class GC { private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP); private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX, - REVERSE_INDEX); + REVERSE_INDEX, OBJECT_SIZE_INDEX); private static final int DEFAULT_AUTOPACKLIMIT = 50; @@ -412,6 +413,10 @@ public class GC { */ private void removeOldPack(PackFile packFile, int deleteOptions) throws IOException { + if (!packFile.exists()) { + return; + } + if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); @@ -1340,6 +1345,7 @@ public class GC { idxChannel.force(true); } + if (pw.isReverseIndexEnabled()) { File tmpReverseIndexFile = new File(packdir, tmpBase + REVERSE_INDEX.getTmpExtension()); @@ -1356,6 +1362,19 @@ public class GC { .newOutputStream(channel)) { pw.writeReverseIndex(stream); channel.force(true); + } + } + + // write the object size + if (pconfig.isWriteObjSizeIndex()) { + File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$ + tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx); + try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx); + FileChannel idxChannel = fos.getChannel(); + OutputStream idxStream = Channels + .newOutputStream(idxChannel)) { + pw.writeObjectSizeIndex(idxStream); + idxChannel.force(true); } } @@ -1525,6 +1544,11 @@ public class GC { */ public long numberOfBitmaps; + /** + * The number of objects in the size-index of the packs + */ + public long numberOfSizeIndexedObjects; + @Override public String toString() { final StringBuilder b = new StringBuilder(); @@ -1540,6 +1564,8 @@ public class GC { b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$ b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$ + b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$ + .append(numberOfSizeIndexedObjects); return b.toString(); } } @@ -1548,7 +1574,7 @@ public class GC { * Returns information about objects and pack files for a FileRepository. * * @return information about objects and pack files for a FileRepository - * @throws java.io.IOException + * @throws java.io.IOException * if an IO error occurred */ public RepoStatistics getStatistics() throws IOException { @@ -1560,16 +1586,16 @@ public class GC { ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); + ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); if (latestBitmapTime == 0L) { latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); } - } - else if (latestBitmapTime == 0L) { - ret.numberOfPackFilesSinceBitmap++; - ret.numberOfObjectsSinceBitmap += packedObjects; - } + } else if (latestBitmapTime == 0L) { + ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 746e124e1f..d97d5a7ccd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser { */ private File tmpIdx; + /** + * Path of the object-size index created for the pack, to filter quickly + * objects by size in partial clones + */ + private File tmpObjectSizeIndex; + /** Read/write handle to {@link #tmpPack} while it is being parsed. */ private RandomAccessFile out; @@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser { throws IOException { tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$ + try { out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$ @@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser { tmpPack.setReadOnly(); tmpIdx.setReadOnly(); + if (pconfig.isWriteObjSizeIndex()) { + tmpObjectSizeIndex = new File(db.getDirectory(), + baseName(tmpPack) + + PackExt.OBJECT_SIZE_INDEX.getExtension()); + writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex()); + tmpObjectSizeIndex.setReadOnly(); + } + return renameAndOpenPack(getLockMessage()); } finally { if (def != null) @@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser { tmpIdx.deleteOnExit(); if (tmpPack != null && !tmpPack.delete() && tmpPack.exists()) tmpPack.deleteOnExit(); + if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete() + && tmpObjectSizeIndex.exists()) + tmpPack.deleteOnExit(); } @Override @@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser { } } + private void writeObjectSizeIndex(int minSize) throws IOException { + try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) { + PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter + .createWriter(os, minSize); + iw.write(getSortedObjectList(null)); + os.getChannel().force(true); + } + } + private PackLock renameAndOpenPack(String lockMessage) throws IOException { if (!keepEmpty && getObjectCount() == 0) { @@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser { JGitText.get().cannotMoveIndexTo, finalIdx), e); } + if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) { + PackFile finalObjectSizeIndex = finalPack + .create(PackExt.OBJECT_SIZE_INDEX); + try { + FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + cleanupTemporaryFiles(); + keep.unlock(); + if (!finalPack.delete()) + finalPack.deleteOnExit(); + if (!finalIdx.delete()) { + finalIdx.deleteOnExit(); + } + throw new IOException(MessageFormat + .format(JGitText.get().cannotMoveIndexTo, + finalObjectSizeIndex), + e); + } + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(finalPack); @@ -404,6 +404,19 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>17</version> + </requireJavaVersion> + </rules> + </configuration> + </execution> </executions> </plugin> |