aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMatthias Sohn <matthias.sohn@sap.com>2025-07-23 00:40:36 +0200
committerMatthias Sohn <matthias.sohn@sap.com>2025-07-23 00:40:36 +0200
commit57c089e6ed79551de5aec2893fa9018a04fd54d3 (patch)
treef9cb181943b90041784279a107c386d2d3de5c76
parent010858e24f1860fc70ecce534a274def79826abb (diff)
parent2072c3e8dd9af42db0535a53743d4fe41dbe7cea (diff)
downloadjgit-servlet-4.tar.gz
jgit-servlet-4.zip
Merge branch 'master' into servlet-4servlet-4
* master: GC: Write object size index if config says so MergeToolTest: update expected error from the tool Add dash license check for 3rd party dependencies ObjectDirectoryPackParser: Write object-size index with the pack PackInserter: write object-size index with the pack Pack: getter for the indexed object size (when available) Prepare 5.13.5-SNAPSHOT builds JGit v5.13.4.202507202350-r AmazonS3: Do not accept DOCTYPE and entities ManifestParser: Do not accept DOCTYPE and entities AdvertisedRequestValidator: fix WantNotValidException caused by race Prepare 5.13.4-SNAPSHOT builds Fix packaging build [releng] Fix maven builds on JVM 21 and with newer git on MacOS Change-Id: I642c5b363b78476b6e6c1937445a3a60f6ba6cd4
-rw-r--r--org.eclipse.jgit.junit/META-INF/MANIFEST.MF1
-rw-r--r--org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java15
-rw-r--r--org.eclipse.jgit.packaging/pom.xml13
-rw-r--r--org.eclipse.jgit.test/pom.xml2
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java3
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java279
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java38
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java27
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java227
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java40
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java48
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java108
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java36
-rw-r--r--pom.xml13
14 files changed, 834 insertions, 16 deletions
diff --git a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
index 92371aec69..01df135c5f 100644
--- a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
@@ -20,6 +20,7 @@ Import-Package: org.eclipse.jgit.annotations;version="[7.4.0,7.5.0)",
org.eclipse.jgit.merge;version="[7.4.0,7.5.0)",
org.eclipse.jgit.revwalk;version="[7.4.0,7.5.0)",
org.eclipse.jgit.storage.file;version="[7.4.0,7.5.0)",
+ org.eclipse.jgit.storage.pack;version="[7.4.0,7.5.0)",
org.eclipse.jgit.transport;version="7.4.0",
org.eclipse.jgit.treewalk;version="[7.4.0,7.5.0)",
org.eclipse.jgit.treewalk.filter;version="[7.4.0,7.5.0)",
diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
index 2d00a850e5..c546ae9082 100644
--- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
+++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
@@ -73,6 +73,7 @@ import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTag;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
import org.eclipse.jgit.util.ChangeIdUtil;
@@ -987,7 +988,7 @@ public class TestRepository<R extends Repository> implements AutoCloseable {
ObjectDirectory odb = (ObjectDirectory) db.getObjectDatabase();
NullProgressMonitor m = NullProgressMonitor.INSTANCE;
- final PackFile pack, idx;
+ PackFile pack;
try (PackWriter pw = new PackWriter(db)) {
Set<ObjectId> all = new HashSet<>();
for (Ref r : db.getRefDatabase().getRefs())
@@ -1002,12 +1003,22 @@ public class TestRepository<R extends Repository> implements AutoCloseable {
}
pack.setReadOnly();
- idx = pack.create(PackExt.INDEX);
+ PackFile idx = pack.create(PackExt.INDEX);
try (OutputStream out =
new BufferedOutputStream(new FileOutputStream(idx))) {
pw.writeIndex(out);
}
idx.setReadOnly();
+
+ PackConfig pc = new PackConfig(db);
+ if (pc.getMinBytesForObjSizeIndex() >= 0) {
+ PackFile oidx = pack.create(PackExt.OBJECT_SIZE_INDEX);
+ try (OutputStream out = new BufferedOutputStream(
+ new FileOutputStream(oidx))) {
+ pw.writeObjectSizeIndex(out);
+ }
+ oidx.setReadOnly();
+ }
}
odb.openPack(pack);
diff --git a/org.eclipse.jgit.packaging/pom.xml b/org.eclipse.jgit.packaging/pom.xml
index 69b57de3d9..74d39560f1 100644
--- a/org.eclipse.jgit.packaging/pom.xml
+++ b/org.eclipse.jgit.packaging/pom.xml
@@ -189,6 +189,19 @@
</rules>
</configuration>
</execution>
+ <execution>
+ <id>enforce-java</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <version>17</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
diff --git a/org.eclipse.jgit.test/pom.xml b/org.eclipse.jgit.test/pom.xml
index 39d317a136..b67a2ab673 100644
--- a/org.eclipse.jgit.test/pom.xml
+++ b/org.eclipse.jgit.test/pom.xml
@@ -165,7 +165,7 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <argLine>@{argLine} -Xmx768m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=${project.build.directory}</argLine>
+ <argLine>@{argLine} -Xmx768m -Dfile.encoding=UTF-8 -Djava.io.tmpdir=${project.build.directory} --add-opens java.base/sun.nio.fs=ALL-UNNAMED</argLine>
<includes>
<include>**/*Test.java</include>
<include>**/*Tests.java</include>
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java
index d02bfcd3f6..1119db3712 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java
@@ -132,7 +132,8 @@ public class IndexDiffWithSymlinkTest extends LocalDiskRepositoryTestCase {
Writer writer = new OutputStreamWriter(out, UTF_8)) {
writer.write("echo `which git` 1>&2\n");
writer.write("echo `git --version` 1>&2\n");
- writer.write("git init " + name + " && \\\n");
+ writer.write("git -c init.defaultBranch=master init " + name
+ + " && \\\n");
writer.write("cd ./" + name + " && \\\n");
writer.write("git fast-import < ../" + name + ".txt && \\\n");
writer.write("git checkout -f\n");
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java
new file mode 100644
index 0000000000..1a05d88583
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright (C) 2025, Google LLC. and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import org.eclipse.jgit.internal.storage.pack.PackExt;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.pack.PackConfig;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GcObjectSizeIndexTest extends GcTestCase {
+
+ @Test
+ public void gc_2commits_noSizeLimit_blobsInIndex() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(4, stats.numberOfSizeIndexedObjects);
+
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ Pack pack = getOnlyPack(repo);
+ assertEquals(7, pack.getIndexedObjectSize(blobA1));
+ assertEquals(11, pack.getIndexedObjectSize(blobA2));
+ assertEquals(1, pack.getIndexedObjectSize(blobB1));
+ assertEquals(2, pack.getIndexedObjectSize(blobB2));
+ }
+
+ @Test
+ public void gc_2commits_sizeLimit_biggerBlobsInIndex() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 5);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ Pack pack = getOnlyPack(repo);
+ assertEquals(7, pack.getIndexedObjectSize(blobA1));
+ assertEquals(11, pack.getIndexedObjectSize(blobA2));
+ assertEquals(-1, pack.getIndexedObjectSize(blobB1));
+ assertEquals(-1, pack.getIndexedObjectSize(blobB2));
+ }
+
+ @Test
+ public void gc_2commits_disableSizeIdx_noIdx() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevBlob blobA1 = tr.blob("7-bytes");
+ RevBlob blobA2 = tr.blob("11-bytes xx");
+ RevBlob blobB1 = tr.blob("B");
+ RevBlob blobB2 = tr.blob("B2");
+ bb.commit().add("A", blobA1).add("B", blobB1).create();
+ bb.commit().add("A", blobA2).add("B", blobB2).create();
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, -1);
+ gc.gc().get();
+
+
+ stats = gc.getStatistics();
+ assertEquals(1, stats.numberOfPackFiles);
+ assertEquals(0, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_alreadyPacked_noChanges()
+ throws Exception {
+ tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B")
+ .create();
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+
+ // Do the gc again and check that it hasn't changed anything
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ assertTrue(getOnlyPack(repo).hasObjectSizeIndex());
+ assertEquals(2, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_twoReachableCommits_oneUnreachable_twoPacks()
+ throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ RevCommit first = bb.commit().add("A", "A").add("B", "B").create();
+ bb.commit().add("A", "A2").add("B", "B2").create();
+ tr.update("refs/heads/master", first);
+
+ stats = gc.getStatistics();
+ assertEquals(8, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ configureGc(gc, 0);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(8, stats.numberOfPackedObjects);
+ assertEquals(2, stats.numberOfPackFiles);
+ assertEquals(4, stats.numberOfSizeIndexedObjects);
+ }
+
+ @Test
+ public void gc_preserved_objSizeIdxIsPreserved() throws Exception {
+ Collection<Pack> oldPacks = preserveOldPacks();
+ assertEquals(1, oldPacks.size());
+ PackFile preserved = oldPacks.iterator().next().getPackFile()
+ .create(PackExt.OBJECT_SIZE_INDEX)
+ .createPreservedForDirectory(
+ repo.getObjectDatabase().getPreservedDirectory());
+ assertTrue(preserved.exists());
+ }
+
+ @Test
+ public void gc_preserved_prune_noPreserves() throws Exception {
+ preserveOldPacks();
+ configureGc(gc, 0).setPrunePreserved(true);
+ gc.gc().get();
+
+ assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists());
+ }
+
+ private Collection<Pack> preserveOldPacks() throws Exception {
+ TestRepository<FileRepository>.BranchBuilder bb = tr
+ .branch("refs/heads/master");
+ bb.commit().message("P").add("P", "P").create();
+
+ // pack loose object into packfile
+ configureGc(gc, 0);
+ gc.setExpireAgeMillis(0);
+ gc.gc().get();
+ Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase()
+ .getPacks();
+ PackFile oldPackfile = oldPacks.iterator().next().getPackFile();
+ assertTrue(oldPackfile.exists());
+
+ fsTick();
+ bb.commit().message("B").add("B", "Q").create();
+
+ // repack again but now without a grace period for packfiles. We should
+ // end up with a new packfile and the old one should be placed in the
+ // preserved directory
+ gc.setPackExpireAgeMillis(0);
+ configureGc(gc, 0).setPreserveOldPacks(true);
+ gc.gc().get();
+
+ File preservedPackFile = oldPackfile.createPreservedForDirectory(
+ repo.getObjectDatabase().getPreservedDirectory());
+ assertTrue(preservedPackFile.exists());
+ return oldPacks;
+ }
+
+ @Ignore
+ public void testPruneAndRestoreOldPacks() throws Exception {
+ String tempRef = "refs/heads/soon-to-be-unreferenced";
+ TestRepository<FileRepository>.BranchBuilder bb = tr.branch(tempRef);
+ bb.commit().add("A", "A").add("B", "B").create();
+
+ // Verify setup conditions
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+
+ // Force all referenced objects into packs (to avoid having loose objects)
+ configureGc(gc, 0);
+ gc.setExpireAgeMillis(0);
+ gc.setPackExpireAgeMillis(0);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+
+ // Delete the temp ref, orphaning its commit
+ RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
+ update.setForceUpdate(true);
+ ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it!
+ RefUpdate.Result result = update.delete();
+ assertEquals(RefUpdate.Result.FORCED, result);
+
+ fsTick();
+
+ // Repack with only orphaned commit, so packfile will be pruned
+ configureGc(gc, 0).setPreserveOldPacks(true);
+ gc.gc().get();
+ stats = gc.getStatistics();
+ assertEquals(0, stats.numberOfLooseObjects);
+ assertEquals(0, stats.numberOfPackedObjects);
+ assertEquals(0, stats.numberOfPackFiles);
+
+ // Restore the temp ref to the deleted commit, should restore old-packs!
+ update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
+ update.setNewObjectId(objectId);
+ update.setExpectedOldObjectId(null);
+ result = update.update();
+ assertEquals(RefUpdate.Result.NEW, result);
+
+ stats = gc.getStatistics();
+ assertEquals(4, stats.numberOfPackedObjects);
+ assertEquals(1, stats.numberOfPackFiles);
+ }
+
+ private PackConfig configureGc(GC myGc, int minSize) {
+ PackConfig pconfig = new PackConfig(repo);
+ pconfig.setMinBytesForObjSizeIndex(minSize);
+ myGc.setPackConfig(pconfig);
+ return pconfig;
+ }
+
+ private Pack getOnlyPack(FileRepository fileRepo)
+ throws IOException {
+ Collection<Pack> packs = fileRepo.getObjectDatabase().getPacks();
+ if (packs.size() != 1) {
+ throw new IOException("More than one pack");
+ }
+
+ return packs.iterator().next();
+ }
+}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
index 85043034aa..cc43d3c2bb 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
@@ -53,6 +53,7 @@ import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
@@ -77,12 +78,14 @@ import org.eclipse.jgit.dircache.DirCacheEntry;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.junit.RepositoryTestCase;
import org.eclipse.jgit.lib.CommitBuilder;
+import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ObjectStream;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.util.IO;
@@ -489,6 +492,38 @@ public class PackInserterTest extends RepositoryTestCase {
}
}
+ @Test
+ public void createsObjectSizeIndex() throws Exception {
+ FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig();
+ jGitConfig.setInt(
+ ConfigConstants.CONFIG_PACK_SECTION,
+ null,
+ ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 10);
+ jGitConfig.save();
+ byte[] oneBlob = Constants.encode("a blob with some content");
+ byte[] anotherBlob = Constants.encode("some more contents");
+ byte[] streamMeBlob = Constants.encode("some more content to write");
+
+ ObjectId oneBlobOid, anotherBlobOid, streamMeBlobOid;
+ try (PackInserter ins = newInserter()) {
+ oneBlobOid = ins.insert(OBJ_BLOB, oneBlob);
+ anotherBlobOid = ins.insert(OBJ_BLOB, anotherBlob);
+ streamMeBlobOid = ins.insert(OBJ_BLOB, streamMeBlob.length,
+ new ByteArrayInputStream(streamMeBlob));
+ ins.flush();
+ }
+
+ List<Pack> listPacks = listPacks(db);
+ assertEquals(1, listPacks.size());
+ Pack thePack = listPacks.get(0);
+ assertTrue(thePack.hasObjectSizeIndex());
+ assertEquals(oneBlob.length, thePack.getIndexedObjectSize(oneBlobOid));
+ assertEquals(anotherBlob.length,
+ thePack.getIndexedObjectSize(anotherBlobOid));
+ assertEquals(streamMeBlob.length,
+ thePack.getIndexedObjectSize(streamMeBlobOid));
+ }
+
private List<Pack> listPacks() throws Exception {
List<Pack> fromOpenDb = listPacks(db);
List<Pack> reopened;
@@ -549,7 +584,8 @@ public class PackInserterTest extends RepositoryTestCase {
}
private void assertPacksOnly() throws Exception {
- new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
+ new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")
+ && !f.endsWith(".objsize"))
.assertNoBadFiles(db.getObjectDatabase().getDirectory());
}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
index e1509456e5..016a6afd70 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
@@ -10,6 +10,7 @@
package org.eclipse.jgit.internal.storage.file;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -29,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import java.util.zip.Deflater;
import org.eclipse.jgit.errors.LargeObjectException;
@@ -39,6 +41,7 @@ import org.eclipse.jgit.junit.JGitTestUtil;
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.junit.TestRng;
+import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
@@ -47,6 +50,7 @@ import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectStream;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.transport.PackParser;
import org.eclipse.jgit.transport.PackedObjectInfo;
@@ -295,6 +299,29 @@ public class PackTest extends LocalDiskRepositoryTestCase {
}
}
+ @Test
+ public void testObjectSize() throws Exception {
+ byte[] data = getRng().nextBytes(300);
+ RevBlob aBlob = tr.blob(data);
+ RevCommit aCommit = tr.branch("master").commit().add("A", aBlob).create();
+ repo.getConfig().setInt(CONFIG_PACK_SECTION, null, ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0);
+ tr.packAndPrune();
+
+ List<Pack> packs = repo.getObjectDatabase().getPacks().stream().collect(Collectors.toList());
+ assertEquals(1, packs.size());
+ // Indexed object
+ assertEquals(300, packs.get(0).getIndexedObjectSize(aBlob));
+ assertEquals(300, packs.get(0).getObjectSize(wc, aBlob));
+ // Non indexed object
+ assertEquals(-1, packs.get(0).getIndexedObjectSize(aCommit));
+ assertEquals(168, packs.get(0).getObjectSize(wc, aCommit));
+ // Object not in pack
+ assertEquals(-1, packs.get(0).getObjectSize(wc,
+ ObjectId.fromString("1111111111111111111111111111111111111111")));
+ assertEquals(-1, packs.get(0).getIndexedObjectSize(
+ ObjectId.fromString("1111111111111111111111111111111111111111")));
+ }
+
private static byte[] clone(int first, byte[] base) {
byte[] r = new byte[base.length];
System.arraycopy(base, 1, r, 1, r.length - 1);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java
new file mode 100644
index 0000000000..b17c577087
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2021, Google LLC. and others
+ * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com>
+ * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
+ * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.transport;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.util.zip.Deflater;
+
+import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser;
+import org.eclipse.jgit.internal.storage.file.Pack;
+import org.eclipse.jgit.junit.JGitTestUtil;
+import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.NB;
+import org.eclipse.jgit.util.TemporaryBuffer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Pack parsing is covered in {@link PackParserTest}.
+ *
+ * Here we test ObjectDirectoryPackParser specific parts. e.g. that is creates
+ * the object-size index.
+ */
+public class ObjectDirectoryPackParserTest extends RepositoryTestCase {
+
+ @Before
+ public void setup() throws IOException {
+ FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig();
+ jGitConfig.setInt(ConfigConstants.CONFIG_PACK_SECTION, null,
+ ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 7);
+ jGitConfig.save();
+ }
+
+ /**
+ * Test indexing one of the test packs in the egit repo. It has deltas.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGitPack() throws IOException {
+ File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
+ try (InputStream is = new FileInputStream(packFile)) {
+ ObjectDirectoryPackParser p = index(is);
+ p.parse(NullProgressMonitor.INSTANCE);
+
+ Pack pack = p.getPack();
+ assertTrue(pack.hasObjectSizeIndex());
+
+ // Only blobs in the pack
+ ObjectId blob1 = ObjectId
+ .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
+ ObjectId blob2 = ObjectId
+ .fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259");
+ assertEquals(18787, pack.getIndexedObjectSize(blob1));
+ assertEquals(18009, pack.getIndexedObjectSize(blob2));
+
+ // Indexed sizes match object db sizes
+ assertEquals(db.getObjectDatabase().open(blob1).getSize(),
+ pack.getIndexedObjectSize(blob1));
+ assertEquals(db.getObjectDatabase().open(blob2).getSize(),
+ pack.getIndexedObjectSize(blob2));
+
+ }
+ }
+
+ /**
+ * This is just another pack. It so happens that we have two convenient pack to
+ * test with in the repository.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testAnotherGitPack() throws IOException {
+ File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack");
+ try (InputStream is = new FileInputStream(packFile)) {
+ ObjectDirectoryPackParser p = index(is);
+ p.parse(NullProgressMonitor.INSTANCE);
+ Pack pack = p.getPack();
+
+ // Blob smaller than threshold:
+ assertEquals(-1, pack.getIndexedObjectSize(ObjectId
+ .fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6")));
+
+ // Blob bigger than threshold
+ assertEquals(10, pack.getIndexedObjectSize(ObjectId
+ .fromString("8230f48330e0055d9e0bc5a2a77718f6dd9324b8")));
+
+ // A commit (not indexed)
+ assertEquals(-1, pack.getIndexedObjectSize(ObjectId
+ .fromString("d0114ab8ac326bab30e3a657a0397578c5a1af88")));
+
+ // Object not in pack
+ assertEquals(-1, pack.getIndexedObjectSize(ObjectId
+ .fromString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")));
+ }
+ }
+
+ @Test
+ public void testTinyThinPack() throws Exception {
+ // less than 16 bytes, so its length fits in a single byte later
+ String base = "abcdefghijklmn";
+ RevBlob a;
+ try (TestRepository d = new TestRepository<Repository>(db)) {
+ a = d.blob(base);
+ }
+
+ TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
+
+ packHeader(pack, 1);
+
+ pack.write(Constants.OBJ_REF_DELTA << 4 | 4);
+ a.copyRawTo(pack);
+ deflate(pack, new byte[] { (byte) base.length(), // size of the base
+ (byte) (base.length() + 1), // size after reconstruction
+ 0x1, 'b' }); // append one byte
+
+ digest(pack);
+
+ ObjectDirectoryPackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
+ p.setAllowThin(true);
+ p.parse(NullProgressMonitor.INSTANCE);
+
+ Pack writtenPack = p.getPack();
+ // base
+ assertEquals(base.length(), writtenPack.getIndexedObjectSize(a));
+ // undeltified blob
+ assertEquals(base.length() + 1,
+ writtenPack.getIndexedObjectSize(ObjectId.fromString(
+ "f177875498138143c9657cc52b049ad4d20d5223")));
+ }
+
+ @Test
+ public void testPackWithDuplicateBlob() throws Exception {
+ final byte[] data = Constants.encode("0123456789abcdefg");
+ RevBlob blob;
+ try (TestRepository<Repository> d = new TestRepository<>(db)) {
+ blob = d.blob(data);
+ assertTrue(db.getObjectDatabase().has(blob));
+ }
+
+ TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
+ packHeader(pack, 1);
+ pack.write(Constants.OBJ_BLOB << 4 | 0x80 | 1);
+ pack.write(1);
+ deflate(pack, data);
+ digest(pack);
+
+ ObjectDirectoryPackParser p = index(
+ new ByteArrayInputStream(pack.toByteArray()));
+ p.setAllowThin(false);
+ p.parse(NullProgressMonitor.INSTANCE);
+
+ assertEquals(data.length, p.getPack().getIndexedObjectSize(blob));
+ }
+
+ private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
+ throws IOException {
+ final byte[] hdr = new byte[8];
+ NB.encodeInt32(hdr, 0, 2);
+ NB.encodeInt32(hdr, 4, cnt);
+
+ tinyPack.write(Constants.PACK_SIGNATURE);
+ tinyPack.write(hdr, 0, 8);
+ }
+
+ private static void deflate(TemporaryBuffer.Heap tinyPack,
+ final byte[] content)
+ throws IOException {
+ final Deflater deflater = new Deflater();
+ final byte[] buf = new byte[128];
+ deflater.setInput(content, 0, content.length);
+ deflater.finish();
+ do {
+ final int n = deflater.deflate(buf, 0, buf.length);
+ if (n > 0)
+ tinyPack.write(buf, 0, n);
+ } while (!deflater.finished());
+ }
+
+ private static void digest(TemporaryBuffer.Heap buf) throws IOException {
+ MessageDigest md = Constants.newMessageDigest();
+ md.update(buf.toByteArray());
+ buf.write(md.digest());
+ }
+
+ private ObjectInserter inserter;
+
+ @After
+ public void release() {
+ if (inserter != null) {
+ inserter.close();
+ }
+ }
+
+ private ObjectDirectoryPackParser index(InputStream in) throws IOException {
+ if (inserter == null)
+ inserter = db.newObjectInserter();
+ return (ObjectDirectoryPackParser) inserter.newPackParser(in);
+ }
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index c08a92e5a7..97473bba2a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
@@ -131,7 +132,7 @@ public class GC {
private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP);
private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX,
- REVERSE_INDEX);
+ REVERSE_INDEX, OBJECT_SIZE_INDEX);
private static final int DEFAULT_AUTOPACKLIMIT = 50;
@@ -412,6 +413,10 @@ public class GC {
*/
private void removeOldPack(PackFile packFile, int deleteOptions)
throws IOException {
+ if (!packFile.exists()) {
+ return;
+ }
+
if (pconfig.isPreserveOldPacks()) {
File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
FileUtils.mkdir(oldPackDir, true);
@@ -1340,6 +1345,7 @@ public class GC {
idxChannel.force(true);
}
+
if (pw.isReverseIndexEnabled()) {
File tmpReverseIndexFile = new File(packdir,
tmpBase + REVERSE_INDEX.getTmpExtension());
@@ -1356,6 +1362,19 @@ public class GC {
.newOutputStream(channel)) {
pw.writeReverseIndex(stream);
channel.force(true);
+ }
+ }
+
+ // write the object size
+ if (pconfig.isWriteObjSizeIndex()) {
+ File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$
+ tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx);
+ try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx);
+ FileChannel idxChannel = fos.getChannel();
+ OutputStream idxStream = Channels
+ .newOutputStream(idxChannel)) {
+ pw.writeObjectSizeIndex(idxStream);
+ idxChannel.force(true);
}
}
@@ -1525,6 +1544,11 @@ public class GC {
*/
public long numberOfBitmaps;
+ /**
+ * The number of objects in the size-index of the packs
+ */
+ public long numberOfSizeIndexedObjects;
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
@@ -1540,6 +1564,8 @@ public class GC {
b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
+ b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$
+ .append(numberOfSizeIndexedObjects);
return b.toString();
}
}
@@ -1548,7 +1574,7 @@ public class GC {
* Returns information about objects and pack files for a FileRepository.
*
* @return information about objects and pack files for a FileRepository
- * @throws java.io.IOException
+ * @throws java.io.IOException
* if an IO error occurred
*/
public RepoStatistics getStatistics() throws IOException {
@@ -1560,16 +1586,16 @@ public class GC {
ret.numberOfPackedObjects += packedObjects;
ret.numberOfPackFiles++;
ret.sizeOfPackedObjects += p.getPackFile().length();
+ ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount();
if (p.getBitmapIndex() != null) {
ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount();
if (latestBitmapTime == 0L) {
latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli();
}
- }
- else if (latestBitmapTime == 0L) {
- ret.numberOfPackFilesSinceBitmap++;
- ret.numberOfObjectsSinceBitmap += packedObjects;
- }
+ } else if (latestBitmapTime == 0L) {
+ ret.numberOfPackFilesSinceBitmap++;
+ ret.numberOfObjectsSinceBitmap += packedObjects;
+ }
}
File objDir = repo.getObjectsDirectory();
String[] fanout = objDir.list();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
index 746e124e1f..d97d5a7ccd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
@@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser {
*/
private File tmpIdx;
+ /**
+ * Path of the object-size index created for the pack, to filter quickly
+ * objects by size in partial clones
+ */
+ private File tmpObjectSizeIndex;
+
/** Read/write handle to {@link #tmpPack} while it is being parsed. */
private RandomAccessFile out;
@@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser {
throws IOException {
tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$
tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$
+
try {
out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$
@@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser {
tmpPack.setReadOnly();
tmpIdx.setReadOnly();
+ if (pconfig.isWriteObjSizeIndex()) {
+ tmpObjectSizeIndex = new File(db.getDirectory(),
+ baseName(tmpPack)
+ + PackExt.OBJECT_SIZE_INDEX.getExtension());
+ writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex());
+ tmpObjectSizeIndex.setReadOnly();
+ }
+
return renameAndOpenPack(getLockMessage());
} finally {
if (def != null)
@@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser {
tmpIdx.deleteOnExit();
if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
tmpPack.deleteOnExit();
+ if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete()
+ && tmpObjectSizeIndex.exists())
+ tmpPack.deleteOnExit();
}
@Override
@@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser {
}
}
+ private void writeObjectSizeIndex(int minSize) throws IOException {
+ try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) {
+ PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter
+ .createWriter(os, minSize);
+ iw.write(getSortedObjectList(null));
+ os.getChannel().force(true);
+ }
+ }
+
private PackLock renameAndOpenPack(String lockMessage)
throws IOException {
if (!keepEmpty && getObjectCount() == 0) {
@@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser {
JGitText.get().cannotMoveIndexTo, finalIdx), e);
}
+ if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) {
+ PackFile finalObjectSizeIndex = finalPack
+ .create(PackExt.OBJECT_SIZE_INDEX);
+ try {
+ FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex,
+ StandardCopyOption.ATOMIC_MOVE);
+ } catch (IOException e) {
+ cleanupTemporaryFiles();
+ keep.unlock();
+ if (!finalPack.delete())
+ finalPack.deleteOnExit();
+ if (!finalIdx.delete()) {
+ finalIdx.deleteOnExit();
+ }
+ throw new IOException(MessageFormat
+ .format(JGitText.get().cannotMoveIndexTo,
+ finalObjectSizeIndex),
+ e);
+ }
+ }
+
boolean interrupted = false;
try {
FileSnapshot snapshot = FileSnapshot.save(finalPack);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
index 5813d39e9a..f2f54947af 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import java.io.EOFException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -126,6 +128,10 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
private Optionally<PackReverseIndex> reverseIdx = Optionally.empty();
+ private volatile PackObjectSizeIndex loadedObjSizeIdx;
+
+ private volatile boolean attemptLoadObjSizeIdx;
+
private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty();
/**
@@ -210,6 +216,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
}
}
+ private PackObjectSizeIndex objectSizeIndex() throws IOException {
+ if (loadedObjSizeIdx != null) {
+ return loadedObjSizeIdx;
+ }
+
+ if (attemptLoadObjSizeIdx) {
+ return null;
+ }
+
+ synchronized (this) {
+ if (loadedObjSizeIdx != null) {
+ return loadedObjSizeIdx;
+ }
+
+ PackObjectSizeIndex sizeIdx;
+ try {
+ long start = System.currentTimeMillis();
+ PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX);
+ if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) {
+ attemptLoadObjSizeIdx = true;
+ return null;
+ }
+ sizeIdx = PackObjectSizeIndexLoader.load(
+ new FileInputStream(sizeIdxFile.getAbsoluteFile()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(
+ "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$
+ sizeIdxFile.getAbsolutePath(),
+ Float.valueOf(
+ sizeIdxFile.length() / (1024f * 1024)),
+ Long.valueOf(System.currentTimeMillis() - start)));
+ }
+
+ loadedObjSizeIdx = sizeIdx;
+ } catch (InterruptedIOException e) {
+ // don't invalidate the pack, we are interrupted from
+ // another thread
+ return null;
+ } finally {
+ attemptLoadObjSizeIdx = true;
+ }
+ }
+
+ return loadedObjSizeIdx;
+ }
+
/**
* Get the File object which locates this pack on disk.
*
@@ -231,6 +283,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
}
/**
+ * Get the object size index for this pack file
+ *
+ * @return the object size index for this pack file if it exists (null
+ * otherwise)
+ * @throws IOException
+ * problem reading the index
+ */
+ public boolean hasObjectSizeIndex() throws IOException {
+ return objectSizeIndex() != null;
+ }
+
+ /**
+ * Number of objects in the object-size index of this pack
+ *
+ * @return number of objects in the index (0 if either the index is empty or
+ * it doesn't exist)
+ * @throws IOException
+ * if an IO error occurred while reading the index
+ */
+ public long getObjectSizeIndexCount() throws IOException {
+ if (!hasObjectSizeIndex()) {
+ return 0;
+ }
+
+ return objectSizeIndex().getObjectCount();
+ }
+
+ /**
+ * Return the size of the object from the object-size index.
+ *
+ * Caller MUST check that the pack has object-size index
+ * ({@link #hasObjectSizeIndex()}) and that the pack contains the object.
+ *
+ * @param id
+ * object id of an object in the pack
+ * @return size of the object from the index. Negative if the object is not
+ * in the index.
+ * @throws IOException
+ * if an IO error occurred while reading the index
+ */
+ public long getIndexedObjectSize(AnyObjectId id) throws IOException {
+ int idxPos = idx().findPosition(id);
+ if (idxPos < 0) {
+ return -1;
+ }
+
+ PackObjectSizeIndex sizeIdx = objectSizeIndex();
+ if (sizeIdx == null) {
+ throw new IllegalStateException(
+ "Asking indexed size from a pack without object size index"); //$NON-NLS-1$
+ }
+
+ return sizeIdx.getSize(idxPos);
+ }
+
+ /**
* Get name extracted from {@code pack-*.pack} pattern.
*
* @return name extracted from {@code pack-*.pack} pattern.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
index 55e047bd43..97a854b8cd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
@@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter {
long offset = beginObject(type, len);
packOut.compress.write(data, off, len);
packOut.compress.finish();
- return endObject(id, offset);
+ return endObject(id, offset, len, type);
}
@Override
@@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter {
len -= n;
}
packOut.compress.finish();
- return endObject(md.toObjectId(), offset);
+ return endObject(md.toObjectId(), offset, len, type);
}
private long beginObject(int type, long len) throws IOException {
@@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter {
return offset;
}
- private ObjectId endObject(ObjectId id, long offset) {
+ private ObjectId endObject(ObjectId id, long offset, long len, int type) {
PackedObjectInfo obj = new PackedObjectInfo(id);
+ obj.setType(type);
obj.setOffset(offset);
obj.setCRC((int) packOut.crc32.getValue());
+ obj.setFullSize(len);
objectList.add(obj);
objectMap.addIfAbsent(obj);
return id;
@@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter {
p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$
}
+ private static File getFileFor(File packFile, PackExt ext) {
+ String p = packFile.getName();
+ return new File(packFile.getParentFile(),
+ p.substring(0, p.lastIndexOf('.')) + ext.getExtension());
+ }
+
private void beginPack() throws IOException {
objectList = new BlockList<>();
objectMap = new ObjectIdOwnerMap<>();
@@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter {
Collections.sort(objectList);
File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile?
writePackIndex(tmpIdx, packHash, objectList);
-
+ File tmpObjSizeIdx = null;
+ if (pconfig.isWriteObjSizeIndex()) {
+ tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX);
+ writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig);
+ }
PackFile realPack = new PackFile(db.getPackDirectory(),
computeName(objectList), PackExt.PACK);
db.closeAllPackHandles(realPack);
@@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter {
realIdx), e);
}
+ if (tmpObjSizeIdx != null) {
+ PackFile realObjSizeIdx = realPack
+ .create(PackExt.OBJECT_SIZE_INDEX);
+ tmpObjSizeIdx.setReadOnly();
+ FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE);
+ }
+
boolean interrupted = false;
try {
FileSnapshot snapshot = FileSnapshot.save(realPack);
@@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter {
}
}
+ private static void writeObjectSizeIndex(File objIdx,
+ List<PackedObjectInfo> list, PackConfig cfg) throws IOException {
+ try (OutputStream os = new FileOutputStream(objIdx)) {
+ PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter
+ .createWriter(os, cfg.getMinBytesForObjSizeIndex());
+ w.write(list);
+ }
+ }
+
private ObjectId computeName(List<PackedObjectInfo> list) {
SHA1 md = digest().reset();
byte[] buf = buffer();
diff --git a/pom.xml b/pom.xml
index 2ad0c5a348..b1e597b68c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -404,6 +404,19 @@
</rules>
</configuration>
</execution>
+ <execution>
+ <id>enforce-java</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <version>17</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
</executions>
</plugin>