From 66ad43a6c7a1c205c76a26dfe95a3dbe93bfd371 Mon Sep 17 00:00:00 2001 From: Xing Huang Date: Tue, 24 Jan 2023 13:17:40 -0600 Subject: DfsReaderIoStats: Add Commit Graph fields into DfsReaderIoStats We are adding commit-graph loading to the DFS stack and the stats object doesn't have fields to track that. This change replicates the stats of the primary index for the commit-graph. Signed-off-by: Xing Huang Change-Id: I4a657bed50083c4ae8bc9f059d4943d612ea2d49 --- .../internal/storage/dfs/DfsReaderIoStats.java | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java index 5c47425013..5ac7985e97 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java @@ -28,6 +28,9 @@ public class DfsReaderIoStats { /** Total number of cache hits for bitmap indexes. */ long bitmapCacheHit; + /** Total number of cache hits for commit graphs. */ + long commitGraphCacheHit; + /** Total number of complete pack indexes read into memory. */ long readIdx; @@ -37,15 +40,24 @@ public class DfsReaderIoStats { /** Total number of reverse indexes added into memory. */ long readReverseIdx; + /** Total number of complete commit graphs read into memory. */ + long readCommitGraph; + /** Total number of bytes read from pack indexes. */ long readIdxBytes; + /** Total number of bytes read from commit graphs. */ + long readCommitGraphBytes; + /** Total microseconds spent reading pack indexes. */ long readIdxMicros; /** Total microseconds spent creating reverse indexes. */ long readReverseIdxMicros; + /** Total microseconds spent creating commit graphs. */ + long readCommitGraphMicros; + /** Total number of bytes read from bitmap indexes. */ long readBitmapIdxBytes; @@ -122,6 +134,15 @@ public class DfsReaderIoStats { return stats.bitmapCacheHit; } + /** + * Get total number of commit graph cache hits. + * + * @return total number of commit graph cache hits. + */ + public long getCommitGraphCacheHits() { + return stats.commitGraphCacheHit; + } + /** * Get total number of complete pack indexes read into memory. * @@ -140,6 +161,15 @@ public class DfsReaderIoStats { return stats.readReverseIdx; } + /** + * Get total number of times the commit graph read into memory. + * + * @return total number of commit graph read into memory. + */ + public long getReadCommitGraphCount() { + return stats.readCommitGraph; + } + /** * Get total number of complete bitmap indexes read into memory. * @@ -158,6 +188,15 @@ public class DfsReaderIoStats { return stats.readIdxBytes; } + /** + * Get total number of bytes read from commit graphs. + * + * @return total number of bytes read from commit graphs. + */ + public long getCommitGraphBytes() { + return stats.readCommitGraphBytes; + } + /** * Get total microseconds spent reading pack indexes. * @@ -176,6 +215,15 @@ public class DfsReaderIoStats { return stats.readReverseIdxMicros; } + /** + * Get total microseconds spent reading commit graphs. + * + * @return total microseconds spent reading commit graphs. + */ + public long getReadCommitGraphMicros() { + return stats.readCommitGraphMicros; + } + /** * Get total number of bytes read from bitmap indexes. * -- cgit v1.2.3 From 0f3a3fde95b59f4832c3ad4234f7c3f00e06e7a4 Mon Sep 17 00:00:00 2001 From: Dmitrii Filippov Date: Tue, 31 Jan 2023 13:31:59 -0500 Subject: Move MemRefDatabase creation in a separate method. The InMemoryRepository is used in tests (e.g. in gerrit tests) and it can be useful to create a custom MemRefDatabase for some tests. Change-Id: I6fbbbfe04400ea1edc988c8788c8eeb06ca8480a --- .../eclipse/jgit/internal/storage/dfs/InMemoryRepository.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java index 5a8207ed01..583b8b3f6b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java @@ -66,7 +66,16 @@ public class InMemoryRepository extends DfsRepository { InMemoryRepository(Builder builder) { super(builder); objdb = new MemObjDatabase(this); - refdb = new MemRefDatabase(); + refdb = createRefDatabase(); + } + + /** + * Creates a new in-memory ref database. + * + * @return a new in-memory reference database. + */ + protected MemRefDatabase createRefDatabase() { + return new MemRefDatabase(); } /** {@inheritDoc} */ -- cgit v1.2.3 From 97e8b4cc71e3e0f5081e5a426874bdcece972483 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Tue, 31 Jan 2023 23:25:19 +0100 Subject: UploadPack: advertise object-info command if enabled Change-Id: Iad8e5b5f4fdd84bd275eb19ee0d01eb6986d79f2 --- org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index a7ce1d7c21..38c7cb94bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1386,6 +1386,9 @@ public class UploadPack implements Closeable { if (transferConfig.isAllowReceiveClientSID()) { caps.add(OPTION_SESSION_ID); } + if (transferConfig.isAdvertiseObjectInfo()) { + caps.add(COMMAND_OBJECT_INFO); + } return caps; } -- cgit v1.2.3 From b30c75be40d34fc844424f8fc5a71eb3fa55b6e0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Tue, 31 Jan 2023 23:24:30 +0100 Subject: Fix unused exception error-prone warning Ignoring the exception seems intended in this case. Change-Id: I9dedf61b9cb5a6ff39fb141dd5da19143f4f6978 --- .../src/org/eclipse/jgit/internal/storage/file/LooseObjects.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index b9af83d24d..326c5f6457 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -254,7 +254,7 @@ class LooseObjects { // refresh directory to work around NFS caching issue } return getSizeWithoutRefresh(curs, id); - } catch (FileNotFoundException e) { + } catch (FileNotFoundException unused) { if (fileFor(id).exists()) { throw noFile; } -- cgit v1.2.3 From a399bd13b1d754d3e0d9c23fc70b1a5705dc7485 Mon Sep 17 00:00:00 2001 From: Nitzan Gur-Furman Date: Tue, 31 Jan 2023 22:01:21 +0100 Subject: PatchApplier fix - init cache with provided tree This change only affects inCore repositories. Before this change, any file that wasn't part of the patch wasn't read, and therefore wasn't part of the output tree. Change-Id: I246ef957088f17aaf367143f7a0b3af0f8264ffb Bug: Google b/267270348 --- .../org/eclipse/jgit/diff/Unaffected_PostImage | 2 + .../org/eclipse/jgit/diff/Unaffected_PreImage | 2 + .../tst-rsrc/org/eclipse/jgit/diff/XAndY.patch | 32 +++++++++++ .../org/eclipse/jgit/patch/PatchApplierTest.java | 67 +++++++++++++++------- .../src/org/eclipse/jgit/patch/PatchApplier.java | 2 +- 5 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/XAndY.patch (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PostImage new file mode 100644 index 0000000000..6b664d90c4 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PostImage @@ -0,0 +1,2 @@ +This file +should not be changed. \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PreImage new file mode 100644 index 0000000000..6b664d90c4 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/Unaffected_PreImage @@ -0,0 +1,2 @@ +This file +should not be changed. \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/XAndY.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/XAndY.patch new file mode 100644 index 0000000000..35950f3d08 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/XAndY.patch @@ -0,0 +1,32 @@ +diff --git a/X b/X +index a3648a1..2d44096 100644 +--- a/X ++++ b/X +@@ -2,2 +2,3 @@ a + b ++c + d +@@ -16,4 +17,2 @@ p + q +-r +-s + t +@@ -22,4 +21,8 @@ v + w +-x +-y ++0 ++1 ++2 ++3 ++4 ++5 + z +diff --git a/Y b/Y +index 2e65efe..7898192 100644 +--- a/Y ++++ b/Y +@@ -1 +1 @@ +-a +\ No newline at end of file ++a \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java index bcde022d40..893fd61556 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.PatchApplyException; import org.eclipse.jgit.api.errors.PatchFormatException; @@ -71,27 +72,21 @@ public class PatchApplierTest { this.inCore = inCore; } + void init(final String aName) throws Exception { + init(aName, true, true); + } + protected void init(String aName, boolean preExists, boolean postExists) throws Exception { // Patch and pre/postimage are read from data // org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ this.name = aName; if (postExists) { - postImage = IO - .readWholeStream(getTestResource(name + "_PostImage"), 0) - .array(); - expectedText = new String(postImage, StandardCharsets.UTF_8); + expectedText = initPostImage(aName); } - File f = new File(db.getWorkTree(), name); if (preExists) { - preImage = IO - .readWholeStream(getTestResource(name + "_PreImage"), 0) - .array(); - try (Git git = new Git(db)) { - Files.write(f.toPath(), preImage); - git.add().addFilepattern(name).call(); - } + initPreImage(aName); } try (Git git = new Git(db)) { RevCommit base = git.commit().setMessage("PreImage").call(); @@ -99,8 +94,22 @@ public class PatchApplierTest { } } - void init(final String aName) throws Exception { - init(aName, true, true); + protected void initPreImage(String aName) throws Exception { + File f = new File(db.getWorkTree(), aName); + preImage = IO + .readWholeStream(getTestResource(aName + "_PreImage"), 0) + .array(); + try (Git git = new Git(db)) { + Files.write(f.toPath(), preImage); + git.add().addFilepattern(aName).call(); + } + } + + protected String initPostImage(String aName) throws Exception { + postImage = IO + .readWholeStream(getTestResource(aName + "_PostImage"), 0) + .array(); + return new String(postImage, StandardCharsets.UTF_8); } protected Result applyPatch() @@ -118,27 +127,33 @@ public class PatchApplierTest { return PatchApplierTest.class.getClassLoader() .getResourceAsStream("org/eclipse/jgit/diff/" + patchFile); } + void verifyChange(Result result, String aName) throws Exception { verifyChange(result, aName, true); } protected void verifyContent(Result result, String path, boolean exists) throws Exception { + verifyContent(result, path, exists ? expectedText : null); + } + + protected void verifyContent(Result result, String path, + @Nullable String expectedContent) throws Exception { if (inCore) { byte[] output = readBlob(result.getTreeId(), path); - if (!exists) + if (expectedContent == null) assertNull(output); else { assertNotNull(output); - assertEquals(expectedText, + assertEquals(expectedContent, new String(output, StandardCharsets.UTF_8)); } } else { File f = new File(db.getWorkTree(), path); - if (!exists) + if (expectedContent == null) assertFalse(f.exists()); else - checkFile(f, expectedText); + checkFile(f, expectedContent); } } @@ -154,7 +169,7 @@ public class PatchApplierTest { RevWalk rw = tr.getRevWalk()) { db.incrementOpen(); RevTree tree = rw.parseTree(treeish); - try (TreeWalk tw = TreeWalk.forPath(db,path,tree)){ + try (TreeWalk tw = TreeWalk.forPath(db, path, tree)) { if (tw == null) { return null; } @@ -300,7 +315,7 @@ public class PatchApplierTest { assertTrue(result.getPaths().contains("RenameNoHunks")); assertTrue(result.getPaths().contains("nested/subdir/Renamed")); - verifyContent(result,"nested/subdir/Renamed", true); + verifyContent(result, "nested/subdir/Renamed", true); } @Test @@ -312,7 +327,7 @@ public class PatchApplierTest { assertTrue(result.getPaths().contains("RenameWithHunks")); assertTrue(result.getPaths().contains("nested/subdir/Renamed")); - verifyContent(result,"nested/subdir/Renamed", true); + verifyContent(result, "nested/subdir/Renamed", true); } @Test @@ -355,6 +370,16 @@ public class PatchApplierTest { verifyChange(result, "ShiftDown2"); } + @Test + public void testDoesNotAffectUnrelatedFiles() throws Exception { + initPreImage("Unaffected"); + String expectedUnaffectedText = initPostImage("Unaffected"); + init("X"); + + Result result = applyPatch(); + verifyChange(result, "X"); + verifyContent(result, "Unaffected", expectedUnaffectedText); + } } public static class InCore extends Base { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index 2a4de1331d..98a2804ee4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -194,7 +194,7 @@ public class PatchApplier { throw new PatchFormatException(p.getErrors()); } - DirCache dirCache = (inCore()) ? DirCache.newInCore() + DirCache dirCache = inCore() ? DirCache.read(reader, beforeTree) : repo.lockDirCache(); DirCacheBuilder dirCacheBuilder = dirCache.builder(); -- cgit v1.2.3 From 341116103e0eab7b8464a74affadf940652669b1 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Tue, 31 Jan 2023 23:27:27 +0100 Subject: UploadPack: consume delimiter in object-info command The 'size' packet line is an argument, so it must be preceeded by a 0001 delimiter. See also git's t5701-git-serve.sh test, https://github.com/git/git/blob/8b8d9a2/t/t5701-git-serve.sh#L329 Without this fix, the server will choke on the delimiter line, saying PackProtocolException: unexpected To test, I ran Gerrit locally with this fix $ curl -X POST -H 'git-protocol: version=2' -H 'content-type: application/x-git-upload-pack-request' -H 'accept: application/x-git-upload-pack-result' --data $'0018command=object-info\n00010009size\n0031oid d38b1b92bdb2893eb4505667375563f2d6d4086b\n0000' http://localhost:8080/git.git/git-upload-pack => 0008size0032d38b1b92bdb2893eb4505667375563f2d6d4086b 268590000 The same command completes identically on Gitlab (which supports the object-info command) $ curl -X POST -H 'git-protocol: version=2' -H 'content-type: application/x-git-upload-pack-request' -H 'accept: application/x-git-upload-pack-result' --data $'0018command=object-info\n00010009size\n0031oid d38b1b92bdb2893eb4505667375563f2d6d4086b\n0000' https://gitlab.com/gitlab-org/git.git/git-upload-pack => 0008size0032d38b1b92bdb2893eb4505667375563f2d6d4086b 268590000 In this case, the blob is for the COPYING file in the Git source tree, which is 26859 bytes long. Change-Id: Ief4ce1eb9303a3b2479547d7950ef01c7c28f472 --- .../tst/org/eclipse/jgit/transport/UploadPackTest.java | 4 +++- .../src/org/eclipse/jgit/transport/ProtocolV2Parser.java | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java index 91b45e447b..4ad7fa3149 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java @@ -2766,7 +2766,9 @@ public class UploadPackTest { TestV2Hook hook = new TestV2Hook(); ByteArrayInputStream recvStream = uploadPackV2((UploadPack up) -> { up.setProtocolV2Hook(hook); - }, "command=object-info\n", "size", + }, "command=object-info\n", + PacketLineIn.delimiter(), + "size", "oid " + ObjectId.toString(blob1.getId()), "oid " + ObjectId.toString(blob2.getId()), PacketLineIn.end()); PacketLineIn pckIn = new PacketLineIn(recvStream); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java index c4129ff4d0..e437f22d02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java @@ -281,6 +281,12 @@ final class ProtocolV2Parser { return builder.build(); } + if (!PacketLineIn.isDelimiter(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().unexpectedPacketLine, line)); + } + + line = pckIn.readString(); if (!line.equals("size")) { //$NON-NLS-1$ throw new PackProtocolException(MessageFormat .format(JGitText.get().unexpectedPacketLine, line)); -- cgit v1.2.3 From ed2cbd9e8a5714295d0651885ccf3f67088c59b0 Mon Sep 17 00:00:00 2001 From: Saša Živkov Date: Fri, 21 Oct 2022 16:32:03 +0200 Subject: Allow to perform PackedBatchRefUpdate without locking loose refs Add another newBatchUpdate method in the RefDirectory where we can control if the created PackedBatchRefUpdate will lock the loose refs or not. This can be useful in cases when we run programs which have exclusive access to a Git repository and we know that locking loose refs is unnecessary and just a performance loss. Change-Id: I7d0932eb1598a3871a2281b1a049021380234df9 (cherry picked from commit cb90ed08526bd51f04e5d72e3ba3cf5bd30c11e4) --- .../jgit/internal/storage/file/PackedBatchRefUpdate.java | 12 +++++++++--- .../eclipse/jgit/internal/storage/file/RefDirectory.java | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java index 9c1d33dc3c..8b0ea4fcd6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java @@ -86,10 +86,16 @@ import org.eclipse.jgit.util.RefList; */ class PackedBatchRefUpdate extends BatchRefUpdate { private RefDirectory refdb; + private boolean shouldLockLooseRefs; PackedBatchRefUpdate(RefDirectory refdb) { - super(refdb); - this.refdb = refdb; + this(refdb, true); + } + + PackedBatchRefUpdate(RefDirectory refdb, boolean shouldLockLooseRefs) { + super(refdb); + this.refdb = refdb; + this.shouldLockLooseRefs = shouldLockLooseRefs; } /** {@inheritDoc} */ @@ -155,7 +161,7 @@ class PackedBatchRefUpdate extends BatchRefUpdate { refdb.inProcessPackedRefsLock.lock(); try { PackedRefList oldPackedList; - if (!refdb.isInClone()) { + if (!refdb.isInClone() && shouldLockLooseRefs) { locks = lockLooseRefs(pending); if (locks == null) { return; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 07e38147f7..b46ffe3670 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -587,6 +587,21 @@ public class RefDirectory extends RefDatabase { return new PackedBatchRefUpdate(this); } + /** + * Create a new batch update to attempt on this database. + * + * @param shouldLockLooseRefs + * whether loose refs should be locked during the batch ref + * update. Note that this should only be set to {@code false} if + * the application using this ensures that no other ref updates + * run concurrently to avoid lost updates caused by a race. In + * such cases it can improve performance. + * @return a new batch update object + */ + public PackedBatchRefUpdate newBatchUpdate(boolean shouldLockLooseRefs) { + return new PackedBatchRefUpdate(this, shouldLockLooseRefs); + } + /** {@inheritDoc} */ @Override public boolean performsAtomicTransactions() { -- cgit v1.2.3 From eccae7cf0bb1ffd4425b6961597b28a8510fc42c Mon Sep 17 00:00:00 2001 From: Xing Huang Date: Mon, 6 Feb 2023 14:18:16 -0600 Subject: ObjectReader: Allow getCommitGraph to throw IOException ObjectReader#getCommitGraph doesn't report errors loading the commit graph. The caller should be aware of the situation and ultimately decide what to do. Add IOException to ObjectReader#getCommitGraph signature. RevWalk defaults to an empty commit-graph on IO errors. Signed-off-by: Xing Huang Change-Id: I38eeacff76c7f926b6dfb192d1e5916e40770024 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java | 7 +++++-- org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index ae0cf42b12..69b2b5104e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -512,9 +512,12 @@ public abstract class ObjectReader implements AutoCloseable { * (default is * {@value org.eclipse.jgit.lib.CoreConfig#DEFAULT_COMMIT_GRAPH_ENABLE}). * + * @throws IOException + * if it cannot open any of the underlying commit graph. + * * @since 6.5 */ - public Optional getCommitGraph() { + public Optional getCommitGraph() throws IOException { return Optional.empty(); } @@ -661,7 +664,7 @@ public abstract class ObjectReader implements AutoCloseable { } @Override - public Optional getCommitGraph() { + public Optional getCommitGraph() throws IOException{ return delegate().getCommitGraph(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index a66e7c86bd..9da7105566 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -1173,8 +1173,13 @@ public class RevWalk implements Iterable, AutoCloseable { @NonNull CommitGraph commitGraph() { if (commitGraph == null) { - commitGraph = reader != null ? reader.getCommitGraph().orElse(EMPTY) - : EMPTY; + try { + commitGraph = reader != null + ? reader.getCommitGraph().orElse(EMPTY) + : EMPTY; + } catch (IOException e) { + commitGraph = EMPTY; + } } return commitGraph; } -- cgit v1.2.3 From df5b7959bedd43443244247a35a690fa9db66442 Mon Sep 17 00:00:00 2001 From: Xing Huang Date: Mon, 6 Feb 2023 14:18:59 -0600 Subject: DfsPackFile/DfsGC: Write commit graphs and expose in pack JGit knows how to read/write commit graphs but the DFS stack is not using it yet. The DFS garbage collector generates a commit-graph with commits reachable from any ref. The pack is stored as extra stream in the GC pack. DfsPackFile mimicks how other indices are loaded storing the reference in DFS cache. Signed-off-by: Xing Huang Change-Id: I3f94997377986d21a56b300d8358dd27be37f5de --- .../internal/storage/dfs/DfsBlockCacheTest.java | 17 ++- .../storage/dfs/DfsGarbageCollectorTest.java | 130 +++++++++++++++++++++ .../jgit/internal/storage/dfs/DfsText.properties | 1 + .../internal/storage/dfs/DfsGarbageCollector.java | 44 +++++++ .../jgit/internal/storage/dfs/DfsPackFile.java | 76 ++++++++++++ .../jgit/internal/storage/dfs/DfsReader.java | 14 +++ .../eclipse/jgit/internal/storage/dfs/DfsText.java | 1 + 7 files changed, 282 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java index 3dd4190c83..fef0563f48 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java @@ -284,12 +284,14 @@ public class DfsBlockCacheTest { asyncRun(() -> pack.getBitmapIndex(reader)); asyncRun(() -> pack.getPackIndex(reader)); asyncRun(() -> pack.getBitmapIndex(reader)); + asyncRun(() -> pack.getCommitGraph(reader)); } waitForExecutorPoolTermination(); assertEquals(1, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(1, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } @SuppressWarnings("resource") @@ -313,12 +315,15 @@ public class DfsBlockCacheTest { } asyncRun(() -> pack1.getBitmapIndex(reader)); asyncRun(() -> pack2.getBitmapIndex(reader)); + asyncRun(() -> pack1.getCommitGraph(reader)); + asyncRun(() -> pack2.getCommitGraph(reader)); } waitForExecutorPoolTermination(); assertEquals(2, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); assertEquals(2, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(2, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(2, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } @SuppressWarnings("resource") @@ -342,12 +347,15 @@ public class DfsBlockCacheTest { } asyncRun(() -> pack1.getBitmapIndex(reader)); asyncRun(() -> pack2.getBitmapIndex(reader)); + asyncRun(() -> pack1.getCommitGraph(reader)); + asyncRun(() -> pack2.getCommitGraph(reader)); } waitForExecutorPoolTermination(); assertEquals(2, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); assertEquals(2, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(2, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(2, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } @SuppressWarnings("resource") @@ -372,7 +380,9 @@ public class DfsBlockCacheTest { } asyncRun(() -> pack1.getBitmapIndex(reader)); asyncRun(() -> pack1.getPackIndex(reader)); + asyncRun(() -> pack1.getCommitGraph(reader)); asyncRun(() -> pack2.getBitmapIndex(reader)); + asyncRun(() -> pack2.getCommitGraph(reader)); } waitForExecutorPoolTermination(); @@ -380,6 +390,7 @@ public class DfsBlockCacheTest { // Index is loaded once for each repo. assertEquals(2, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(2, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(2, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } @Test @@ -396,12 +407,14 @@ public class DfsBlockCacheTest { asyncRun(() -> pack.getBitmapIndex(reader)); asyncRun(() -> pack.getPackIndex(reader)); asyncRun(() -> pack.getBitmapIndex(reader)); + asyncRun(() -> pack.getCommitGraph(reader)); } waitForExecutorPoolTermination(); assertEquals(1, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(1, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } @Test @@ -420,12 +433,14 @@ public class DfsBlockCacheTest { asyncRun(() -> pack.getBitmapIndex(reader)); asyncRun(() -> pack.getPackIndex(reader)); asyncRun(() -> pack.getBitmapIndex(reader)); + asyncRun(() -> pack.getCommitGraph(reader)); } waitForExecutorPoolTermination(); assertEquals(1, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.INDEX.ordinal()]); assertEquals(1, cache.getMissCount()[PackExt.REVERSE_INDEX.ordinal()]); + assertEquals(1, cache.getMissCount()[PackExt.COMMIT_GRAPH.ordinal()]); } private void resetCache() { @@ -450,7 +465,7 @@ public class DfsBlockCacheTest { repository.branch("/refs/ref2" + repoName).commit() .add("blob2", "blob2" + repoName).parent(commit).create(); } - new DfsGarbageCollector(repo).pack(null); + new DfsGarbageCollector(repo).setWriteCommitGraph(true).pack(null); return repo; } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index f235b2eaa4..ab998951f3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.Collections; import java.util.concurrent.TimeUnit; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.reftable.RefCursor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; @@ -976,10 +977,139 @@ public class DfsGarbageCollectorTest { assertNull(refdb.exactRef(NEXT)); } + @Test + public void produceCommitGraphAllRefsIncludedFromDisk() throws Exception { + String tag = "refs/tags/tag1"; + String head = "refs/heads/head1"; + String nonHead = "refs/something/nonHead"; + + RevCommit rootCommitTagged = git.branch(tag).commit().message("0") + .noParents().create(); + RevCommit headTip = git.branch(head).commit().message("1") + .parent(rootCommitTagged).create(); + RevCommit nonHeadTip = git.branch(nonHead).commit().message("2") + .parent(rootCommitTagged).create(); + + gcWithCommitGraph(); + + assertEquals(2, odb.getPacks().length); + DfsPackFile gcPack = odb.getPacks()[0]; + assertEquals(GC, gcPack.getPackDescription().getPackSource()); + + DfsReader reader = odb.newReader(); + CommitGraph cg = gcPack.getCommitGraph(reader); + assertNotNull(cg); + + assertTrue("all commits in commit graph", cg.getCommitCnt() == 3); + // GC packed + assertTrue("tag referenced commit is in graph", + cg.findGraphPosition(rootCommitTagged) != -1); + assertTrue("head referenced commit is in graph", + cg.findGraphPosition(headTip) != -1); + // GC_REST packed + assertTrue("nonHead referenced commit is in graph", + cg.findGraphPosition(nonHeadTip) != -1); + } + + @Test + public void produceCommitGraphAllRefsIncludedFromCache() throws Exception { + String tag = "refs/tags/tag1"; + String head = "refs/heads/head1"; + String nonHead = "refs/something/nonHead"; + + RevCommit rootCommitTagged = git.branch(tag).commit().message("0") + .noParents().create(); + RevCommit headTip = git.branch(head).commit().message("1") + .parent(rootCommitTagged).create(); + RevCommit nonHeadTip = git.branch(nonHead).commit().message("2") + .parent(rootCommitTagged).create(); + + gcWithCommitGraph(); + + assertEquals(2, odb.getPacks().length); + DfsPackFile gcPack = odb.getPacks()[0]; + assertEquals(GC, gcPack.getPackDescription().getPackSource()); + + DfsReader reader = odb.newReader(); + gcPack.getCommitGraph(reader); + // Invoke cache hit + CommitGraph cachedCG = gcPack.getCommitGraph(reader); + assertNotNull(cachedCG); + assertTrue("commit graph have been read from disk once", + reader.stats.readCommitGraph == 1); + assertTrue("commit graph read contains content", + reader.stats.readCommitGraphBytes > 0); + assertTrue("commit graph read time is recorded", + reader.stats.readCommitGraphMicros > 0); + + assertTrue("all commits in commit graph", cachedCG.getCommitCnt() == 3); + // GC packed + assertTrue("tag referenced commit is in graph", + cachedCG.findGraphPosition(rootCommitTagged) != -1); + assertTrue("head referenced commit is in graph", + cachedCG.findGraphPosition(headTip) != -1); + // GC_REST packed + assertTrue("nonHead referenced commit is in graph", + cachedCG.findGraphPosition(nonHeadTip) != -1); + } + + @Test + public void noCommitGraphWithoutGcPack() throws Exception { + String nonHead = "refs/something/nonHead"; + RevCommit nonHeadCommit = git.branch(nonHead).commit() + .message("nonhead").noParents().create(); + commit().message("unreachable").parent(nonHeadCommit).create(); + + gcWithCommitGraph(); + + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + assertNull(pack.getCommitGraph(odb.newReader())); + } + } + + @Test + public void commitGraphWithoutGCrestPack() throws Exception { + String head = "refs/heads/head1"; + RevCommit headCommit = git.branch(head).commit().message("head") + .noParents().create(); + RevCommit unreachableCommit = commit().message("unreachable") + .parent(headCommit).create(); + + gcWithCommitGraph(); + + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC) { + CommitGraph cg = pack.getCommitGraph(odb.newReader()); + assertNotNull(cg); + assertTrue("commit graph only contains 1 commit", + cg.getCommitCnt() == 1); + assertTrue("head exists in commit graph", + cg.findGraphPosition(headCommit) != -1); + assertTrue("unreachable commit does not exist in commit graph", + cg.findGraphPosition(unreachableCommit) == -1); + } else if (d.getPackSource() == UNREACHABLE_GARBAGE) { + CommitGraph cg = pack.getCommitGraph(odb.newReader()); + assertNull(cg); + } else { + fail("unexpected " + d.getPackSource()); + break; + } + } + } + private TestRepository.CommitBuilder commit() { return git.commit(); } + private void gcWithCommitGraph() throws IOException { + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.setWriteCommitGraph(true); + run(gc); + } + private void gcNoTtl() throws IOException { DfsGarbageCollector gc = new DfsGarbageCollector(repo); gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties index 2c4bd06a33..d9d43dc71f 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties @@ -1,4 +1,5 @@ cannotReadIndex=Cannot read index {0} +cannotReadCommitGraph=Cannot read commit graph {0} shortReadOfBlock=Short read of block at {0} in pack {1}; expected {2} bytes, received only {3} shortReadOfIndex=Short read of index {0} willNotStoreEmptyPack=Cannot store empty pack diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 26d5b5b176..66bcf73987 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -18,6 +18,7 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; @@ -34,8 +35,11 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; +import org.eclipse.jgit.internal.storage.commitgraph.GraphCommits; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.file.PackReverseIndex; @@ -75,6 +79,7 @@ public class DfsGarbageCollector { private PackConfig packConfig; private ReftableConfig reftableConfig; private boolean convertToReftable = true; + private boolean writeCommitGraph; private boolean includeDeletes; private long reftableInitialMinUpdateIndex = 1; private long reftableInitialMaxUpdateIndex = 1; @@ -278,6 +283,20 @@ public class DfsGarbageCollector { return this; } + /** + * Toggle commit graph generation. + *

+ * False by default. + * + * @param enable + * Allow/Disallow commit graph generation. + * @return {@code this} + */ + public DfsGarbageCollector setWriteCommitGraph(boolean enable) { + writeCommitGraph = enable; + return this; + } + /** * Create a single new pack file containing all of the live objects. *

@@ -642,6 +661,10 @@ public class DfsGarbageCollector { writeReftable(pack); } + if (source == GC) { + writeCommitGraph(pack, pm); + } + try (DfsOutputStream out = objdb.writeFile(pack, PACK)) { pw.writePack(pm, pm, out); pack.addFileExt(PACK); @@ -724,4 +747,25 @@ public class DfsGarbageCollector { pack.setReftableStats(writer.getStats()); } } + + private void writeCommitGraph(DfsPackDescription pack, ProgressMonitor pm) + throws IOException { + if (!writeCommitGraph || !objdb.getShallowCommits().isEmpty()) { + return; + } + + Set allTips = refsBefore.stream().map(Ref::getObjectId) + .collect(Collectors.toUnmodifiableSet()); + + try (DfsOutputStream out = objdb.writeFile(pack, COMMIT_GRAPH); + RevWalk pool = new RevWalk(ctx)) { + GraphCommits gcs = GraphCommits.fromWalk(pm, allTips, pool); + CountingOutputStream cnt = new CountingOutputStream(out); + CommitGraphWriter writer = new CommitGraphWriter(gcs); + writer.write(pm, cnt); + pack.addFileExt(COMMIT_GRAPH); + pack.setFileSize(COMMIT_GRAPH, cnt.getCount()); + pack.setBlockSize(COMMIT_GRAPH, out.blockSize()); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 15511fed30..411777c7ad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -14,6 +14,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; @@ -37,6 +38,8 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphLoader; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.file.PackReverseIndex; @@ -69,6 +72,9 @@ public final class DfsPackFile extends BlockBasedFile { /** Index of compressed bitmap mapping entire object graph. */ private volatile PackBitmapIndex bitmapIndex; + /** Index of compressed commit graph mapping entire object graph. */ + private volatile CommitGraph commitGraph; + /** * Objects we have tried to read, and discovered to be corrupt. *

@@ -215,6 +221,43 @@ public final class DfsPackFile extends BlockBasedFile { return bitmapIndex; } + /** + * Get the Commit Graph for this PackFile. + * + * @param ctx + * reader context to support reading from the backing store if + * the index is not already loaded in memory. + * @return {@link org.eclipse.jgit.internal.storage.commitgraph.CommitGraph}, + * null if pack doesn't have it. + * @throws java.io.IOException + * the Commit Graph is not available, or is corrupt. + */ + public CommitGraph getCommitGraph(DfsReader ctx) throws IOException { + if (invalid || isGarbage() || !desc.hasFileExt(COMMIT_GRAPH)) { + return null; + } + + if (commitGraph != null) { + return commitGraph; + } + + DfsStreamKey commitGraphKey = desc.getStreamKey(COMMIT_GRAPH); + AtomicBoolean cacheHit = new AtomicBoolean(true); + DfsBlockCache.Ref cgref = cache + .getOrLoadRef(commitGraphKey, REF_POSITION, () -> { + cacheHit.set(false); + return loadCommitGraph(ctx, commitGraphKey); + }); + if (cacheHit.get()) { + ctx.stats.commitGraphCacheHit++; + } + CommitGraph cg = cgref.get(); + if (commitGraph == null && cg != null) { + commitGraph = cg; + } + return commitGraph; + } + PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException { if (reverseIndex != null) { return reverseIndex; @@ -1081,4 +1124,37 @@ public final class DfsPackFile extends BlockBasedFile { desc.getFileName(BITMAP_INDEX)), e); } } + + private DfsBlockCache.Ref loadCommitGraph(DfsReader ctx, + DfsStreamKey cgkey) throws IOException { + ctx.stats.readCommitGraph++; + long start = System.nanoTime(); + try (ReadableChannel rc = ctx.db.openFile(desc, COMMIT_GRAPH)) { + long size; + CommitGraph cg; + try { + InputStream in = Channels.newInputStream(rc); + int wantSize = 8192; + int bs = rc.blockSize(); + if (0 < bs && bs < wantSize) { + bs = (wantSize / bs) * bs; + } else if (bs <= 0) { + bs = wantSize; + } + in = new BufferedInputStream(in, bs); + cg = CommitGraphLoader.read(in); + } finally { + size = rc.position(); + ctx.stats.readCommitGraphBytes += size; + ctx.stats.readCommitGraphMicros += elapsedMicros(start); + } + commitGraph = cg; + return new DfsBlockCache.Ref<>(cgkey, REF_POSITION, size, cg); + } catch (IOException e) { + throw new IOException( + MessageFormat.format(DfsText.get().cannotReadCommitGraph, + desc.getFileName(COMMIT_GRAPH)), + e); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index d043b05fb9..8d8a766b0f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -31,6 +32,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; @@ -121,6 +123,18 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { return null; } + /** {@inheritDoc} */ + @Override + public Optional getCommitGraph() throws IOException { + for (DfsPackFile pack : db.getPacks()) { + CommitGraph cg = pack.getCommitGraph(this); + if (cg != null) { + return Optional.of(cg); + } + } + return Optional.empty(); + } + /** {@inheritDoc} */ @Override public Collection getCachedPacksAndUpdate( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java index df565e568d..f36ec06d3f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java @@ -28,6 +28,7 @@ public class DfsText extends TranslationBundle { // @formatter:off /***/ public String cannotReadIndex; + /***/ public String cannotReadCommitGraph; /***/ public String shortReadOfBlock; /***/ public String shortReadOfIndex; /***/ public String willNotStoreEmptyPack; -- cgit v1.2.3 From 9424052f2797fabd8e1cee7f404eddadc7e3f72b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 10 Feb 2023 21:05:47 +0100 Subject: Add pack options to preserve and prune old pack files Add the options - pack.preserveOldPacks - pack.prunePreserved This allows to configure in git config if old packs should be preserved during gc and pruned during the next gc. The original implementation in 91132bb0 only allows to set these options using the API. Change-Id: I5b23ab4f317d12f5ccd234401419913e8263cc9a --- Documentation/config-options.md | 4 ++-- org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java | 18 ++++++++++++------ .../src/org/eclipse/jgit/lib/ConfigConstants.java | 14 ++++++++++++++ .../src/org/eclipse/jgit/storage/pack/PackConfig.java | 6 ++++++ 4 files changed, 34 insertions(+), 8 deletions(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 169a786579..5d76483acf 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -100,8 +100,8 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.depth` | `50` | ✅ | Maximum depth of delta chain set up for the pack writer. | | `pack.indexVersion` | `2` | ✅ | Pack index file format version. | | `pack.minSizePreventRacyPack` | `100 MiB` | ⃞ | Minimum packfile size for which we wait before opening a newly written pack to prevent its lastModified timestamp could be racy if `pack.waitPreventRacyPack` is `true`. | -| `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs in a preserved directory. | -| `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. | +| `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs during gc in the `objects/pack/preserved` directory. This can avoid rare races between gc removing pack files and other concurrent operations. If this option is false data loss can occur in rare cases when an object is believed to be unreferenced when object repacking is running, and then garbage collection deletes it while another concurrent operation references this object shortly before garbage collection deletes it. When this happens, a new reference is created which points to a now missing object. | +| `pack.prunePreserved` | `false` | ⃞ | Whether to prune preserved pack files from the previous run of gc from the `objects/pack/preserved` directory. This helps to limit the additional storage space needed to preserve old packs when `pack.preserveOldPacks = true`. | | `pack.reuseDeltas` | `true` |⃞ | Whether to reuse deltas existing in repository. | | `pack.reuseObjects` | `true` | ⃞ | Whether to reuse existing objects representation in repository. | | `pack.searchForReuseTimeout` | | ⃞ | Search for reuse phase timeout. Expressed as a `Duration`, i.e.: `50sec`. | diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java index 177be7066d..c87f0b6dcf 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.pgm; +import org.eclipse.jgit.api.GarbageCollectCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.TextProgressMonitor; @@ -21,20 +22,25 @@ class Gc extends TextBuiltin { private boolean aggressive; @Option(name = "--preserve-oldpacks", usage = "usage_PreserveOldPacks") - private boolean preserveOldPacks; + private Boolean preserveOldPacks; @Option(name = "--prune-preserved", usage = "usage_PrunePreserved") - private boolean prunePreserved; + private Boolean prunePreserved; /** {@inheritDoc} */ @Override protected void run() { Git git = Git.wrap(db); try { - git.gc().setAggressive(aggressive) - .setPreserveOldPacks(preserveOldPacks) - .setPrunePreserved(prunePreserved) - .setProgressMonitor(new TextProgressMonitor(errw)).call(); + GarbageCollectCommand command = git.gc().setAggressive(aggressive) + .setProgressMonitor(new TextProgressMonitor(errw)); + if (preserveOldPacks != null) { + command.setPreserveOldPacks(preserveOldPacks.booleanValue()); + } + if (prunePreserved != null) { + command.setPrunePreserved(prunePreserved.booleanValue()); + } + command.call(); } catch (GitAPIException e) { throw die(e.getMessage(), e); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 6f76326bc9..7d3430ffc5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -771,4 +771,18 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT = "searchforreusetimeout"; + /** + * The "pack.preserveOldPacks" key + * + * @since 5.13.2 + */ + public static final String CONFIG_KEY_PRESERVE_OLD_PACKS = "preserveoldpacks"; + + /** + * The "pack.prunePreserved" key + * + * @since 5.13.2 + */ + public static final String CONFIG_KEY_PRUNE_PRESERVED = "prunepreserved"; + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a10f6cf88a..163e475887 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -37,6 +37,8 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYP import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRESERVE_OLD_PACKS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRUNE_PRESERVED; import java.time.Duration; import java.util.concurrent.Executor; @@ -1267,6 +1269,10 @@ public class PackConfig { setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK, getMinSizePreventRacyPack())); + setPreserveOldPacks(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_PRESERVE_OLD_PACKS, DEFAULT_PRESERVE_OLD_PACKS)); + setPrunePreserved(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_PRUNE_PRESERVED, DEFAULT_PRUNE_PRESERVED)); } /** {@inheritDoc} */ -- cgit v1.2.3 From 5b9ca7df42c0524140e24faec73d663beb7202fb Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 17 Jan 2023 10:01:29 -0800 Subject: PackIndex: expose the position of an object-id in the index The primary index returns the offset in the pack for an objectId. Internally it keeps the object-ids in lexicographical order, but doesn't expose an API to find the position of an object-id in that list. This is needed for the object-size index, that we want to store as "position-in-idx, size". Add a #findPosition(object-id) method to the PackIndex interface to know where an object-id sits in the ordered list of ids in the pack. Note that this index position is over the list of ordered object-ids, while reverse-index position is over the list of objects in packed order. Change-Id: I89fa146599e347a26d3012d3477d7f5bbbda7ba4 --- .../internal/storage/file/PackIndexTestCase.java | 39 ++++++++++++++++ .../jgit/internal/storage/file/PackIndex.java | 11 +++++ .../jgit/internal/storage/file/PackIndexV1.java | 54 +++++++++++++++++++--- .../jgit/internal/storage/file/PackIndexV2.java | 12 +++++ 4 files changed, 109 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java index 910b928864..67bba18e2b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java @@ -25,6 +25,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.ObjectId; import org.junit.Test; public abstract class PackIndexTestCase extends RepositoryTestCase { @@ -122,6 +123,37 @@ public abstract class PackIndexTestCase extends RepositoryTestCase { assertFalse(iter.hasNext()); } + @Test + public void testEntriesPositionsRamdomAccess() { + assertEquals(4, smallIdx.findPosition(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); + assertEquals(7, smallIdx.findPosition(ObjectId + .fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); + assertEquals(0, smallIdx.findPosition(ObjectId + .fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); + } + + @Test + public void testEntriesPositionsWithIteratorOrder() { + int i = 0; + for (MutableEntry me : smallIdx) { + assertEquals(smallIdx.findPosition(me.toObjectId()), i); + i++; + } + i = 0; + for (MutableEntry me : denseIdx) { + assertEquals(denseIdx.findPosition(me.toObjectId()), i); + i++; + } + } + + @Test + public void testEntriesPositionsObjectNotInPack() { + assertEquals(-1, smallIdx.findPosition(ObjectId.zeroId())); + assertEquals(-1, smallIdx.findPosition(ObjectId + .fromString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))); + } + /** * Compare offset from iterator entries with output of findOffset() method. */ @@ -135,6 +167,13 @@ public abstract class PackIndexTestCase extends RepositoryTestCase { } } + @Test + public void testEntriesOffsetsObjectNotInPack() { + assertEquals(-1, smallIdx.findOffset(ObjectId.zeroId())); + assertEquals(-1, smallIdx.findOffset(ObjectId + .fromString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))); + } + /** * Compare offset from iterator entries with output of getOffset() method. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 942cc96745..f4f62d4205 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -240,6 +240,17 @@ public abstract class PackIndex */ public abstract long findOffset(AnyObjectId objId); + /** + * Locate the position of this id in the list of object-ids in the index + * + * @param objId + * name of the object to locate within the index + * @return position of the object-id in the lexicographically ordered list + * of ids stored in this index; -1 if the object does not exist in + * this index and is thus not stored in the associated pack. + */ + public abstract int findPosition(AnyObjectId objId); + /** * Retrieve stored CRC32 checksum of the requested object raw-data * (including header). diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index eb0ac6a062..fff410b4ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -32,6 +32,8 @@ import org.eclipse.jgit.util.NB; class PackIndexV1 extends PackIndex { private static final int IDX_HDR_LEN = 256 * 4; + private static final int RECORD_SIZE = 4 + Constants.OBJECT_ID_LENGTH; + private final long[] idxHeader; byte[][] idxdata; @@ -131,8 +133,50 @@ class PackIndexV1 extends PackIndex { public long findOffset(AnyObjectId objId) { final int levelOne = objId.getFirstByte(); byte[] data = idxdata[levelOne]; - if (data == null) + int pos = levelTwoPosition(objId, data); + if (pos < 0) { + return -1; + } + // The records are (offset, objectid), pos points to objectId + int b0 = data[pos - 4] & 0xff; + int b1 = data[pos - 3] & 0xff; + int b2 = data[pos - 2] & 0xff; + int b3 = data[pos - 1] & 0xff; + return (((long) b0) << 24) | (b1 << 16) | (b2 << 8) | (b3); + } + + /** {@inheritDoc} */ + @Override + public int findPosition(AnyObjectId objId) { + int levelOne = objId.getFirstByte(); + int levelTwo = levelTwoPosition(objId, idxdata[levelOne]); + if (levelTwo < 0) { return -1; + } + long objsBefore = levelOne == 0 ? 0 : idxHeader[levelOne - 1]; + return (int) objsBefore + ((levelTwo - 4) / RECORD_SIZE); + } + + /** + * Find position in level two data of this objectId + * + * Records are (offset, objectId), so to read the corresponding offset, + * caller must substract from this position. + * + * @param objId + * ObjectId we are looking for + * @param data + * Blob of second level data with a series of (offset, objectid) + * pairs where we should find objId + * + * @return position in the byte[] where the objectId starts. -1 if not + * found. + */ + private int levelTwoPosition(AnyObjectId objId, byte[] data) { + if (data == null || data.length == 0) { + return -1; + } + int high = data.length / (4 + Constants.OBJECT_ID_LENGTH); int low = 0; do { @@ -142,11 +186,7 @@ class PackIndexV1 extends PackIndex { if (cmp < 0) high = mid; else if (cmp == 0) { - int b0 = data[pos - 4] & 0xff; - int b1 = data[pos - 3] & 0xff; - int b2 = data[pos - 2] & 0xff; - int b3 = data[pos - 1] & 0xff; - return (((long) b0) << 24) | (b1 << 16) | (b2 << 8) | (b3); + return pos; } else low = mid + 1; } while (low < high); @@ -204,7 +244,7 @@ class PackIndexV1 extends PackIndex { } private static int idOffset(int mid) { - return ((4 + Constants.OBJECT_ID_LENGTH) * mid) + 4; + return (RECORD_SIZE * mid) + 4; } private class IndexV1Iterator extends EntriesIterator { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index 09397e316d..7a390060c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -192,6 +192,18 @@ class PackIndexV2 extends PackIndex { return getOffset(levelOne, levelTwo); } + /** {@inheritDoc} */ + @Override + public int findPosition(AnyObjectId objId) { + int levelOne = objId.getFirstByte(); + int levelTwo = binarySearchLevelTwo(objId, levelOne); + if (levelTwo < 0) { + return -1; + } + long objsBefore = levelOne == 0 ? 0 : fanoutTable[levelOne - 1]; + return (int) objsBefore + levelTwo; + } + private long getOffset(int levelOne, int levelTwo) { final long p = NB.decodeUInt32(offset32[levelOne], levelTwo << 2); if ((p & IS_O64) != 0) -- cgit v1.2.3 From 62d0e7be7c2a3c72e50dd9474ea72dfd7ec586b9 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 10 Feb 2023 12:21:01 -0800 Subject: UInt24Array: Array of unsigned ints encoded in 3 bytes. The object size index stores positions of objects in the main index (when ordered by sha1). These positions are per-pack and usually a pack has <16 million objects (there are exceptions but rather rare). It could save some memory storing these positions in three bytes instead of four. Note that these positions are sorted and always positive. Implement a wrapper around a byte[] to access and search "ints" while they are stored as unsigned 3 bytes. Change-Id: Iaa26ce8e2272e706e35fe4cdb648fb6ca7591972 --- .../internal/storage/file/UInt24ArrayTest.java | 79 ++++++++++++++++++ .../jgit/internal/storage/file/UInt24Array.java | 93 ++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/UInt24ArrayTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/UInt24Array.java (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/UInt24ArrayTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/UInt24ArrayTest.java new file mode 100644 index 0000000000..a96c217e4f --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/UInt24ArrayTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2023, Google LLC + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.junit.Assert; +import org.junit.Test; + +public class UInt24ArrayTest { + + private static final byte[] DATA = { 0x00, 0x00, 0x00, // 0 + 0x00, 0x00, 0x05, // 5 + 0x00, 0x00, 0x0a, // 10 + 0x00, 0x00, 0x0f, // 15 + 0x00, 0x00, 0x14, // 20 + 0x00, 0x00, 0x19, // 25 + (byte) 0xff, 0x00, 0x00, // Uint with MSB=1 + (byte) 0xff, (byte) 0xff, (byte) 0xff, // MAX + }; + + private static final UInt24Array asArray = new UInt24Array(DATA); + + @Test + public void uInt24Array_size() { + assertEquals(8, asArray.size()); + } + + @Test + public void uInt24Array_get() { + assertEquals(0, asArray.get(0)); + assertEquals(5, asArray.get(1)); + assertEquals(10, asArray.get(2)); + assertEquals(15, asArray.get(3)); + assertEquals(20, asArray.get(4)); + assertEquals(25, asArray.get(5)); + assertEquals(0xff0000, asArray.get(6)); + assertEquals(0xffffff, asArray.get(7)); + assertThrows(IndexOutOfBoundsException.class, () -> asArray.get(9)); + } + + @Test + public void uInt24Array_getLastValue() { + assertEquals(0xffffff, asArray.getLastValue()); + } + + @Test + public void uInt24Array_find() { + assertEquals(0, asArray.binarySearch(0)); + assertEquals(1, asArray.binarySearch(5)); + assertEquals(2, asArray.binarySearch(10)); + assertEquals(3, asArray.binarySearch(15)); + assertEquals(4, asArray.binarySearch(20)); + assertEquals(5, asArray.binarySearch(25)); + assertEquals(6, asArray.binarySearch(0xff0000)); + assertEquals(7, asArray.binarySearch(0xffffff)); + assertThrows(IllegalArgumentException.class, + () -> asArray.binarySearch(Integer.MAX_VALUE)); + } + + @Test + public void uInt24Array_empty() { + Assert.assertTrue(UInt24Array.EMPTY.isEmpty()); + assertEquals(0, UInt24Array.EMPTY.size()); + assertEquals(-1, UInt24Array.EMPTY.binarySearch(1)); + assertThrows(IndexOutOfBoundsException.class, + () -> UInt24Array.EMPTY.getLastValue()); + assertThrows(IndexOutOfBoundsException.class, + () -> UInt24Array.EMPTY.get(0)); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/UInt24Array.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/UInt24Array.java new file mode 100644 index 0000000000..3a0a18bdb3 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/UInt24Array.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2023, Google LLC + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +/** + * A view of a byte[] as a list of integers stored in 3 bytes. + * + * The ints are stored in big-endian ("network order"), so + * byte[]{aa,bb,cc} becomes the int 0x00aabbcc + */ +final class UInt24Array { + + public static final UInt24Array EMPTY = new UInt24Array( + new byte[0]); + + private static final int ENTRY_SZ = 3; + + private final byte[] data; + + private final int size; + + UInt24Array(byte[] data) { + this.data = data; + this.size = data.length / ENTRY_SZ; + } + + boolean isEmpty() { + return size == 0; + } + + int size() { + return size; + } + + int get(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + int offset = index * ENTRY_SZ; + int e = data[offset] & 0xff; + e <<= 8; + e |= data[offset + 1] & 0xff; + e <<= 8; + e |= data[offset + 2] & 0xff; + return e; + } + + /** + * Search needle in the array. + * + * This assumes a sorted array. + * + * @param needle + * It cannot be bigger than 0xffffff (max unsigned three bytes). + * @return position of the needle in the array, -1 if not found. Runtime + * exception if the value is too big for 3 bytes. + */ + int binarySearch(int needle) { + if ((needle & 0xff000000) != 0) { + throw new IllegalArgumentException("Too big value for 3 bytes"); //$NON-NLS-1$ + } + if (size == 0) { + return -1; + } + int high = size; + if (high == 0) + return -1; + int low = 0; + do { + int mid = (low + high) >>> 1; + int cmp; + cmp = Integer.compare(needle, get(mid)); + if (cmp < 0) + high = mid; + else if (cmp == 0) { + return mid; + } else + low = mid + 1; + } while (low < high); + return -1; + } + + int getLastValue() { + return get(size - 1); + } +} -- cgit v1.2.3 From c9552abaf3286f5b277d64f78c46020fe3978aa3 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 14 Dec 2021 16:36:54 -0800 Subject: PackObjectSizeIndex: interface and impl for the object-size index Operations like "clone --filter=blob:limit=N" or the "object-info" command need to read the size of the objects from the storage. An index would provide those sizes at once rather than having to seek in the packfile. Introduce an interface for the Object-size index. This index returns the inflated size of an object. Not all objects could be indexed (to limit memory usage). This implementation indexes only blobs (no trees, nor commits) *above* certain size threshold (configurable). Lower threshold adds more objects to the index, consumes more memory and provides better performance. 0 means "all blobs" and -1 "disabled". If we don't index everything, for the filter use case is more efficient to index the biggest objects first: the set is small and most objects are filtered by NOT being in the index. For the object-size, the more objects in the index the better, regardless their size. All together, it is more helpful to index above threshold. Change-Id: I9ed608ac240677e199b90ca40d420bcad9231489 --- .../storage/file/PackObjectSizeIndexV1Test.java | 392 +++++++++++++++++++++ .../internal/storage/file/PackObjectSizeIndex.java | 45 +++ .../storage/file/PackObjectSizeIndexLoader.java | 43 +++ .../storage/file/PackObjectSizeIndexV1.java | 224 ++++++++++++ .../storage/file/PackObjectSizeIndexWriter.java | 285 +++++++++++++++ 5 files changed, 989 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1Test.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndex.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexLoader.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1Test.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1Test.java new file mode 100644 index 0000000000..11e3746020 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1Test.java @@ -0,0 +1,392 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.BlockList; +import org.junit.Test; + +public class PackObjectSizeIndexV1Test { + private static final ObjectId OBJ_ID = ObjectId + .fromString("b8b1d53172fb3fb19647adce4b38fab4371c2454"); + + private static final long GB = 1 << 30; + + private static final int MAX_24BITS_UINT = 0xffffff; + + @Test + public void write_24bPositions_32bSizes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + objs.add(blobWithSize(100)); + objs.add(blobWithSize(400)); + objs.add(blobWithSize(200)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x03, // obj count + 0x18, // Unsigned 3 bytes + 0x00, 0x00, 0x00, 0x03, // 3 positions + 0x00, 0x00, 0x00, // positions + 0x00, 0x00, 0x01, // + 0x00, 0x00, 0x02, // + 0x00, // No more positions + 0x00, 0x00, 0x00, 0x64, // size 100 + 0x00, 0x00, 0x01, (byte) 0x90, // size 400 + 0x00, 0x00, 0x00, (byte) 0xc8, // size 200 + 0x00, 0x00, 0x00, 0x00 // 64bit sizes counter + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_32bPositions_32bSizes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new BlockList<>(9_000_000); + // The 24 bit range is full of commits and trees + PackedObjectInfo commit = objInfo(Constants.OBJ_COMMIT, 100); + for (int i = 0; i <= MAX_24BITS_UINT; i++) { + objs.add(commit); + } + objs.add(blobWithSize(100)); + objs.add(blobWithSize(400)); + objs.add(blobWithSize(200)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x03, // obj count + (byte) 0x20, // Signed 4 bytes + 0x00, 0x00, 0x00, 0x03, // 3 positions + 0x01, 0x00, 0x00, 0x00, // positions + 0x01, 0x00, 0x00, 0x01, // + 0x01, 0x00, 0x00, 0x02, // + 0x00, // No more positions + 0x00, 0x00, 0x00, 0x64, // size 100 + 0x00, 0x00, 0x01, (byte) 0x90, // size 400 + 0x00, 0x00, 0x00, (byte) 0xc8, // size 200 + 0x00, 0x00, 0x00, 0x00 // 64bit sizes counter + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_24b32bPositions_32bSizes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new BlockList<>(9_000_000); + // The 24 bit range is full of commits and trees + PackedObjectInfo commit = objInfo(Constants.OBJ_COMMIT, 100); + for (int i = 0; i < MAX_24BITS_UINT; i++) { + objs.add(commit); + } + objs.add(blobWithSize(100)); + objs.add(blobWithSize(400)); + objs.add(blobWithSize(200)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x03, // obj count + 0x18, // 3 bytes + 0x00, 0x00, 0x00, 0x01, // 1 position + (byte) 0xff, (byte) 0xff, (byte) 0xff, + (byte) 0x20, // 4 bytes (32 bits) + 0x00, 0x00, 0x00, 0x02, // 2 positions + 0x01, 0x00, 0x00, 0x00, // positions + 0x01, 0x00, 0x00, 0x01, // + 0x00, // No more positions + 0x00, 0x00, 0x00, 0x64, // size 100 + 0x00, 0x00, 0x01, (byte) 0x90, // size 400 + 0x00, 0x00, 0x00, (byte) 0xc8, // size 200 + 0x00, 0x00, 0x00, 0x00 // 64bit sizes counter + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_64bitsSize() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + objs.add(blobWithSize(100)); + objs.add(blobWithSize(3 * GB)); + objs.add(blobWithSize(4 * GB)); + objs.add(blobWithSize(400)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x04, // Object count + 0x18, // Unsigned 3 byte positions + 0x00, 0x00, 0x00, 0x04, // 4 positions + 0x00, 0x00, 0x00, // positions + 0x00, 0x00, 0x01, // + 0x00, 0x00, 0x02, // + 0x00, 0x00, 0x03, // + 0x00, // No more positions + 0x00, 0x00, 0x00, 0x64, // size 100 + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, // -1 (3GB) + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xfe, // -2 (4GB) + 0x00, 0x00, 0x01, (byte) 0x90, // size 400 + 0x00, 0x00, 0x00, (byte) 0x02, // 64bit sizes counter (2) + 0x00, 0x00, 0x00, 0x00, // size 3Gb + (byte) 0xc0, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x01, // size 4GB + (byte) 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x00 // 128bit sizes counter + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_allObjectsTooSmall() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 1 << 10); + List objs = new ArrayList<>(); + // too small blobs + objs.add(blobWithSize(100)); + objs.add(blobWithSize(200)); + objs.add(blobWithSize(400)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x04, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x00, // Object count + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_onlyBlobsIndexed() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + objs.add(objInfo(Constants.OBJ_COMMIT, 1000)); + objs.add(blobWithSize(100)); + objs.add(objInfo(Constants.OBJ_TAG, 1000)); + objs.add(blobWithSize(400)); + objs.add(blobWithSize(200)); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x03, // Object count + 0x18, // Positions in 3 bytes + 0x00, 0x00, 0x00, 0x03, // 3 entries + 0x00, 0x00, 0x01, // positions + 0x00, 0x00, 0x03, // + 0x00, 0x00, 0x04, // + 0x00, // No more positions + 0x00, 0x00, 0x00, 0x64, // size 100 + 0x00, 0x00, 0x01, (byte) 0x90, // size 400 + 0x00, 0x00, 0x00, (byte) 0xc8, // size 200 + 0x00, 0x00, 0x00, 0x00 // 64bit sizes counter + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void write_noObjects() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + + writer.write(objs); + byte[] expected = new byte[] { -1, 's', 'i', 'z', // header + 0x01, // version + 0x00, 0x00, 0x00, 0x00, // minimum object size + 0x00, 0x00, 0x00, 0x00, // Object count + }; + byte[] output = out.toByteArray(); + assertArrayEquals(expected, output); + } + + @Test + public void read_empty() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + assertEquals(-1, index.getSize(0)); + assertEquals(-1, index.getSize(1)); + assertEquals(-1, index.getSize(1 << 30)); + assertEquals(0, index.getThreshold()); + } + + @Test + public void read_only24bitsPositions() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + objs.add(blobWithSize(100)); + objs.add(blobWithSize(200)); + objs.add(blobWithSize(400)); + objs.add(blobWithSize(1500)); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + assertEquals(100, index.getSize(0)); + assertEquals(200, index.getSize(1)); + assertEquals(400, index.getSize(2)); + assertEquals(1500, index.getSize(3)); + assertEquals(0, index.getThreshold()); + } + + @Test + public void read_only32bitsPositions() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 2000); + List objs = new ArrayList<>(); + PackedObjectInfo smallObj = blobWithSize(100); + for (int i = 0; i <= MAX_24BITS_UINT; i++) { + objs.add(smallObj); + } + objs.add(blobWithSize(1000)); + objs.add(blobWithSize(3000)); + objs.add(blobWithSize(2500)); + objs.add(blobWithSize(1000)); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + assertEquals(-1, index.getSize(5)); + assertEquals(-1, index.getSize(MAX_24BITS_UINT+1)); + assertEquals(3000, index.getSize(MAX_24BITS_UINT+2)); + assertEquals(2500, index.getSize(MAX_24BITS_UINT+3)); + assertEquals(-1, index.getSize(MAX_24BITS_UINT+4)); // Not indexed + assertEquals(2000, index.getThreshold()); + } + + @Test + public void read_24and32BitsPositions() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 2000); + List objs = new ArrayList<>(); + PackedObjectInfo smallObj = blobWithSize(100); + for (int i = 0; i <= MAX_24BITS_UINT; i++) { + if (i == 500 || i == 1000 || i == 1500) { + objs.add(blobWithSize(2500)); + continue; + } + objs.add(smallObj); + } + objs.add(blobWithSize(3000)); + objs.add(blobWithSize(1000)); + objs.add(blobWithSize(2500)); + objs.add(blobWithSize(1000)); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + // 24 bit positions + assertEquals(-1, index.getSize(5)); + assertEquals(2500, index.getSize(500)); + // 32 bit positions + assertEquals(3000, index.getSize(MAX_24BITS_UINT+1)); + assertEquals(-1, index.getSize(MAX_24BITS_UINT+2)); + assertEquals(2500, index.getSize(MAX_24BITS_UINT+3)); + assertEquals(-1, index.getSize(MAX_24BITS_UINT+4)); // Not indexed + assertEquals(2000, index.getThreshold()); + } + + @Test + public void read_only64bits() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, 0); + List objs = new ArrayList<>(); + objs.add(blobWithSize(3 * GB)); + objs.add(blobWithSize(8 * GB)); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + assertEquals(3 * GB, index.getSize(0)); + assertEquals(8 * GB, index.getSize(1)); + assertEquals(0, index.getThreshold()); + } + + @Test + public void read_withMinSize() throws IOException { + int minSize = 1000; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackObjectSizeIndexWriter writer = PackObjectSizeIndexWriter + .createWriter(out, minSize); + List objs = new ArrayList<>(); + objs.add(blobWithSize(3 * GB)); + objs.add(blobWithSize(1500)); + objs.add(blobWithSize(500)); + objs.add(blobWithSize(1000)); + objs.add(blobWithSize(2000)); + + writer.write(objs); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackObjectSizeIndex index = PackObjectSizeIndexLoader.load(in); + assertEquals(3 * GB, index.getSize(0)); + assertEquals(1500, index.getSize(1)); + assertEquals(-1, index.getSize(2)); + assertEquals(1000, index.getSize(3)); + assertEquals(2000, index.getSize(4)); + assertEquals(minSize, index.getThreshold()); + } + + private static PackedObjectInfo blobWithSize(long size) { + return objInfo(Constants.OBJ_BLOB, size); + } + + private static PackedObjectInfo objInfo(int type, long size) { + PackedObjectInfo objectInfo = new PackedObjectInfo(OBJ_ID); + objectInfo.setType(type); + objectInfo.setFullSize(size); + return objectInfo; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndex.java new file mode 100644 index 0000000000..1c3797c509 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndex.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +/** + * Index of object sizes in a pack + * + * It is not guaranteed that the implementation contains the sizes of all + * objects (e.g. it could store only objects over certain threshold). + */ +public interface PackObjectSizeIndex { + + /** + * Returns the inflated size of the object. + * + * @param idxOffset + * position in the pack (as returned from PackIndex) + * @return size of the object, -1 if not found in the index. + */ + long getSize(int idxOffset); + + /** + * Number of objects in the index + * + * @return number of objects in the index + */ + long getObjectCount(); + + + /** + * Minimal size of an object to be included in this index + * + * Cut-off value used at generation time to decide what objects to index. + * + * @return size in bytes + */ + int getThreshold(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexLoader.java new file mode 100644 index 0000000000..9d6941823a --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexLoader.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +/** + * Chooses the specific implementation of the object-size index based on the + * file version. + */ +public class PackObjectSizeIndexLoader { + + /** + * Read an object size index from the stream + * + * @param in + * input stream at the beginning of the object size data + * @return an implementation of the object size index + * @throws IOException + * error reading the streams + */ + public static PackObjectSizeIndex load(InputStream in) throws IOException { + byte[] header = in.readNBytes(4); + if (!Arrays.equals(header, PackObjectSizeIndexWriter.HEADER)) { + throw new IOException("Stream is not an object index"); //$NON-NLS-1$ + } + + int version = in.readNBytes(1)[0]; + if (version != 1) { + throw new IOException("Unknown object size version: " + version); //$NON-NLS-1$ + } + return PackObjectSizeIndexV1.parse(in); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java new file mode 100644 index 0000000000..68befc6ca5 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; + +import org.eclipse.jgit.util.NB; + +/** + * Memory representation of the object-size index + * + * The object size index is a map from position in the primary idx (i.e. + * position of the object-id in lexicographical order) to size. + * + * Most of the positions fit in unsigned 3 bytes (up to 16 million) + */ +class PackObjectSizeIndexV1 implements PackObjectSizeIndex { + + private static final byte BITS_24 = 0x18; + + private static final byte BITS_32 = 0x20; + + private final int threshold; + + private final UInt24Array positions24; + + private final int[] positions32; + + /** + * Parallel array to concat(positions24, positions32) with the size of the + * objects. + * + * A value >= 0 is the size of the object. A negative value means the size + * doesn't fit in an int and |value|-1 is the position for the size in the + * size64 array e.g. a value of -1 is sizes64[0], -2 = sizes64[1], ... + */ + private final int[] sizes32; + + private final long[] sizes64; + + static PackObjectSizeIndex parse(InputStream in) throws IOException { + /** Header and version already out of the input */ + IndexInputStreamReader stream = new IndexInputStreamReader(in); + int threshold = stream.readInt(); // minSize + int objCount = stream.readInt(); + if (objCount == 0) { + return new EmptyPackObjectSizeIndex(threshold); + } + return new PackObjectSizeIndexV1(stream, threshold, objCount); + } + + private PackObjectSizeIndexV1(IndexInputStreamReader stream, int threshold, + int objCount) throws IOException { + this.threshold = threshold; + UInt24Array pos24 = null; + int[] pos32 = null; + + byte positionEncoding; + while ((positionEncoding = stream.readByte()) != 0) { + if (Byte.compareUnsigned(positionEncoding, BITS_24) == 0) { + int sz = stream.readInt(); + pos24 = new UInt24Array(stream.readNBytes(sz * 3)); + } else if (Byte.compareUnsigned(positionEncoding, BITS_32) == 0) { + int sz = stream.readInt(); + pos32 = stream.readIntArray(sz); + } else { + throw new UnsupportedEncodingException( + String.format("Unknown position encoding %s", + Integer.toHexString(positionEncoding))); + } + } + positions24 = pos24 != null ? pos24 : UInt24Array.EMPTY; + positions32 = pos32 != null ? pos32 : new int[0]; + + sizes32 = stream.readIntArray(objCount); + int c64sizes = stream.readInt(); + if (c64sizes == 0) { + sizes64 = new long[0]; + return; + } + sizes64 = stream.readLongArray(c64sizes); + int c128sizes = stream.readInt(); + if (c128sizes != 0) { + // this MUST be 0 (we don't support 128 bits sizes yet) + throw new IOException("Unsupported sizes in object-size-index"); + } + } + + @Override + public long getSize(int idxOffset) { + int pos = -1; + if (!positions24.isEmpty() && idxOffset <= positions24.getLastValue()) { + pos = positions24.binarySearch(idxOffset); + } else if (positions32.length > 0 && idxOffset >= positions32[0]) { + int pos32 = Arrays.binarySearch(positions32, idxOffset); + if (pos32 >= 0) { + pos = pos32 + positions24.size(); + } + } + if (pos < 0) { + return -1; + } + + int objSize = sizes32[pos]; + if (objSize < 0) { + int secondPos = Math.abs(objSize) - 1; + return sizes64[secondPos]; + } + return objSize; + } + + @Override + public long getObjectCount() { + return positions24.size() + positions32.length; + } + + @Override + public int getThreshold() { + return threshold; + } + + /** + * Wrapper to read parsed content from the byte stream + */ + private static class IndexInputStreamReader { + + private final byte[] buffer = new byte[8]; + + private final InputStream in; + + IndexInputStreamReader(InputStream in) { + this.in = in; + } + + int readInt() throws IOException { + int n = in.readNBytes(buffer, 0, 4); + if (n < 4) { + throw new IOException( + "Unable to read a full int from the stream"); + } + return NB.decodeInt32(buffer, 0); + } + + int[] readIntArray(int intsCount) throws IOException { + if (intsCount == 0) { + return new int[0]; + } + + int[] dest = new int[intsCount]; + for (int i = 0; i < intsCount; i++) { + dest[i] = readInt(); + } + return dest; + } + + long readLong() throws IOException { + int n = in.readNBytes(buffer, 0, 8); + if (n < 8) { + throw new IOException( + "Unable to read a full int from the stream"); + } + return NB.decodeInt64(buffer, 0); + } + + long[] readLongArray(int longsCount) throws IOException { + if (longsCount == 0) { + return new long[0]; + } + + long[] dest = new long[longsCount]; + for (int i = 0; i < longsCount; i++) { + dest[i] = readLong(); + } + return dest; + } + + byte readByte() throws IOException { + int n = in.readNBytes(buffer, 0, 1); + if (n != 1) { + throw new IOException("Cannot read byte from stream"); + } + return buffer[0]; + } + + byte[] readNBytes(int sz) throws IOException { + return in.readNBytes(sz); + } + } + + private static class EmptyPackObjectSizeIndex + implements PackObjectSizeIndex { + + private final int threshold; + + EmptyPackObjectSizeIndex(int threshold) { + this.threshold = threshold; + } + + @Override + public long getSize(int idxOffset) { + return -1; + } + + @Override + public long getObjectCount() { + return 0; + } + + @Override + public int getThreshold() { + return threshold; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java new file mode 100644 index 0000000000..ae03797cad --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.NB; + +/** + * Write an object index in the output stream + */ +public abstract class PackObjectSizeIndexWriter { + + private static final int MAX_24BITS_UINT = 0xffffff; + + private static final PackObjectSizeIndexWriter NULL_WRITER = new PackObjectSizeIndexWriter() { + @Override + public void write(List objs) { + // Do nothing + } + }; + + /** Magic constant for the object size index file */ + protected static final byte[] HEADER = { -1, 's', 'i', 'z' }; + + /** + * Returns a writer for the latest index version + * + * @param os + * Output stream where to write the index + * @param minSize + * objects strictly smaller than this size won't be added to the + * index. Negative size won't write AT ALL. Other sizes could write + * an empty index. + * @return the index writer + */ + public static PackObjectSizeIndexWriter createWriter(OutputStream os, + int minSize) { + if (minSize < 0) { + return NULL_WRITER; + } + return new PackObjectSizeWriterV1(os, minSize); + } + + /** + * Add the objects to the index + * + * @param objs + * objects in the pack, in sha1 order. Their position in the list + * matches their position in the primary index. + * @throws IOException + * problem writing to the stream + */ + public abstract void write(List objs) + throws IOException; + + /** + * Object size index v1. + * + * Store position (in the main index) to size as parallel arrays. + * + *

Positions in the main index fit well in unsigned 24 bits (16M) for most + * repositories, but some outliers have even more objects, so we need to + * store also 32 bits positions. + * + *

Sizes are stored as a first array parallel to positions. If a size + * doesn't fit in an element of that array, then we encode there a position + * on the next-size array. This "overflow" array doesn't have entries for + * all positions. + * + * 

+	 *
+	 *      positions       [10, 500, 1000, 1001]
+	 *      sizes (32bits)  [15MB, -1, 6MB, -2]
+   *                          ___/  ______/
+	 *                        /     /
+	 *      sizes (64 bits) [3GB, 6GB]
+	 *
+ * + *

For sizes we use 32 bits as the first level and 64 for the rare objects + * over 2GB. + * + *

A 24/32/64 bits hierarchy of arrays saves space if we have a lot of small + * objects, but wastes space if we have only big ones. The min size to index is + * controlled by conf and in principle we want to index only rather + * big objects (e.g. > 10MB). We could support more dynamics read/write of sizes + * (e.g. 24 only if the threshold will include many of those objects) but it + * complicates a lot code and spec. If needed it could go for a v2 of the protocol. + * + *

Format: + * + *

  • A header with the magic number (4 bytes) + *
  • The index version (1 byte) + *
  • The minimum object size (4 bytes) + *
  • Total count of objects indexed (C, 4 bytes) + * (if count == 0, stop here) + * + * Blocks of + *
  • Size per entry in bits (1 byte, either 24 (0x18) or 32 (0x20)) + *
  • Count of entries (4 bytes) (c, as a signed int) + *
  • positions encoded in s bytes each (i.e s*c bytes) + * + *
  • 0 (as a "size-per-entry = 0", marking end of the section) + * + *
  • 32 bit sizes (C * 4 bytes). Negative size means + * nextLevel[abs(size)-1] + *
  • Count of 64 bit sizes (s64) (or 0 if no more indirections) + *
  • 64 bit sizes (s64 * 8 bytes) + *
  • 0 (end) + */ + static class PackObjectSizeWriterV1 extends PackObjectSizeIndexWriter { + + private final OutputStream os; + + private final int minObjSize; + + private final byte[] intBuffer = new byte[4]; + + PackObjectSizeWriterV1(OutputStream os, int minSize) { + this.os = new BufferedOutputStream(os); + this.minObjSize = minSize; + } + + @Override + public void write(List allObjects) + throws IOException { + os.write(HEADER); + writeUInt8(1); // Version + writeInt32(minObjSize); + + PackedObjectStats stats = countIndexableObjects(allObjects); + int[] indexablePositions = findIndexablePositions(allObjects, + stats.indexableObjs); + writeInt32(indexablePositions.length); // Total # of objects + if (indexablePositions.length == 0) { + os.flush(); + return; + } + + // Positions that fit in 3 bytes + if (stats.pos24Bits > 0) { + writeUInt8(24); + writeInt32(stats.pos24Bits); + applyToRange(indexablePositions, 0, stats.pos24Bits, + this::writeInt24); + } + // Positions that fit in 4 bytes + // We only use 31 bits due to sign, + // but that covers 2 billion objs + if (stats.pos31Bits > 0) { + writeUInt8(32); + writeInt32(stats.pos31Bits); + applyToRange(indexablePositions, stats.pos24Bits, + stats.pos24Bits + stats.pos31Bits, this::writeInt32); + } + writeUInt8(0); + writeSizes(allObjects, indexablePositions, stats.sizeOver2GB); + os.flush(); + } + + private void writeUInt8(int i) throws IOException { + if (i > 255) { + throw new IllegalStateException( + "Number doesn't fit in a single byte"); + } + NB.encodeInt32(intBuffer, 0, i); + os.write(intBuffer, 3, 1); + } + + private void writeInt24(int i) throws IOException { + NB.encodeInt24(intBuffer, 1, i); + os.write(intBuffer, 1, 3); + } + + private void writeInt32(int i) throws IOException { + NB.encodeInt32(intBuffer, 0, i); + os.write(intBuffer); + } + + private void writeSizes(List allObjects, + int[] indexablePositions, int objsBiggerThan2Gb) + throws IOException { + if (indexablePositions.length == 0) { + writeInt32(0); + return; + } + + byte[] sizes64bits = new byte[8 * objsBiggerThan2Gb]; + int s64 = 0; + for (int i = 0; i < indexablePositions.length; i++) { + PackedObjectInfo info = allObjects.get(indexablePositions[i]); + if (info.getFullSize() < Integer.MAX_VALUE) { + writeInt32((int) info.getFullSize()); + } else { + // Size needs more than 32 bits. Store -1 * offset in the + // next table as size. + writeInt32(-1 * (s64 + 1)); + NB.encodeInt64(sizes64bits, s64 * 8, info.getFullSize()); + s64++; + } + } + if (objsBiggerThan2Gb > 0) { + writeInt32(objsBiggerThan2Gb); + os.write(sizes64bits); + } + writeInt32(0); + } + + private int[] findIndexablePositions( + List allObjects, + int indexableObjs) { + int[] positions = new int[indexableObjs]; + int positionIdx = 0; + for (int i = 0; i < allObjects.size(); i++) { + PackedObjectInfo o = allObjects.get(i); + if (!shouldIndex(o)) { + continue; + } + positions[positionIdx++] = i; + } + return positions; + } + + private PackedObjectStats countIndexableObjects( + List objs) { + PackedObjectStats stats = new PackedObjectStats(); + for (int i = 0; i < objs.size(); i++) { + PackedObjectInfo o = objs.get(i); + if (!shouldIndex(o)) { + continue; + } + stats.indexableObjs++; + if (o.getFullSize() > Integer.MAX_VALUE) { + stats.sizeOver2GB++; + } + if (i <= MAX_24BITS_UINT) { + stats.pos24Bits++; + } else { + stats.pos31Bits++; + // i is a positive int, cannot be bigger than this + } + } + return stats; + } + + private boolean shouldIndex(PackedObjectInfo o) { + return (o.getType() == Constants.OBJ_BLOB) + && (o.getFullSize() >= minObjSize); + } + + private static class PackedObjectStats { + int indexableObjs; + + int pos24Bits; + + int pos31Bits; + + int sizeOver2GB; + } + + @FunctionalInterface + interface IntEncoder { + void encode(int i) throws IOException; + } + + private static void applyToRange(int[] allPositions, int start, int end, + IntEncoder encoder) throws IOException { + for (int i = start; i < end; i++) { + encoder.encode(allPositions[i]); + } + } + } +} -- cgit v1.2.3 From 012cb779304ccc6c089be06a2f5c12c293f11453 Mon Sep 17 00:00:00 2001 From: Prudhvi Akhil Alahari Date: Thu, 16 Feb 2023 16:41:55 +0530 Subject: Fix getPackedRefs to not throw NoSuchFileException Since Files.newInputStream is from java.nio package, it throws java.nio.file.NoSuchFileException. This was missed in the change I00da88e. Without this change, getPackedRefs fails with NoSuchFileException when there is no packed-refs file in a project. Change-Id: I93c202ddb73a0a5979af8e4d09e45f5645664b45 Signed-off-by: Prudhvi Akhil Alahari --- .../src/org/eclipse/jgit/internal/storage/file/RefDirectory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index d4ad190ffd..c7322b17bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -35,6 +35,7 @@ import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.security.DigestInputStream; import java.security.MessageDigest; @@ -911,7 +912,7 @@ public class RefDirectory extends RefDatabase { try (InputStream stream = Files .newInputStream(packedRefsFile.toPath())) { // open the file to refresh attributes (on some NFS clients) - } catch (FileNotFoundException e) { + } catch (FileNotFoundException | NoSuchFileException e) { // Ignore as packed-refs may not exist } //$FALL-THROUGH$ -- cgit v1.2.3 From 596c445af22ed9b6e0b7e35de44c127fcb8ecf7d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 4 Jan 2022 11:10:04 -0800 Subject: PackConfig: add entry for minimum size to index The object size index can have up to #(blobs-in-repo) entries, taking a relevant amount of memory. Let operators configure the threshold size to include objects in the size index. The index will include objects with size *at or above* this value (with -1 for none). This is more effective for the filter-by-size case. Lowering the threshold adds more objects to the index. This improves performance at the cost of memory/storage space. For the object-size case, more calls will use the index instead of reading IO. For the filter-by-size case, lower threshold means better granularity (if ObjectReader#isSmallerThan is implemented based only on the index). Change-Id: I6ccd9334adbbc2abf95fde51dbbfc85b8230ade0 --- Documentation/config-options.md | 1 + .../src/org/eclipse/jgit/lib/ConfigConstants.java | 7 +++ .../org/eclipse/jgit/storage/pack/PackConfig.java | 57 ++++++++++++++++++++++ 3 files changed, 65 insertions(+) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 612b8456ae..754accf787 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -99,6 +99,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.deltaCompression` | `true` | ⃞ | Whether the writer will create new deltas on the fly. `true` if the pack writer will create a new delta when either `pack.reuseDeltas` is false, or no suitable delta is available for reuse. | | `pack.depth` | `50` | ✅ | Maximum depth of delta chain set up for the pack writer. | | `pack.indexVersion` | `2` | ✅ | Pack index file format version. | +| `pack.minBytesForObjSizeIndex` | `-1` | ⃞ | Minimum size of an object (inclusive, in bytes) to be included in the size index. -1 to disable the object size index. | | `pack.minSizePreventRacyPack` | `100 MiB` | ⃞ | Minimum packfile size for which we wait before opening a newly written pack to prevent its lastModified timestamp could be racy if `pack.waitPreventRacyPack` is `true`. | | `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs in a preserved directory. | | `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. | diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index ed4bf315e2..608a4842a4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -822,6 +822,13 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_WINDOW_MEMORY = "windowmemory"; + /** + * the "pack.minBytesForObjSizeIndex" key + * + * @since 6.5 + */ + public static final String CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX = "minBytesForObjSizeIndex"; + /** * The "feature" section * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a10f6cf88a..2d0d4ee700 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -36,6 +36,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import java.time.Duration; @@ -234,6 +235,15 @@ public class PackConfig { */ public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0]; + /** + * Default minimum size for an object to be included in the size index: + * {@value} + * + * @see #setMinBytesForObjSizeIndex(int) + * @since 6.5 + */ + public static final int DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX = -1; + /** * Default max time to spend during the search for reuse phase. This * optimization is disabled by default: {@value} @@ -302,6 +312,8 @@ public class PackConfig { private boolean singlePack; + private int minBytesForObjSizeIndex = DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX; + /** * Create a default configuration. */ @@ -369,6 +381,7 @@ public class PackConfig { this.cutDeltaChains = cfg.cutDeltaChains; this.singlePack = cfg.singlePack; this.searchForReuseTimeout = cfg.searchForReuseTimeout; + this.minBytesForObjSizeIndex = cfg.minBytesForObjSizeIndex; } /** @@ -1189,6 +1202,45 @@ public class PackConfig { searchForReuseTimeout = timeout; } + /** + * Minimum size of an object (inclusive) to be added in the object size + * index. + * + * A negative value disables the writing of the object size index. + * + * @return minimum size an object must have to be included in the object + * index. + * @since 6.5 + */ + public int getMinBytesForObjSizeIndex() { + return minBytesForObjSizeIndex; + } + + /** + * Set minimum size an object must have to be included in the object size + * index. + * + * A negative value disables the object index. + * + * @param minBytesForObjSizeIndex + * minimum size (inclusive) of an object to be included in the + * object size index. -1 disables the index. + * @since 6.5 + */ + public void setMinBytesForObjSizeIndex(int minBytesForObjSizeIndex) { + this.minBytesForObjSizeIndex = minBytesForObjSizeIndex; + } + + /** + * Should writers add an object size index when writing a pack. + * + * @return true to write an object-size index with the pack + * @since 6.5 + */ + public boolean isWriteObjSizeIndex() { + return this.minBytesForObjSizeIndex >= 0; + } + /** * Update properties by setting fields from the configuration. * @@ -1267,6 +1319,9 @@ public class PackConfig { setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK, getMinSizePreventRacyPack())); + setMinBytesForObjSizeIndex(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, + DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX)); } /** {@inheritDoc} */ @@ -1302,6 +1357,8 @@ public class PackConfig { b.append(", searchForReuseTimeout") //$NON-NLS-1$ .append(getSearchForReuseTimeout()); b.append(", singlePack=").append(getSinglePack()); //$NON-NLS-1$ + b.append(", minBytesForObjSizeIndex=") //$NON-NLS-1$ + .append(getMinBytesForObjSizeIndex()); return b.toString(); } } -- cgit v1.2.3 From 5d5a0d537697480ae5a7d530bad283a972aadf52 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 20 Feb 2023 21:40:40 +0100 Subject: Externalize strings introduced in c9552aba Change-Id: I81bb78344df61e6eb42622fcef6235d4da0ae052 --- .../resources/org/eclipse/jgit/internal/JGitText.properties | 5 +++++ .../src/org/eclipse/jgit/internal/JGitText.java | 5 +++++ .../jgit/internal/storage/file/PackObjectSizeIndexV1.java | 13 ++++++------- .../internal/storage/file/PackObjectSizeIndexWriter.java | 9 +++++---- 4 files changed, 21 insertions(+), 11 deletions(-) (limited to 'org.eclipse.jgit/src/org/eclipse/jgit') diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index bdf35d8d73..72a3d1cdea 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -111,6 +111,7 @@ cannotPullOnARepoWithState=Cannot pull into a repository with state: {0} cannotRead=Cannot read {0} cannotReadBackDelta=Cannot read delta type {0} cannotReadBlob=Cannot read blob {0} +cannotReadByte=Cannot read byte from stream cannotReadCommit=Cannot read commit {0} cannotReadFile=Cannot read file {0} cannotReadHEAD=cannot read HEAD: {0} {1} @@ -538,6 +539,7 @@ nothingToPush=Nothing to push. notMergedExceptionMessage=Branch was not deleted as it has not been merged yet; use the force option to delete it anyway notShallowedUnshallow=The server sent a unshallow for a commit that wasn''t marked as shallow: {0} noXMLParserAvailable=No XML parser available. +numberDoesntFit=Number doesn't fit in a single byte objectAtHasBadZlibStream=Object at {0} in {1} has bad zlib stream objectIsCorrupt=Object {0} is corrupt: {1} objectIsCorrupt3={0}: object {1}: {2} @@ -773,6 +775,7 @@ truncatedHunkOldLinesMissing=Truncated hunk, at least {0} old lines is missing tSizeMustBeGreaterOrEqual1=tSize must be >= 1 unableToCheckConnectivity=Unable to check connectivity. unableToCreateNewObject=Unable to create new object: {0} +unableToReadFullInt=Unable to read a full int from the stream unableToReadPackfile=Unable to read packfile {0} unableToRemovePath=Unable to remove path ''{0}'' unableToWrite=Unable to write {0} @@ -798,6 +801,7 @@ unknownObject=unknown object unknownObjectInIndex=unknown object {0} found in index but not in pack file unknownObjectType=Unknown object type {0}. unknownObjectType2=unknown +unknownPositionEncoding=Unknown position encoding %s unknownRefStorageFormat=Unknown ref storage format "{0}" unknownRepositoryFormat=Unknown repository format unknownRepositoryFormat2=Unknown repository format "{0}"; expected "0". @@ -825,6 +829,7 @@ unsupportedPackIndexVersion=Unsupported pack index version {0} unsupportedPackVersion=Unsupported pack version {0}. unsupportedReftableVersion=Unsupported reftable version {0}. unsupportedRepositoryDescription=Repository description not supported +unsupportedSizesObjSizeIndex=Unsupported sizes in object-size-index updateRequiresOldIdAndNewId=Update requires both old ID and new ID to be nonzero updatingHeadFailed=Updating HEAD failed updatingReferences=Updating references diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index b502a1a98b..d8720be56f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -139,6 +139,7 @@ public class JGitText extends TranslationBundle { /***/ public String cannotRead; /***/ public String cannotReadBackDelta; /***/ public String cannotReadBlob; + /***/ public String cannotReadByte; /***/ public String cannotReadCommit; /***/ public String cannotReadFile; /***/ public String cannotReadHEAD; @@ -566,6 +567,7 @@ public class JGitText extends TranslationBundle { /***/ public String notMergedExceptionMessage; /***/ public String notShallowedUnshallow; /***/ public String noXMLParserAvailable; + /***/ public String numberDoesntFit; /***/ public String objectAtHasBadZlibStream; /***/ public String objectIsCorrupt; /***/ public String objectIsCorrupt3; @@ -801,6 +803,7 @@ public class JGitText extends TranslationBundle { /***/ public String tSizeMustBeGreaterOrEqual1; /***/ public String unableToCheckConnectivity; /***/ public String unableToCreateNewObject; + /***/ public String unableToReadFullInt; /***/ public String unableToReadPackfile; /***/ public String unableToRemovePath; /***/ public String unableToWrite; @@ -826,6 +829,7 @@ public class JGitText extends TranslationBundle { /***/ public String unknownObjectInIndex; /***/ public String unknownObjectType; /***/ public String unknownObjectType2; + /***/ public String unknownPositionEncoding; /***/ public String unknownRefStorageFormat; /***/ public String unknownRepositoryFormat; /***/ public String unknownRepositoryFormat2; @@ -853,6 +857,7 @@ public class JGitText extends TranslationBundle { /***/ public String unsupportedPackVersion; /***/ public String unsupportedReftableVersion; /***/ public String unsupportedRepositoryDescription; + /***/ public String unsupportedSizesObjSizeIndex; /***/ public String updateRequiresOldIdAndNewId; /***/ public String updatingHeadFailed; /***/ public String updatingReferences; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java index 68befc6ca5..be2ff67e4f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java @@ -14,6 +14,7 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.util.Arrays; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.NB; /** @@ -75,7 +76,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { pos32 = stream.readIntArray(sz); } else { throw new UnsupportedEncodingException( - String.format("Unknown position encoding %s", + String.format(JGitText.get().unknownPositionEncoding, Integer.toHexString(positionEncoding))); } } @@ -92,7 +93,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { int c128sizes = stream.readInt(); if (c128sizes != 0) { // this MUST be 0 (we don't support 128 bits sizes yet) - throw new IOException("Unsupported sizes in object-size-index"); + throw new IOException(JGitText.get().unsupportedSizesObjSizeIndex); } } @@ -145,8 +146,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { int readInt() throws IOException { int n = in.readNBytes(buffer, 0, 4); if (n < 4) { - throw new IOException( - "Unable to read a full int from the stream"); + throw new IOException(JGitText.get().unableToReadFullInt); } return NB.decodeInt32(buffer, 0); } @@ -166,8 +166,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { long readLong() throws IOException { int n = in.readNBytes(buffer, 0, 8); if (n < 8) { - throw new IOException( - "Unable to read a full int from the stream"); + throw new IOException(JGitText.get().unableToReadFullInt); } return NB.decodeInt64(buffer, 0); } @@ -187,7 +186,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { byte readByte() throws IOException { int n = in.readNBytes(buffer, 0, 1); if (n != 1) { - throw new IOException("Cannot read byte from stream"); + throw new IOException(JGitText.get().cannotReadByte); } return buffer[0]; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java index ae03797cad..65a065dd55 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexWriter.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.NB; @@ -70,11 +71,11 @@ public abstract class PackObjectSizeIndexWriter { * Object size index v1. * * Store position (in the main index) to size as parallel arrays. - * + * *

    Positions in the main index fit well in unsigned 24 bits (16M) for most * repositories, but some outliers have even more objects, so we need to * store also 32 bits positions. - * + * *

    Sizes are stored as a first array parallel to positions. If a size * doesn't fit in an element of that array, then we encode there a position * on the next-size array. This "overflow" array doesn't have entries for @@ -88,7 +89,7 @@ public abstract class PackObjectSizeIndexWriter { * / / * sizes (64 bits) [3GB, 6GB] * - * + * *

    For sizes we use 32 bits as the first level and 64 for the rare objects * over 2GB. * @@ -173,7 +174,7 @@ public abstract class PackObjectSizeIndexWriter { private void writeUInt8(int i) throws IOException { if (i > 255) { throw new IllegalStateException( - "Number doesn't fit in a single byte"); + JGitText.get().numberDoesntFit); } NB.encodeInt32(intBuffer, 0, i); os.write(intBuffer, 3, 1); -- cgit v1.2.3