diff options
Diffstat (limited to 'org.eclipse.jgit.test/tst/org/eclipse/jgit/internal')
18 files changed, 1707 insertions, 100 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java index 37ff40bdf7..0e73588c66 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java @@ -51,7 +51,6 @@ public abstract class ObjectReachabilityTestCase } } - /** {@inheritDoc} */ @Override @Before public void setUp() throws Exception { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java index 7679c11098..eeb13cc8b9 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java @@ -32,7 +32,6 @@ public abstract class ReachabilityCheckerTestCase TestRepository<FileRepository> repo; - /** {@inheritDoc} */ @Override @Before public void setUp() throws Exception { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java index 97976564d8..4d05360252 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java @@ -10,24 +10,31 @@ package org.eclipse.jgit.internal.storage.commitgraph; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.COMMIT_GENERATION_UNKNOWN; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.junit.Before; import org.junit.Test; @@ -45,6 +52,7 @@ public class CommitGraphTest extends RepositoryTestCase { public void setUp() throws Exception { super.setUp(); tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); + mockSystemReader.setJGitConfig(new MockConfig()); } @Test @@ -196,11 +204,32 @@ public class CommitGraphTest extends RepositoryTestCase { assertEquals(getGenerationNumber(c8), 5); } + @Test + public void testGraphComputeChangedPaths() throws Exception { + RevCommit a = tr.commit(tr.tree(tr.file("d/f", tr.blob("a")))); + RevCommit b = tr.commit(tr.tree(tr.file("d/f", tr.blob("a"))), a); + RevCommit c = tr.commit(tr.tree(tr.file("d/f", tr.blob("b"))), b); + + writeAndReadCommitGraph(Collections.singleton(c)); + ChangedPathFilter acpf = commitGraph + .getChangedPathFilter(commitGraph.findGraphPosition(a)); + assertTrue(acpf.maybeContains("d".getBytes(UTF_8))); + assertTrue(acpf.maybeContains("d/f".getBytes(UTF_8))); + ChangedPathFilter bcpf = commitGraph + .getChangedPathFilter(commitGraph.findGraphPosition(b)); + assertFalse(bcpf.maybeContains("d".getBytes(UTF_8))); + assertFalse(bcpf.maybeContains("d/f".getBytes(UTF_8))); + ChangedPathFilter ccpf = commitGraph + .getChangedPathFilter(commitGraph.findGraphPosition(c)); + assertTrue(ccpf.maybeContains("d".getBytes(UTF_8))); + assertTrue(ccpf.maybeContains("d/f".getBytes(UTF_8))); + } + void writeAndReadCommitGraph(Set<ObjectId> wants) throws Exception { NullProgressMonitor m = NullProgressMonitor.INSTANCE; try (RevWalk walk = new RevWalk(db)) { CommitGraphWriter writer = new CommitGraphWriter( - GraphCommits.fromWalk(m, wants, walk)); + GraphCommits.fromWalk(m, wants, walk), true); ByteArrayOutputStream os = new ByteArrayOutputStream(); writer.write(m, os); InputStream inputStream = new ByteArrayInputStream( @@ -252,4 +281,41 @@ public class CommitGraphTest extends RepositoryTestCase { RevCommit commit(RevCommit... parents) throws Exception { return tr.commit(parents); } + + private static final class MockConfig extends FileBasedConfig { + private MockConfig() { + super(null, null); + } + + @Override + public void load() throws IOException, ConfigInvalidException { + // Do nothing + } + + @Override + public void save() throws IOException { + // Do nothing + } + + @Override + public boolean isOutdated() { + return false; + } + + @Override + public String toString() { + return "MockConfig"; + } + + @Override + public boolean getBoolean(final String section, final String name, + final boolean defaultValue) { + if (section.equals(ConfigConstants.CONFIG_COMMIT_GRAPH_SECTION) + && name.equals( + ConfigConstants.CONFIG_KEY_READ_CHANGED_PATHS)) { + return true; + } + return defaultValue; + } + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java index 6c5e5e5605..5040a3b6ad 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java @@ -10,21 +10,31 @@ package org.eclipse.jgit.internal.storage.commitgraph; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.Set; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.util.NB; import org.junit.Before; import org.junit.Test; @@ -46,6 +56,7 @@ public class CommitGraphWriterTest extends RepositoryTestCase { os = new ByteArrayOutputStream(); tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); walk = new RevWalk(db); + mockSystemReader.setJGitConfig(new MockConfig()); } @Test @@ -68,7 +79,7 @@ public class CommitGraphWriterTest extends RepositoryTestCase { Set<ObjectId> wants = Collections.singleton(tip); NullProgressMonitor m = NullProgressMonitor.INSTANCE; GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); - writer = new CommitGraphWriter(graphCommits); + writer = new CommitGraphWriter(graphCommits, true); writer.write(m, os); assertEquals(5, graphCommits.size()); @@ -76,11 +87,20 @@ public class CommitGraphWriterTest extends RepositoryTestCase { assertTrue(data.length > 0); byte[] headers = new byte[8]; System.arraycopy(data, 0, headers, 0, 8); - assertArrayEquals(new byte[] {'C', 'G', 'P', 'H', 1, 1, 4, 0}, headers); - assertEquals(CommitGraphConstants.CHUNK_ID_OID_FANOUT, NB.decodeInt32(data, 8)); - assertEquals(CommitGraphConstants.CHUNK_ID_OID_LOOKUP, NB.decodeInt32(data, 20)); - assertEquals(CommitGraphConstants.CHUNK_ID_COMMIT_DATA, NB.decodeInt32(data, 32)); - assertEquals(CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST, NB.decodeInt32(data, 44)); + assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H', 1, 1, 6, 0 }, + headers); + assertEquals(CommitGraphConstants.CHUNK_ID_OID_FANOUT, + NB.decodeInt32(data, 8)); + assertEquals(CommitGraphConstants.CHUNK_ID_OID_LOOKUP, + NB.decodeInt32(data, 20)); + assertEquals(CommitGraphConstants.CHUNK_ID_COMMIT_DATA, + NB.decodeInt32(data, 32)); + assertEquals(CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST, + NB.decodeInt32(data, 44)); + assertEquals(CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_INDEX, + NB.decodeInt32(data, 56)); + assertEquals(CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_DATA, + NB.decodeInt32(data, 68)); } @Test @@ -93,7 +113,7 @@ public class CommitGraphWriterTest extends RepositoryTestCase { Set<ObjectId> wants = Collections.singleton(tip); NullProgressMonitor m = NullProgressMonitor.INSTANCE; GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); - writer = new CommitGraphWriter(graphCommits); + writer = new CommitGraphWriter(graphCommits, true); writer.write(m, os); assertEquals(4, graphCommits.size()); @@ -101,13 +121,281 @@ public class CommitGraphWriterTest extends RepositoryTestCase { assertTrue(data.length > 0); byte[] headers = new byte[8]; System.arraycopy(data, 0, headers, 0, 8); - assertArrayEquals(new byte[] {'C', 'G', 'P', 'H', 1, 1, 3, 0}, headers); - assertEquals(CommitGraphConstants.CHUNK_ID_OID_FANOUT, NB.decodeInt32(data, 8)); - assertEquals(CommitGraphConstants.CHUNK_ID_OID_LOOKUP, NB.decodeInt32(data, 20)); - assertEquals(CommitGraphConstants.CHUNK_ID_COMMIT_DATA, NB.decodeInt32(data, 32)); + assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H', 1, 1, 5, 0 }, + headers); + assertEquals(CommitGraphConstants.CHUNK_ID_OID_FANOUT, + NB.decodeInt32(data, 8)); + assertEquals(CommitGraphConstants.CHUNK_ID_OID_LOOKUP, + NB.decodeInt32(data, 20)); + assertEquals(CommitGraphConstants.CHUNK_ID_COMMIT_DATA, + NB.decodeInt32(data, 32)); + assertEquals(CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_INDEX, + NB.decodeInt32(data, 44)); + assertEquals(CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_DATA, + NB.decodeInt32(data, 56)); + } + + static HashSet<String> changedPathStrings(byte[] data) { + int oidf_offset = -1; + int bidx_offset = -1; + int bdat_offset = -1; + for (int i = 8; i < data.length - 4; i += 12) { + switch (NB.decodeInt32(data, i)) { + case CommitGraphConstants.CHUNK_ID_OID_FANOUT: + oidf_offset = (int) NB.decodeInt64(data, i + 4); + break; + case CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_INDEX: + bidx_offset = (int) NB.decodeInt64(data, i + 4); + break; + case CommitGraphConstants.CHUNK_ID_BLOOM_FILTER_DATA: + bdat_offset = (int) NB.decodeInt64(data, i + 4); + break; + } + } + assertTrue(oidf_offset > 0); + assertTrue(bidx_offset > 0); + assertTrue(bdat_offset > 0); + bdat_offset += 12; // skip version, hash count, bits per entry + int commit_count = NB.decodeInt32(data, oidf_offset + 255 * 4); + int[] changed_path_length_cumuls = new int[commit_count]; + for (int i = 0; i < commit_count; i++) { + changed_path_length_cumuls[i] = NB.decodeInt32(data, + bidx_offset + i * 4); + } + HashSet<String> changed_paths = new HashSet<>(); + for (int i = 0; i < commit_count; i++) { + int prior_cumul = i == 0 ? 0 : changed_path_length_cumuls[i - 1]; + String changed_path = ""; + for (int j = prior_cumul; j < changed_path_length_cumuls[i]; j++) { + changed_path += data[bdat_offset + j] + ","; + } + changed_paths.add(changed_path); + } + return changed_paths; + } + + /** + * Expected value generated using the following: + * + * <pre> + * # apply into git-repo: https://lore.kernel.org/git/cover.1684790529.git.jonathantanmy@google.com/ + * (cd git-repo; make) + * git-repo/bin-wrappers/git init tested + * (cd tested; touch foo.txt; mkdir -p onedir/twodir; touch onedir/twodir/bar.txt) + * git-repo/bin-wrappers/git -C tested add foo.txt onedir + * git-repo/bin-wrappers/git -C tested commit -m first_commit + * (cd tested; mv foo.txt foo-new.txt; mv onedir/twodir/bar.txt onedir/twodir/bar-new.txt) + * git-repo/bin-wrappers/git -C tested add foo-new.txt onedir + * git-repo/bin-wrappers/git -C tested commit -a -m second_commit + * git-repo/bin-wrappers/git -C tested maintenance run + * git-repo/bin-wrappers/git -C tested commit-graph write --changed-paths + * (cd tested; $JGIT debug-read-changed-path-filter .git/objects/info/commit-graph) + * </pre> + * + * @throws Exception + */ + @Test + public void testChangedPathFilterRootAndNested() throws Exception { + RevBlob emptyBlob = tr.blob(new byte[] {}); + RevCommit root = tr.commit(tr.tree(tr.file("foo.txt", emptyBlob), + tr.file("onedir/twodir/bar.txt", emptyBlob))); + RevCommit tip = tr.commit(tr.tree(tr.file("foo-new.txt", emptyBlob), + tr.file("onedir/twodir/bar-new.txt", emptyBlob)), root); + + Set<ObjectId> wants = Collections.singleton(tip); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + writer.write(m, os); + + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder( + "109,-33,2,60,20,79,-11,116,", + "119,69,63,-8,0,")); + } + + /** + * Expected value generated using the following: + * + * <pre> + * git -C git-repo checkout todo get version number when it is merged + * (cd git-repo; make) + * git-repo/bin-wrappers/git init tested + * (cd tested; mkdir -p onedir/twodir; touch onedir/twodir/a.txt; touch onedir/twodir/b.txt) + * git-repo/bin-wrappers/git -C tested add onedir + * git-repo/bin-wrappers/git -C tested commit -m first_commit + * (cd tested; mv onedir/twodir/a.txt onedir/twodir/c.txt; mv onedir/twodir/b.txt onedir/twodir/d.txt) + * git-repo/bin-wrappers/git -C tested add onedir + * git-repo/bin-wrappers/git -C tested commit -a -m second_commit + * git-repo/bin-wrappers/git -C tested maintenance run + * git-repo/bin-wrappers/git -C tested commit-graph write --changed-paths + * (cd tested; $JGIT debug-read-changed-path-filter .git/objects/info/commit-graph) + * </pre> + * + * @throws Exception + */ + @Test + public void testChangedPathFilterOverlappingNested() throws Exception { + RevBlob emptyBlob = tr.blob(new byte[] {}); + RevCommit root = tr + .commit(tr.tree(tr.file("onedir/twodir/a.txt", emptyBlob), + tr.file("onedir/twodir/b.txt", emptyBlob))); + RevCommit tip = tr + .commit(tr.tree(tr.file("onedir/twodir/c.txt", emptyBlob), + tr.file("onedir/twodir/d.txt", emptyBlob)), root); + + Set<ObjectId> wants = Collections.singleton(tip); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + writer.write(m, os); + + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder("61,30,23,-24,1,", + "-58,-51,-46,60,29,-121,113,90,")); + } + + /** + * Expected value generated using the following: + * + * <pre> + * git -C git-repo checkout todo get version number when it is merged + * (cd git-repo; make) + * git-repo/bin-wrappers/git init tested + * (cd tested; touch 你好) + * git-repo/bin-wrappers/git -C tested add 你好 + * git-repo/bin-wrappers/git -C tested commit -m first_commit + * git-repo/bin-wrappers/git -C tested maintenance run + * git-repo/bin-wrappers/git -C tested commit-graph write --changed-paths + * (cd tested; $JGIT debug-read-changed-path-filter .git/objects/info/commit-graph) + * </pre> + * + * @throws Exception + */ + @Test + public void testChangedPathFilterHighBit() throws Exception { + RevBlob emptyBlob = tr.blob(new byte[] {}); + // tr.file encodes using UTF-8 + RevCommit root = tr.commit(tr.tree(tr.file("你好", emptyBlob))); + + Set<ObjectId> wants = Collections.singleton(root); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + writer.write(m, os); + + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder("16,16,")); + } + + @Test + public void testChangedPathFilterEmptyChange() throws Exception { + RevCommit root = commit(); + + Set<ObjectId> wants = Collections.singleton(root); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + writer.write(m, os); + + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder("0,")); + } + + @Test + public void testChangedPathFilterManyChanges() throws Exception { + RevBlob emptyBlob = tr.blob(new byte[] {}); + DirCacheEntry[] entries = new DirCacheEntry[513]; + for (int i = 0; i < entries.length; i++) { + entries[i] = tr.file(i + ".txt", emptyBlob); + } + + RevCommit root = tr.commit(tr.tree(entries)); + + Set<ObjectId> wants = Collections.singleton(root); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + writer.write(m, os); + + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder("-1,")); + } + + @Test + public void testReuseBloomFilters() throws Exception { + RevBlob emptyBlob = tr.blob(new byte[] {}); + RevCommit root = tr.commit(tr.tree(tr.file("foo.txt", emptyBlob), + tr.file("onedir/twodir/bar.txt", emptyBlob))); + tr.branch("master").update(root); + + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, true); + db.getConfig().setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true); + db.getConfig().setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_CHANGED_PATHS, true); + GC gc = new GC(db); + gc.gc().get(); + + RevCommit tip = tr.commit(tr.tree(tr.file("foo-new.txt", emptyBlob), + tr.file("onedir/twodir/bar-new.txt", emptyBlob)), root); + + Set<ObjectId> wants = Collections.singleton(tip); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + GraphCommits graphCommits = GraphCommits.fromWalk(m, wants, walk); + writer = new CommitGraphWriter(graphCommits, true); + CommitGraphWriter.Stats stats = writer.write(m, os); + + assertEquals(1, stats.getChangedPathFiltersReused()); + assertEquals(1, stats.getChangedPathFiltersComputed()); + + // Expected strings are the same as in + // #testChangedPathFilterRootAndNested + HashSet<String> changedPaths = changedPathStrings(os.toByteArray()); + assertThat(changedPaths, containsInAnyOrder( + "109,-33,2,60,20,79,-11,116,", + "119,69,63,-8,0,")); } RevCommit commit(RevCommit... parents) throws Exception { return tr.commit(parents); } -} + + private static final class MockConfig extends FileBasedConfig { + private MockConfig() { + super(null, null); + } + + @Override + public void load() throws IOException, ConfigInvalidException { + // Do nothing + } + + @Override + public void save() throws IOException { + // Do nothing + } + + @Override + public boolean isOutdated() { + return false; + } + + @Override + public String toString() { + return "MockConfig"; + } + + @Override + public boolean getBoolean(final String section, final String name, + final boolean defaultValue) { + if (section.equals(ConfigConstants.CONFIG_COMMIT_GRAPH_SECTION) + && name.equals( + ConfigConstants.CONFIG_KEY_READ_CHANGED_PATHS)) { + return true; + } + return defaultValue; + } + } +}
\ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index ab998951f3..05360dc052 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -15,10 +15,12 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.reftable.RefCursor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; @@ -978,7 +980,7 @@ public class DfsGarbageCollectorTest { } @Test - public void produceCommitGraphAllRefsIncludedFromDisk() throws Exception { + public void produceCommitGraphOnlyHeadsAndTags() throws Exception { String tag = "refs/tags/tag1"; String head = "refs/heads/head1"; String nonHead = "refs/something/nonHead"; @@ -1000,19 +1002,20 @@ public class DfsGarbageCollectorTest { CommitGraph cg = gcPack.getCommitGraph(reader); assertNotNull(cg); - assertTrue("all commits in commit graph", cg.getCommitCnt() == 3); + assertTrue("Only heads and tags reachable commits in commit graph", + cg.getCommitCnt() == 2); // GC packed assertTrue("tag referenced commit is in graph", cg.findGraphPosition(rootCommitTagged) != -1); assertTrue("head referenced commit is in graph", cg.findGraphPosition(headTip) != -1); - // GC_REST packed - assertTrue("nonHead referenced commit is in graph", - cg.findGraphPosition(nonHeadTip) != -1); + // GC_REST not in commit graph + assertEquals("nonHead referenced commit is NOT in graph", + -1, cg.findGraphPosition(nonHeadTip)); } @Test - public void produceCommitGraphAllRefsIncludedFromCache() throws Exception { + public void produceCommitGraphOnlyHeadsAndTagsIncludedFromCache() throws Exception { String tag = "refs/tags/tag1"; String head = "refs/heads/head1"; String nonHead = "refs/something/nonHead"; @@ -1042,15 +1045,16 @@ public class DfsGarbageCollectorTest { assertTrue("commit graph read time is recorded", reader.stats.readCommitGraphMicros > 0); - assertTrue("all commits in commit graph", cachedCG.getCommitCnt() == 3); + assertTrue("Only heads and tags reachable commits in commit graph", + cachedCG.getCommitCnt() == 2); // GC packed assertTrue("tag referenced commit is in graph", cachedCG.findGraphPosition(rootCommitTagged) != -1); assertTrue("head referenced commit is in graph", cachedCG.findGraphPosition(headTip) != -1); - // GC_REST packed - assertTrue("nonHead referenced commit is in graph", - cachedCG.findGraphPosition(nonHeadTip) != -1); + // GC_REST not in commit graph + assertEquals("nonHead referenced commit is not in graph", + -1, cachedCG.findGraphPosition(nonHeadTip)); } @Test @@ -1100,6 +1104,86 @@ public class DfsGarbageCollectorTest { } } + @Test + public void produceCommitGraphAndBloomFilter() throws Exception { + String head = "refs/heads/head1"; + + git.branch(head).commit().message("0").noParents().create(); + + gcWithCommitGraphAndBloomFilter(); + + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + DfsPackDescription desc = pack.getPackDescription(); + CommitGraphWriter.Stats stats = desc.getCommitGraphStats(); + assertNotNull(stats); + assertEquals(1, stats.getChangedPathFiltersComputed()); + } + + @Test + public void objectSizeIdx_reachableBlob_bigEnough_indexed() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + RevBlob headsBlob = git.blob("twelve bytes"); + git.branch(master).commit() + .message("commit on head") + .add("file.txt", headsBlob) + .parent(root) + .create(); + + gcWithObjectSizeIndex(10); + + DfsReader reader = odb.newReader(); + DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); + assertTrue(gcPack.hasObjectSizeIndex(reader)); + assertEquals(12, gcPack.getIndexedObjectSize(reader, headsBlob)); + } + + @Test + public void objectSizeIdx_reachableBlob_tooSmall_notIndexed() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + RevBlob tooSmallBlob = git.blob("small"); + git.branch(master).commit() + .message("commit on head") + .add("small.txt", tooSmallBlob) + .parent(root) + .create(); + + gcWithObjectSizeIndex(10); + + DfsReader reader = odb.newReader(); + DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); + assertTrue(gcPack.hasObjectSizeIndex(reader)); + assertEquals(-1, gcPack.getIndexedObjectSize(reader, tooSmallBlob)); + } + + @Test + public void objectSizeIndex_unreachableGarbage_noIdx() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + git.branch(master).commit() + .message("commit on head") + .add("file.txt", git.blob("a blob")) + .parent(root) + .create(); + git.update(master, root); // blob is unreachable + gcWithObjectSizeIndex(0); + + DfsReader reader = odb.newReader(); + DfsPackFile gcRestPack = findFirstBySource(odb.getPacks(), UNREACHABLE_GARBAGE); + assertFalse(gcRestPack.hasObjectSizeIndex(reader)); + } + + private static DfsPackFile findFirstBySource(DfsPackFile[] packs, PackSource source) { + return Arrays.stream(packs) + .filter(p -> p.getPackDescription().getPackSource() == source) + .findFirst().get(); + } + private TestRepository<InMemoryRepository>.CommitBuilder commit() { return git.commit(); } @@ -1110,6 +1194,19 @@ public class DfsGarbageCollectorTest { run(gc); } + private void gcWithCommitGraphAndBloomFilter() throws IOException { + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.setWriteCommitGraph(true); + gc.setWriteBloomFilter(true); + run(gc); + } + + private void gcWithObjectSizeIndex(int threshold) throws IOException { + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.getPackConfig().setMinBytesForObjSizeIndex(threshold); + run(gc); + } + private void gcNoTtl() throws IOException { DfsGarbageCollector gc = new DfsGarbageCollector(repo); gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java index adf577b0f7..b84a0b00ae 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -29,11 +31,16 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRng; import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.TagBuilder; +import org.eclipse.jgit.lib.TreeFormatter; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; import org.junit.Before; @@ -240,6 +247,71 @@ public class DfsInserterTest { } } + @Test + public void testObjectSizePopulated() throws IOException { + // Blob + byte[] contents = Constants.encode("foo"); + + // Commit + PersonIdent person = new PersonIdent("Committer a", "jgit@eclipse.org"); + CommitBuilder c = new CommitBuilder(); + c.setAuthor(person); + c.setCommitter(person); + c.setTreeId(ObjectId + .fromString("45c4c6767a3945815371a7016532751dd558be40")); + c.setMessage("commit message"); + + // Tree + TreeFormatter treeBuilder = new TreeFormatter(2); + treeBuilder.append("filea", FileMode.REGULAR_FILE, ObjectId + .fromString("45c4c6767a3945815371a7016532751dd558be40")); + treeBuilder.append("fileb", FileMode.GITLINK, ObjectId + .fromString("1c458e25ca624bb8d4735bec1379a4a29ba786d0")); + + // Tag + TagBuilder tagBuilder = new TagBuilder(); + tagBuilder.setObjectId( + ObjectId.fromString("c97fe131649e80de55bd153e9a8d8629f7ca6932"), + Constants.OBJ_COMMIT); + tagBuilder.setTag("short name"); + + try (DfsInserter ins = (DfsInserter) db.newObjectInserter()) { + ObjectId aBlob = ins.insert(Constants.OBJ_BLOB, contents); + assertEquals(contents.length, + ins.objectMap.get(aBlob).getFullSize()); + + ObjectId aCommit = ins.insert(c); + assertEquals(174, ins.objectMap.get(aCommit).getFullSize()); + + ObjectId tree = ins.insert(treeBuilder); + assertEquals(66, ins.objectMap.get(tree).getFullSize()); + + ObjectId tag = ins.insert(tagBuilder); + assertEquals(76, ins.objectMap.get(tag).getFullSize()); + } + } + + @Test + public void testObjectSizeIndexOnInsert() throws IOException { + db.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0); + + byte[] contents = Constants.encode("foo"); + ObjectId fooId; + try (ObjectInserter ins = db.newObjectInserter()) { + fooId = ins.insert(Constants.OBJ_BLOB, contents); + ins.flush(); + } + + DfsReader reader = db.getObjectDatabase().newReader(); + assertEquals(1, db.getObjectDatabase().listPacks().size()); + DfsPackFile insertPack = db.getObjectDatabase().getPacks()[0]; + assertEquals(PackSource.INSERT, + insertPack.getPackDescription().getPackSource()); + assertTrue(insertPack.hasObjectSizeIndex(reader)); + assertEquals(contents.length, insertPack.getIndexedObjectSize(reader, fooId)); + } + private static String readString(ObjectLoader loader) throws IOException { return RawParseUtils.decode(readStream(loader)); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java index ea5787309b..77e5b7cb14 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java @@ -10,12 +10,20 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.zip.Deflater; +import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; +import org.eclipse.jgit.internal.storage.dfs.DfsReader.PackLoadListener; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.PackWriter; @@ -23,6 +31,7 @@ import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRng; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; import org.junit.Before; import org.junit.Test; @@ -100,7 +109,120 @@ public class DfsPackFileTest { assertPackSize(); } - private void setupPack(int bs, int ps) throws IOException { + @Test + public void testLoadObjectSizeIndex() throws IOException { + bypassCache = false; + clearCache = true; + setObjectSizeIndexMinBytes(0); + ObjectId blobId = setupPack(512, 800); + + DfsReader reader = db.getObjectDatabase().newReader(); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + assertTrue(pack.hasObjectSizeIndex(reader)); + assertEquals(800, pack.getIndexedObjectSize(reader, blobId)); + } + + @Test + public void testLoadObjectSizeIndex_noIndex() throws IOException { + bypassCache = false; + clearCache = true; + setObjectSizeIndexMinBytes(-1); + setupPack(512, 800); + + DfsReader reader = db.getObjectDatabase().newReader(); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + assertFalse(pack.hasObjectSizeIndex(reader)); + } + + private static class TestPackLoadListener implements PackLoadListener { + final Map<PackExt, Integer> indexLoadCount = new HashMap<>(); + + int blockLoadCount; + + @SuppressWarnings("boxing") + @Override + public void onIndexLoad(String packName, PackSource src, PackExt ext, + long size, Object loadedIdx) { + indexLoadCount.merge(ext, 1, Integer::sum); + } + + @Override + public void onBlockLoad(String packName, PackSource src, PackExt ext, long position, + DfsBlockData dfsBlockData) { + blockLoadCount += 1; + } + } + + @Test + public void testIndexLoadCallback_indexNotInCache() throws IOException { + bypassCache = false; + clearCache = true; + setObjectSizeIndexMinBytes(-1); + setupPack(512, 800); + + TestPackLoadListener tal = new TestPackLoadListener(); + DfsReader reader = db.getObjectDatabase().newReader(); + reader.addPackLoadListener(tal); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + pack.getPackIndex(reader); + + assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue()); + } + + @Test + public void testIndexLoadCallback_indexInCache() throws IOException { + bypassCache = false; + clearCache = false; + setObjectSizeIndexMinBytes(-1); + setupPack(512, 800); + + TestPackLoadListener tal = new TestPackLoadListener(); + DfsReader reader = db.getObjectDatabase().newReader(); + reader.addPackLoadListener(tal); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + pack.getPackIndex(reader); + pack.getPackIndex(reader); + pack.getPackIndex(reader); + + assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue()); + } + + @Test + public void testIndexLoadCallback_multipleReads() throws IOException { + bypassCache = false; + clearCache = true; + setObjectSizeIndexMinBytes(-1); + setupPack(512, 800); + + TestPackLoadListener tal = new TestPackLoadListener(); + DfsReader reader = db.getObjectDatabase().newReader(); + reader.addPackLoadListener(tal); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + pack.getPackIndex(reader); + pack.getPackIndex(reader); + pack.getPackIndex(reader); + + assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue()); + } + + + @Test + public void testBlockLoadCallback_loadInCache() throws IOException { + bypassCache = false; + clearCache = true; + setObjectSizeIndexMinBytes(-1); + setupPack(512, 800); + + TestPackLoadListener tal = new TestPackLoadListener(); + DfsReader reader = db.getObjectDatabase().newReader(); + reader.addPackLoadListener(tal); + DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; + ObjectId anObject = pack.getPackIndex(reader).getObjectId(0); + pack.get(reader, anObject).getBytes(); + assertEquals(2, tal.blockLoadCount); + } + + private ObjectId setupPack(int bs, int ps) throws IOException { DfsBlockCacheConfig cfg = new DfsBlockCacheConfig().setBlockSize(bs) .setBlockLimit(bs * 100).setStreamRatio(bypassCache ? 0F : 1F); DfsBlockCache.reconfigure(cfg); @@ -108,13 +230,14 @@ public class DfsPackFileTest { byte[] data = new TestRng(JGitTestUtil.getName()).nextBytes(ps); DfsInserter ins = (DfsInserter) db.newObjectInserter(); ins.setCompressionLevel(Deflater.NO_COMPRESSION); - ins.insert(Constants.OBJ_BLOB, data); + ObjectId blobId = ins.insert(Constants.OBJ_BLOB, data); ins.flush(); if (clearCache) { DfsBlockCache.reconfigure(cfg); db.getObjectDatabase().clearCache(); } + return blobId; } private void assertPackSize() throws IOException { @@ -129,4 +252,9 @@ public class DfsPackFileTest { assertEquals(packSize - (12 + 20), os.size()); } } + + private void setObjectSizeIndexMinBytes(int threshold) { + db.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, threshold); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java new file mode 100644 index 0000000000..845d5fcca1 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2023, Google LLC and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.transport.InMemoryPack; +import org.eclipse.jgit.transport.PackParser; +import org.junit.Before; +import org.junit.Test; + +public class DfsPackParserTest { + private InMemoryRepository repo; + + + @Before + public void setUp() throws Exception { + DfsRepositoryDescription desc = new DfsRepositoryDescription("test"); + repo = new InMemoryRepository(desc); + repo.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0); + } + + @Test + public void parse_writeObjSizeIdx() throws IOException { + InMemoryPack pack = new InMemoryPack(); + + // Sha1 of the blob "a" + ObjectId blobA = ObjectId + .fromString("2e65efe2a145dda7ee51d1741299f848e5bf752e"); + + pack.header(2); + pack.write((Constants.OBJ_BLOB) << 4 | 1); + pack.deflate(new byte[] { 'a' }); + + pack.write((Constants.OBJ_REF_DELTA) << 4 | 4); + pack.copyRaw(blobA); + pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' }); + pack.digest(); + + try (ObjectInserter ins = repo.newObjectInserter()) { + PackParser parser = ins.newPackParser(pack.toInputStream()); + parser.parse(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE); + ins.flush(); + } + + DfsReader reader = repo.getObjectDatabase().newReader(); + PackList packList = repo.getObjectDatabase().getPackList(); + assertEquals(1, packList.packs.length); + assertEquals(1, packList.packs[0].getIndexedObjectSize(reader, blobA)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java new file mode 100644 index 0000000000..eb8ceecd81 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2023, Google LLC. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; +import org.eclipse.jgit.internal.storage.dfs.DfsReader.PackLoadListener; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.TestRng; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.junit.Before; +import org.junit.Test; + +public class DfsReaderTest { + InMemoryRepository db; + + @Before + public void setUp() { + db = new InMemoryRepository(new DfsRepositoryDescription("test")); + } + + @Test + public void isNotLargerThan_objAboveThreshold() + throws IOException { + setObjectSizeIndexMinBytes(100); + ObjectId obj = insertBlobWithSize(200); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertFalse("limit < threshold < obj", + ctx.isNotLargerThan(obj, OBJ_BLOB, 50)); + assertEquals(1, ctx.stats.isNotLargerThanCallCount); + assertEquals(1, ctx.stats.objectSizeIndexHit); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + + assertFalse("limit = threshold < obj", + ctx.isNotLargerThan(obj, OBJ_BLOB, 100)); + assertEquals(2, ctx.stats.isNotLargerThanCallCount); + assertEquals(2, ctx.stats.objectSizeIndexHit); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + + assertFalse("threshold < limit < obj", + ctx.isNotLargerThan(obj, OBJ_BLOB, 150)); + assertEquals(3, ctx.stats.isNotLargerThanCallCount); + assertEquals(3, ctx.stats.objectSizeIndexHit); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + + assertTrue("threshold < limit = obj", + ctx.isNotLargerThan(obj, OBJ_BLOB, 200)); + assertEquals(4, ctx.stats.isNotLargerThanCallCount); + assertEquals(4, ctx.stats.objectSizeIndexHit); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + + assertTrue("threshold < obj < limit", + ctx.isNotLargerThan(obj, OBJ_BLOB, 250)); + assertEquals(5, ctx.stats.isNotLargerThanCallCount); + assertEquals(5, ctx.stats.objectSizeIndexHit); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + } + } + + + @Test + public void isNotLargerThan_objBelowThreshold() + throws IOException { + setObjectSizeIndexMinBytes(100); + insertBlobWithSize(1000); // index not empty + ObjectId obj = insertBlobWithSize(50); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertFalse("limit < obj < threshold", + ctx.isNotLargerThan(obj, OBJ_BLOB, 10)); + assertEquals(1, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexHit); + assertEquals(1, ctx.stats.objectSizeIndexMiss); + + assertTrue("limit = obj < threshold", + ctx.isNotLargerThan(obj, OBJ_BLOB, 50)); + assertEquals(2, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexHit); + assertEquals(2, ctx.stats.objectSizeIndexMiss); + + assertTrue("obj < limit < threshold", + ctx.isNotLargerThan(obj, OBJ_BLOB, 80)); + assertEquals(3, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexHit); + assertEquals(3, ctx.stats.objectSizeIndexMiss); + + assertTrue("obj < limit = threshold", + ctx.isNotLargerThan(obj, OBJ_BLOB, 100)); + assertEquals(4, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexHit); + assertEquals(4, ctx.stats.objectSizeIndexMiss); + + assertTrue("obj < threshold < limit", + ctx.isNotLargerThan(obj, OBJ_BLOB, 120)); + assertEquals(5, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexHit); + assertEquals(5, ctx.stats.objectSizeIndexMiss); + } + } + + @Test + public void isNotLargerThan_emptyIdx() throws IOException { + setObjectSizeIndexMinBytes(100); + ObjectId obj = insertBlobWithSize(10); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertFalse(ctx.isNotLargerThan(obj, OBJ_BLOB, 0)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 10)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 40)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 50)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 100)); + + assertEquals(5, ctx.stats.isNotLargerThanCallCount); + assertEquals(5, ctx.stats.objectSizeIndexMiss); + assertEquals(0, ctx.stats.objectSizeIndexHit); + } + } + + @Test + public void isNotLargerThan_noObjectSizeIndex() throws IOException { + setObjectSizeIndexMinBytes(-1); + ObjectId obj = insertBlobWithSize(10); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + assertFalse(ctx.isNotLargerThan(obj, OBJ_BLOB, 0)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 10)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 40)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 50)); + assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 100)); + + assertEquals(5, ctx.stats.isNotLargerThanCallCount); + assertEquals(0, ctx.stats.objectSizeIndexMiss); + assertEquals(0, ctx.stats.objectSizeIndexHit); + } + } + + @Test + public void packLoadListener_noInvocations() throws IOException { + insertBlobWithSize(100); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + CounterPackLoadListener listener = new CounterPackLoadListener(); + ctx.addPackLoadListener(listener); + assertEquals(null, listener.callsPerExt.get(PackExt.INDEX)); + } + } + + @Test + public void packLoadListener_has_openIdx() throws IOException { + ObjectId obj = insertBlobWithSize(100); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + CounterPackLoadListener listener = new CounterPackLoadListener(); + ctx.addPackLoadListener(listener); + boolean has = ctx.has(obj); + assertTrue(has); + assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.INDEX)); + } + } + + @Test + public void packLoadListener_notLargerThan_openMultipleIndices() throws IOException { + setObjectSizeIndexMinBytes(100); + ObjectId obj = insertBlobWithSize(200); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + CounterPackLoadListener listener = new CounterPackLoadListener(); + ctx.addPackLoadListener(listener); + boolean notLargerThan = ctx.isNotLargerThan(obj, OBJ_BLOB, 1000); + assertTrue(notLargerThan); + assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.INDEX)); + assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.OBJECT_SIZE_INDEX)); + } + } + + @Test + public void packLoadListener_has_openMultipleIndices() throws IOException { + setObjectSizeIndexMinBytes(100); + insertBlobWithSize(200); + insertBlobWithSize(230); + insertBlobWithSize(100); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + CounterPackLoadListener listener = new CounterPackLoadListener(); + ctx.addPackLoadListener(listener); + ObjectId oid = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f129"); + boolean has = ctx.has(oid); + assertFalse(has); + // Open 3 indices trying to find the pack + assertEquals(Integer.valueOf(3), listener.callsPerExt.get(PackExt.INDEX)); + } + } + + + @Test + public void packLoadListener_has_repeatedCalls_openMultipleIndices() throws IOException { + // Two objects NOT in the repo + ObjectId oid = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f129"); + ObjectId oid2 = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f130"); + + setObjectSizeIndexMinBytes(100); + insertBlobWithSize(200); + insertBlobWithSize(230); + insertBlobWithSize(100); + CounterPackLoadListener listener = new CounterPackLoadListener(); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + ctx.addPackLoadListener(listener); + boolean has = ctx.has(oid); + ctx.has(oid); + ctx.has(oid2); + assertFalse(has); + // The 3 indices were loaded only once each + assertEquals(Integer.valueOf(3), listener.callsPerExt.get(PackExt.INDEX)); + } + } + + private static class CounterPackLoadListener implements PackLoadListener { + final Map<PackExt, Integer> callsPerExt = new HashMap<>(); + + @SuppressWarnings("boxing") + @Override + public void onIndexLoad(String packName, PackSource src, PackExt ext, long size, + Object loadedIdx) { + callsPerExt.merge(ext, 1, Integer::sum); + } + + @Override + public void onBlockLoad(String packName, PackSource src, PackExt ext, + long size, DfsBlockData dfsBlockData) { + // empty + } + } + + private ObjectId insertBlobWithSize(int size) + throws IOException { + TestRng testRng = new TestRng(JGitTestUtil.getName()); + ObjectId oid; + try (ObjectInserter ins = db.newObjectInserter()) { + oid = ins.insert(OBJ_BLOB, + testRng.nextBytes(size)); + ins.flush(); + } + return oid; + } + + private void setObjectSizeIndexMinBytes(int threshold) { + db.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, threshold); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileRepositoryBuilderAfterOpenConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileRepositoryBuilderAfterOpenConfigTest.java index 100bd32ad8..ed5a6990ac 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileRepositoryBuilderAfterOpenConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileRepositoryBuilderAfterOpenConfigTest.java @@ -18,7 +18,6 @@ import org.eclipse.jgit.util.SystemReader; import org.junit.Before; public class FileRepositoryBuilderAfterOpenConfigTest extends FileRepositoryBuilderTest { - /** {@inheritDoc} */ @Before @Override public void setUp() throws Exception { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReverseIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReverseIndexTest.java new file mode 100644 index 0000000000..cbb0943426 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReverseIndexTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2023, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Collections; + +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.util.IO; +import org.junit.Test; + +public class GcReverseIndexTest extends GcTestCase { + + @Test + public void testWriteDefault() throws Exception { + PackConfig config = new PackConfig(repo); + gc.setPackConfig(config); + + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/main"); + bb.update(tip); + + gc.gc().get(); + assertRidxDoesNotExist(repo); + } + + @Test + public void testWriteDisabled() throws Exception { + PackConfig config = new PackConfig(repo); + config.setWriteReverseIndex(false); + gc.setPackConfig(config); + + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/main"); + bb.update(tip); + + gc.gc().get(); + assertRidxDoesNotExist(repo); + } + + @Test + public void testWriteEmptyRepo() throws Exception { + PackConfig config = new PackConfig(repo); + config.setWriteReverseIndex(true); + gc.setPackConfig(config); + + gc.gc().get(); + assertRidxDoesNotExist(repo); + } + + @Test + public void testWriteShallowRepo() throws Exception { + PackConfig config = new PackConfig(repo); + config.setWriteReverseIndex(true); + gc.setPackConfig(config); + + RevCommit tip = commitChain(2); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/main"); + bb.update(tip); + repo.getObjectDatabase().setShallowCommits(Collections.singleton(tip)); + + gc.gc().get(); + assertValidRidxExists(repo); + } + + @Test + public void testWriteEnabled() throws Exception { + PackConfig config = new PackConfig(repo); + config.setWriteReverseIndex(true); + gc.setPackConfig(config); + + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/main"); + bb.update(tip); + + gc.gc().get(); + assertValidRidxExists(repo); + } + + private static void assertValidRidxExists(FileRepository repo) + throws Exception { + PackFile packFile = repo.getObjectDatabase().getPacks().iterator() + .next().getPackFile(); + File file = packFile.create(REVERSE_INDEX); + assertTrue(file.exists()); + try (InputStream os = new FileInputStream(file)) { + byte[] magic = new byte[4]; + IO.readFully(os, magic, 0, 4); + assertArrayEquals(new byte[] { 'R', 'I', 'D', 'X' }, magic); + } + } + + private static void assertRidxDoesNotExist(FileRepository repo) { + File packDir = repo.getObjectDatabase().getPackDirectory(); + String[] reverseIndexFilenames = packDir.list( + (dir, name) -> name.endsWith(REVERSE_INDEX.getExtension())); + assertEquals(0, reverseIndexFilenames.length); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexComputedTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexComputedTest.java new file mode 100644 index 0000000000..ea5aaf5dd4 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexComputedTest.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com> + * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.junit.Before; +import org.junit.Test; + +public class PackReverseIndexComputedTest extends RepositoryTestCase { + + private PackIndex idx; + + private PackReverseIndex reverseIdx; + + /** + * Set up tested class instance, test constructor by the way. + */ + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + // index with both small (< 2^31) and big offsets + idx = PackIndex.open(JGitTestUtil.getTestResourceFile("pack-huge.idx")); + reverseIdx = PackReverseIndexFactory.computeFromIndex(idx); + } + + /** + * Test findObject() for all index entries. + */ + @Test + public void testFindObject() { + for (MutableEntry me : idx) + assertEquals(me.toObjectId(), reverseIdx.findObject(me.getOffset())); + } + + /** + * Test findObject() with illegal argument. + */ + @Test + public void testFindObjectWrongOffset() { + assertNull(reverseIdx.findObject(0)); + } + + /** + * Test findNextOffset() for all index entries. + * + * @throws CorruptObjectException + */ + @Test + public void testFindNextOffset() throws CorruptObjectException { + long offset = findFirstOffset(); + assertTrue(offset > 0); + for (int i = 0; i < idx.getObjectCount(); i++) { + long newOffset = reverseIdx.findNextOffset(offset, Long.MAX_VALUE); + assertTrue(newOffset > offset); + if (i == idx.getObjectCount() - 1) + assertEquals(newOffset, Long.MAX_VALUE); + else + assertEquals(newOffset, idx.findOffset(reverseIdx + .findObject(newOffset))); + offset = newOffset; + } + } + + /** + * Test findNextOffset() with wrong illegal argument as offset. + */ + @Test + public void testFindNextOffsetWrongOffset() { + try { + reverseIdx.findNextOffset(0, Long.MAX_VALUE); + fail("findNextOffset() should throw exception"); + } catch (CorruptObjectException x) { + // expected + } + } + + @Test + public void testVerifyChecksum() throws PackMismatchException { + // ComputedReverseIndex doesn't have a file containing a checksum. + reverseIdx.verifyPackChecksum(null); + } + + private long findFirstOffset() { + long min = Long.MAX_VALUE; + for (MutableEntry me : idx) + min = Math.min(min, me.getOffset()); + return min; + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexTest.java index 292e3e758a..f8fb4c15e7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexTest.java @@ -1,6 +1,5 @@ /* - * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com> - * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> and others + * Copyright (C) 2022, Google LLC and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -8,95 +7,94 @@ * * SPDX-License-Identifier: BSD-3-Clause */ - package org.eclipse.jgit.internal.storage.file; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; + +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.RepositoryTestCase; -import org.junit.Before; import org.junit.Test; -public class PackReverseIndexTest extends RepositoryTestCase { +public class PackReverseIndexTest { - private PackIndex idx; - - private PackReverseIndex reverseIdx; + @Test + public void open_fallbackToComputed() throws IOException { + String noRevFilePrefix = "pack-3280af9c07ee18a87705ef50b0cc4cd20266cf12."; + PackReverseIndex computed = PackReverseIndexFactory.openOrCompute( + getResourceFileFor(noRevFilePrefix, PackExt.REVERSE_INDEX), 7, + () -> PackIndex.open( + getResourceFileFor(noRevFilePrefix, PackExt.INDEX))); - /** - * Set up tested class instance, test constructor by the way. - */ - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - // index with both small (< 2^31) and big offsets - idx = PackIndex.open(JGitTestUtil.getTestResourceFile( - "pack-huge.idx")); - reverseIdx = new PackReverseIndex(idx); + assertTrue(computed instanceof PackReverseIndexComputed); } - /** - * Test findObject() for all index entries. - */ @Test - public void testFindObject() { - for (MutableEntry me : idx) - assertEquals(me.toObjectId(), reverseIdx.findObject(me.getOffset())); + public void open_readGoodFile() throws IOException { + String hasRevFilePrefix = "pack-cbdeda40019ae0e6e789088ea0f51f164f489d14."; + PackReverseIndex version1 = PackReverseIndexFactory.openOrCompute( + getResourceFileFor(hasRevFilePrefix, PackExt.REVERSE_INDEX), 6, + () -> PackIndex.open( + getResourceFileFor(hasRevFilePrefix, PackExt.INDEX))); + + assertTrue(version1 instanceof PackReverseIndexV1); } - /** - * Test findObject() with illegal argument. - */ @Test - public void testFindObjectWrongOffset() { - assertNull(reverseIdx.findObject(0)); + public void open_readCorruptFile() { + String hasRevFilePrefix = "pack-cbdeda40019ae0e6e789088ea0f51f164f489d14."; + + assertThrows(IOException.class, + () -> PackReverseIndexFactory.openOrCompute( + getResourceFileFor(hasRevFilePrefix + "corrupt.", + PackExt.REVERSE_INDEX), + 6, () -> PackIndex.open(getResourceFileFor( + hasRevFilePrefix, PackExt.INDEX)))); } - /** - * Test findNextOffset() for all index entries. - * - * @throws CorruptObjectException - */ @Test - public void testFindNextOffset() throws CorruptObjectException { - long offset = findFirstOffset(); - assertTrue(offset > 0); - for (int i = 0; i < idx.getObjectCount(); i++) { - long newOffset = reverseIdx.findNextOffset(offset, Long.MAX_VALUE); - assertTrue(newOffset > offset); - if (i == idx.getObjectCount() - 1) - assertEquals(newOffset, Long.MAX_VALUE); - else - assertEquals(newOffset, idx.findOffset(reverseIdx - .findObject(newOffset))); - offset = newOffset; - } + public void read_badMagic() { + byte[] badMagic = new byte[] { 'R', 'B', 'A', 'D', // magic + 0x00, 0x00, 0x00, 0x01, // file version + 0x00, 0x00, 0x00, 0x01, // oid version + // pack checksum + 'P', 'A', 'C', 'K', 'C', 'H', 'E', 'C', 'K', 'S', 'U', 'M', '3', + '4', '5', '6', '7', '8', '9', '0', + // checksum + 0x66, 0x01, (byte) 0xbc, (byte) 0xe8, 0x51, 0x4b, 0x2f, + (byte) 0xa1, (byte) 0xa9, (byte) 0xcd, (byte) 0xbe, (byte) 0xd6, + 0x4f, (byte) 0xa8, 0x7d, (byte) 0xab, 0x50, (byte) 0xa3, + (byte) 0xf7, (byte) 0xcc, }; + ByteArrayInputStream in = new ByteArrayInputStream(badMagic); + + assertThrows(IOException.class, + () -> PackReverseIndexFactory.readFromFile(in, 0, () -> null)); } - /** - * Test findNextOffset() with wrong illegal argument as offset. - */ @Test - public void testFindNextOffsetWrongOffset() { - try { - reverseIdx.findNextOffset(0, Long.MAX_VALUE); - fail("findNextOffset() should throw exception"); - } catch (CorruptObjectException x) { - // expected - } + public void read_unsupportedVersion2() { + byte[] version2 = new byte[] { 'R', 'I', 'D', 'X', // magic + 0x00, 0x00, 0x00, 0x02, // file version + 0x00, 0x00, 0x00, 0x01, // oid version + // pack checksum + 'P', 'A', 'C', 'K', 'C', 'H', 'E', 'C', 'K', 'S', 'U', 'M', '3', + '4', '5', '6', '7', '8', '9', '0', + // checksum + 0x70, 0x17, 0x10, 0x51, (byte) 0xfe, (byte) 0xab, (byte) 0x9b, + 0x68, (byte) 0xed, 0x3a, 0x3f, 0x27, 0x1d, (byte) 0xce, + (byte) 0xff, 0x38, 0x09, (byte) 0x9b, 0x29, 0x58, }; + ByteArrayInputStream in = new ByteArrayInputStream(version2); + + assertThrows(IOException.class, + () -> PackReverseIndexFactory.readFromFile(in, 0, () -> null)); } - private long findFirstOffset() { - long min = Long.MAX_VALUE; - for (MutableEntry me : idx) - min = Math.min(min, me.getOffset()); - return min; + private File getResourceFileFor(String packFilePrefix, PackExt ext) { + return JGitTestUtil + .getTestResourceFile(packFilePrefix + ext.getExtension()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1Test.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1Test.java new file mode 100644 index 0000000000..38b28b501b --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1Test.java @@ -0,0 +1,289 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; +import static org.eclipse.jgit.lib.Constants.OBJ_TREE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.junit.Before; +import org.junit.Test; + +public class PackReverseIndexV1Test { + private static final byte[] FAKE_PACK_CHECKSUM = new byte[] { 'P', 'A', 'C', + 'K', 'C', 'H', 'E', 'C', 'K', 'S', 'U', 'M', '3', '4', '5', '6', + '7', '8', '9', '0', }; + + private static final byte[] NO_OBJECTS = new byte[] { 'R', 'I', 'D', 'X', // magic + 0x00, 0x00, 0x00, 0x01, // file version + 0x00, 0x00, 0x00, 0x01, // oid version + // pack checksum to copy into at byte 12 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // checksum + (byte) 0xd1, 0x1d, 0x17, (byte) 0xd5, (byte) 0xa1, 0x5c, + (byte) 0x8f, 0x45, 0x7e, 0x06, (byte) 0x91, (byte) 0xf2, 0x7e, 0x20, + 0x35, 0x2c, (byte) 0xdc, 0x4c, 0x46, (byte) 0xe4, }; + + private static final byte[] SMALL_PACK_CHECKSUM = new byte[] { (byte) 0xbb, + 0x1d, 0x25, 0x3d, (byte) 0xd3, (byte) 0xf0, 0x08, 0x75, (byte) 0xc8, + 0x04, (byte) 0xd0, 0x6f, 0x73, (byte) 0xe9, 0x00, (byte) 0x82, + (byte) 0xdb, 0x09, (byte) 0xc8, 0x13, }; + + private static final byte[] SMALL_CONTENTS = new byte[] { 'R', 'I', 'D', + 'X', // magic + 0x00, 0x00, 0x00, 0x01, // file version + 0x00, 0x00, 0x00, 0x01, // oid version + 0x00, 0x00, 0x00, 0x04, // offset 12: "68" -> index @ 4 + 0x00, 0x00, 0x00, 0x02, // offset 165: "5c" -> index @ 2 + 0x00, 0x00, 0x00, 0x03, // offset 257: "62" -> index @ 3 + 0x00, 0x00, 0x00, 0x01, // offset 450: "58" -> index @ 1 + 0x00, 0x00, 0x00, 0x05, // offset 556: "c5" -> index @ 5 + 0x00, 0x00, 0x00, 0x00, // offset 614: "2d" -> index @ 0 + // pack checksum to copy into at byte 36 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // checksum + (byte) 0xf0, 0x6d, 0x03, (byte) 0xd7, 0x6f, (byte) 0x9f, + (byte) 0xc1, 0x36, 0x26, (byte) 0xbc, (byte) 0xcb, 0x75, 0x36, + (byte) 0xa1, 0x26, 0x6a, 0x2b, (byte) 0x84, 0x16, (byte) 0x83, }; + + private PackReverseIndex emptyReverseIndex; + + /** + * Reverse index for the pack-cbdeda40019ae0e6e789088ea0f51f164f489d14.idx + * with contents `SHA-1 type size size-in-packfile offset-in-packfile` as + * shown by `verify-pack`: + * 2d04ee74dba30078c2dcdb713ddb8be4bc084d76 blob 8 17 614 + * 58728c938a9a8b9970cc09236caf94ada4689923 blob 140 106 450 + * 5ce00008cf3fb8f194f52742020bd40d78f3f1b3 commit 81 92 165 1 68cb1f232964f3cd698afc1dafe583937203c587 + * 62299a7ae290d685196e948a2fcb7d8c07f95c7d tree 198 193 257 + * 68cb1f232964f3cd698afc1dafe583937203c587 commit 220 153 12 + * c5ab27309491cf641eb11bb4b7a78641f280b482 tree 46 58 556 1 62299a7ae290d685196e948a2fcb7d8c07f95c7d + */ + private PackReverseIndex smallReverseIndex; + + private final PackedObjectInfo object614 = objectInfo( + "2d04ee74dba30078c2dcdb713ddb8be4bc084d76", OBJ_BLOB, 614); + + private final PackedObjectInfo object450 = objectInfo( + "58728c938a9a8b9970cc09236caf94ada4689923", OBJ_BLOB, 450); + + private final PackedObjectInfo object165 = objectInfo( + "5ce00008cf3fb8f194f52742020bd40d78f3f1b3", OBJ_COMMIT, 165); + + private final PackedObjectInfo object257 = objectInfo( + "62299a7ae290d685196e948a2fcb7d8c07f95c7d", OBJ_TREE, 257); + + private final PackedObjectInfo object12 = objectInfo( + "68cb1f232964f3cd698afc1dafe583937203c587", OBJ_COMMIT, 12); + + private final PackedObjectInfo object556 = objectInfo( + "c5ab27309491cf641eb11bb4b7a78641f280b482", OBJ_TREE, 556); + + // last object's offset + last object's length + private final long smallMaxOffset = 631; + + @Before + public void setUp() throws Exception { + System.arraycopy(SMALL_PACK_CHECKSUM, 0, SMALL_CONTENTS, 36, + SMALL_PACK_CHECKSUM.length); + ByteArrayInputStream smallIn = new ByteArrayInputStream(SMALL_CONTENTS); + smallReverseIndex = PackReverseIndexFactory.readFromFile(smallIn, 6, + () -> PackIndex.open(JGitTestUtil.getTestResourceFile( + "pack-cbdeda40019ae0e6e789088ea0f51f164f489d14.idx"))); + + System.arraycopy(FAKE_PACK_CHECKSUM, 0, NO_OBJECTS, 12, + FAKE_PACK_CHECKSUM.length); + ByteArrayInputStream emptyIn = new ByteArrayInputStream(NO_OBJECTS); + emptyReverseIndex = PackReverseIndexFactory.readFromFile(emptyIn, 0, + () -> null); + } + + @Test + public void read_unsupportedOidSHA256() { + byte[] version2 = new byte[] { 'R', 'I', 'D', 'X', // magic + 0x00, 0x00, 0x00, 0x01, // file version + 0x00, 0x00, 0x00, 0x02, // oid version + // pack checksum to copy into at byte 12 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // checksum + 0x6e, 0x78, 0x75, 0x67, (byte) 0x84, (byte) 0x89, (byte) 0xde, + (byte) 0xe3, (byte) 0x86, 0x6a, 0x3b, (byte) 0x98, 0x51, + (byte) 0xd8, (byte) 0x8c, (byte) 0xec, 0x50, (byte) 0xe7, + (byte) 0xfb, 0x22, }; + System.arraycopy(FAKE_PACK_CHECKSUM, 0, version2, 12, + FAKE_PACK_CHECKSUM.length); + ByteArrayInputStream in = new ByteArrayInputStream(version2); + + assertThrows(IOException.class, + () -> PackReverseIndexFactory.readFromFile(in, 0, () -> null)); + } + + @Test + public void read_objectCountTooLarge() { + ByteArrayInputStream dummyInput = new ByteArrayInputStream(NO_OBJECTS); + long biggerThanInt = ((long) Integer.MAX_VALUE) + 1; + + assertThrows(IllegalArgumentException.class, + () -> PackReverseIndexFactory.readFromFile(dummyInput, + biggerThanInt, + () -> null)); + } + + @Test + public void read_incorrectChecksum() { + byte[] badChecksum = new byte[] { 'R', 'I', 'D', 'X', // magic + 0x00, 0x00, 0x00, 0x01, // file version + 0x00, 0x00, 0x00, 0x01, // oid version + // pack checksum to copy into at byte 12 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + // checksum + (byte) 0xf2, 0x1a, 0x1a, (byte) 0xaa, 0x32, 0x2d, (byte) 0xb9, + (byte) 0xfd, 0x0f, (byte) 0xa5, 0x4c, (byte) 0xea, (byte) 0xcf, + (byte) 0xbb, (byte) 0x99, (byte) 0xde, (byte) 0xd3, 0x4e, + (byte) 0xb1, (byte) 0xee, // would be 0x74 if correct + }; + System.arraycopy(FAKE_PACK_CHECKSUM, 0, badChecksum, 12, + FAKE_PACK_CHECKSUM.length); + ByteArrayInputStream in = new ByteArrayInputStream(badChecksum); + assertThrows(CorruptObjectException.class, + () -> PackReverseIndexFactory.readFromFile(in, 0, () -> null)); + } + + @Test + public void findObject_noObjects() { + assertNull(emptyReverseIndex.findObject(0)); + } + + @Test + public void findObject_multipleObjects() { + assertEquals(object614, smallReverseIndex.findObject(614)); + assertEquals(object450, smallReverseIndex.findObject(450)); + assertEquals(object165, smallReverseIndex.findObject(165)); + assertEquals(object257, smallReverseIndex.findObject(257)); + assertEquals(object12, smallReverseIndex.findObject(12)); + assertEquals(object556, smallReverseIndex.findObject(556)); + } + + @Test + public void findObject_badOffset() { + assertNull(smallReverseIndex.findObject(0)); + } + + @Test + public void findNextOffset_noObjects() { + assertThrows(IOException.class, + () -> emptyReverseIndex.findNextOffset(0, Long.MAX_VALUE)); + } + + @Test + public void findNextOffset_multipleObjects() throws CorruptObjectException { + assertEquals(smallMaxOffset, + smallReverseIndex.findNextOffset(614, smallMaxOffset)); + assertEquals(614, + smallReverseIndex.findNextOffset(556, smallMaxOffset)); + assertEquals(556, + smallReverseIndex.findNextOffset(450, smallMaxOffset)); + assertEquals(450, + smallReverseIndex.findNextOffset(257, smallMaxOffset)); + assertEquals(257, + smallReverseIndex.findNextOffset(165, smallMaxOffset)); + assertEquals(165, smallReverseIndex.findNextOffset(12, smallMaxOffset)); + } + + @Test + public void findNextOffset_badOffset() { + assertThrows(IOException.class, + () -> smallReverseIndex.findNextOffset(0, Long.MAX_VALUE)); + } + + @Test + public void findPosition_noObjects() { + assertEquals(-1, emptyReverseIndex.findPosition(0)); + } + + @Test + public void findPosition_multipleObjects() { + assertEquals(0, smallReverseIndex.findPosition(12)); + assertEquals(1, smallReverseIndex.findPosition(165)); + assertEquals(2, smallReverseIndex.findPosition(257)); + assertEquals(3, smallReverseIndex.findPosition(450)); + assertEquals(4, smallReverseIndex.findPosition(556)); + assertEquals(5, smallReverseIndex.findPosition(614)); + } + + @Test + public void findPosition_badOffset() { + assertEquals(-1, smallReverseIndex.findPosition(10)); + } + + @Test + public void findObjectByPosition_noObjects() { + assertThrows(AssertionError.class, + () -> emptyReverseIndex.findObjectByPosition(0)); + } + + @Test + public void findObjectByPosition_multipleObjects() { + assertEquals(object12, smallReverseIndex.findObjectByPosition(0)); + assertEquals(object165, smallReverseIndex.findObjectByPosition(1)); + assertEquals(object257, smallReverseIndex.findObjectByPosition(2)); + assertEquals(object450, smallReverseIndex.findObjectByPosition(3)); + assertEquals(object556, smallReverseIndex.findObjectByPosition(4)); + assertEquals(object614, smallReverseIndex.findObjectByPosition(5)); + } + + @Test + public void findObjectByPosition_badOffset() { + assertThrows(AssertionError.class, + () -> smallReverseIndex.findObjectByPosition(10)); + } + + @Test + public void verifyChecksum_match() throws IOException { + smallReverseIndex.verifyPackChecksum("smallPackFilePath"); + } + + @Test + public void verifyChecksum_mismatch() throws IOException { + ByteArrayInputStream in = new ByteArrayInputStream(NO_OBJECTS); + PackIndex mockForwardIndex = mock(PackIndex.class); + when(mockForwardIndex.getChecksum()).thenReturn( + new byte[] { 'D', 'I', 'F', 'F', 'P', 'A', 'C', 'K', 'C', 'H', + 'E', 'C', 'K', 'S', 'U', 'M', '7', '8', '9', '0', }); + PackReverseIndex reverseIndex = PackReverseIndexFactory.readFromFile(in, + 0, + () -> mockForwardIndex); + + assertThrows(PackMismatchException.class, + () -> reverseIndex.verifyPackChecksum("packFilePath")); + } + + private static PackedObjectInfo objectInfo(String objectId, int type, + long offset) { + PackedObjectInfo objectInfo = new PackedObjectInfo( + ObjectId.fromString(objectId)); + objectInfo.setType(type); + objectInfo.setOffset(offset); + return objectInfo; + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1WriteReadTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1WriteReadTest.java new file mode 100644 index 0000000000..372a4c7cba --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackReverseIndexV1WriteReadTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2022, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; +import static org.eclipse.jgit.lib.Constants.OBJ_TREE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.junit.Test; + +public class PackReverseIndexV1WriteReadTest { + + private static byte[] PACK_CHECKSUM = new byte[] { 'P', 'A', 'C', 'K', 'C', + 'H', 'E', 'C', 'K', 'S', 'U', 'M', '3', '4', '5', '6', '7', '8', + '9', '0', }; + + @Test + public void writeThenRead_noObjects() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackReverseIndexWriter writer = PackReverseIndexWriter.createWriter(out, + 1); + List<PackedObjectInfo> objectsSortedByName = new ArrayList<>(); + + // write + writer.write(objectsSortedByName, PACK_CHECKSUM); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + + // read + PackReverseIndex noObjectsReverseIndex = PackReverseIndexFactory + .readFromFile(in, 0, () -> null); + + // use + assertThrows(AssertionError.class, + () -> noObjectsReverseIndex.findObjectByPosition(0)); + } + + @Test + public void writeThenRead_oneObject() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackReverseIndexWriter writer = PackReverseIndexWriter.createWriter(out, + 1); + PackedObjectInfo a = objectInfo("a", OBJ_COMMIT, 0); + List<PackedObjectInfo> objectsSortedByName = List.of(a); + + // write + writer.write(objectsSortedByName, PACK_CHECKSUM); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackIndex mockForwardIndex = mock(PackIndex.class); + when(mockForwardIndex.getObjectId(0)).thenReturn(a); + + // read + PackReverseIndex oneObjectReverseIndex = PackReverseIndexFactory + .readFromFile(in, 1, () -> mockForwardIndex); + + // use + assertEquals(a, oneObjectReverseIndex.findObjectByPosition(0)); + } + + @Test + public void writeThenRead_multipleObjectsLargeOffsets() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PackReverseIndexWriter writer = PackReverseIndexWriter.createWriter(out, + 1); + PackedObjectInfo a = objectInfo("a", OBJ_BLOB, 200000000); + PackedObjectInfo b = objectInfo("b", OBJ_COMMIT, 0); + PackedObjectInfo c = objectInfo("c", OBJ_COMMIT, 52000000000L); + PackedObjectInfo d = objectInfo("d", OBJ_TREE, 7); + PackedObjectInfo e = objectInfo("e", OBJ_COMMIT, 38000000000L); + List<PackedObjectInfo> objectsSortedByName = List.of(a, b, c, d, e); + + writer.write(objectsSortedByName, PACK_CHECKSUM); + + // write + writer.write(objectsSortedByName, PACK_CHECKSUM); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + PackIndex mockForwardIndex = mock(PackIndex.class); + when(mockForwardIndex.getObjectId(4)).thenReturn(e); + + // read + PackReverseIndex multipleObjectsReverseIndex = PackReverseIndexFactory + .readFromFile(in, 5, () -> mockForwardIndex); + + // use with minimal mocked forward index use + assertEquals(e, multipleObjectsReverseIndex.findObjectByPosition(3)); + } + + private static PackedObjectInfo objectInfo(String objectId, int type, + long offset) { + assert (objectId.length() == 1); + PackedObjectInfo objectInfo = new PackedObjectInfo( + ObjectId.fromString(objectId.repeat(OBJECT_ID_STRING_LENGTH))); + objectInfo.setType(type); + objectInfo.setOffset(offset); + return objectInfo; + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index a3596541fe..e1509456e5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -261,7 +261,7 @@ public class PackTest extends LocalDiskRepositoryTestCase { new PackIndexWriterV1(f).write(list, footer); } - Pack pack = new Pack(packName, null); + Pack pack = new Pack(repo.getConfig(), packName, null); try { pack.get(wc, b); fail("expected LargeObjectException.ExceedsByteArrayLimit"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryAfterOpenConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryAfterOpenConfigTest.java index 42304e2253..3ea4a167cb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryAfterOpenConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryAfterOpenConfigTest.java @@ -17,7 +17,6 @@ import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.util.SystemReader; public class RefDirectoryAfterOpenConfigTest extends RefDirectoryTest { - /** {@inheritDoc} */ @Override public void refDirectorySetup() throws Exception { StoredConfig userConfig = SystemReader.getInstance().getUserConfig(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/SnapshottingRefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/SnapshottingRefDirectoryTest.java index c3dafe4aa2..90a2aa601e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/SnapshottingRefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/SnapshottingRefDirectoryTest.java @@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals; public class SnapshottingRefDirectoryTest extends RefDirectoryTest { private RefDirectory originalRefDirectory; - /** {@inheritDoc} */ @Before @Override public void setUp() throws Exception { |