diff options
Diffstat (limited to 'org.eclipse.jgit.test/tst/org')
16 files changed, 860 insertions, 50 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java index fca27d32aa..0949d040e9 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java @@ -12,12 +12,16 @@ package org.eclipse.jgit.gitrepo; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -221,4 +225,33 @@ public class ManifestParserTest { testNormalize("", ""); testNormalize("a/b", "a/b"); } + + @Test + public void testXXE() throws Exception { + File externalEntity = File.createTempFile("injected", "xml"); + externalEntity.deleteOnExit(); + Files.write(externalEntity.toPath(), + "<evil>injected xml</evil>" + .getBytes(UTF_8), + StandardOpenOption.WRITE); + String baseUrl = "https://git.google.com/"; + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n") + .append("<!DOCTYPE booo [ <!ENTITY foobar SYSTEM \"") + .append(externalEntity.getPath()).append("\"> ]>\n") + .append("<manifest>") + .append("<remote name=\"remote1\" fetch=\".\" />") + .append("<default revision=\"master\" remote=\"remote1\" />") + .append("&foobar;") + .append("<project path=\"foo\" name=\"foo\" groups=\"a,test\" />") + .append("</manifest>"); + + IOException e = assertThrows(IOException.class, + () -> new ManifestParser(null, null, "master", baseUrl, null, + null) + .read(new ByteArrayInputStream( + xmlContent.toString().getBytes(UTF_8)))); + assertTrue(e.getCause().getMessage().contains("DOCTYPE")); + } + } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java index d02bfcd3f6..1119db3712 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/indexdiff/IndexDiffWithSymlinkTest.java @@ -132,7 +132,8 @@ public class IndexDiffWithSymlinkTest extends LocalDiskRepositoryTestCase { Writer writer = new OutputStreamWriter(out, UTF_8)) { writer.write("echo `which git` 1>&2\n"); writer.write("echo `git --version` 1>&2\n"); - writer.write("git init " + name + " && \\\n"); + writer.write("git -c init.defaultBranch=master init " + name + + " && \\\n"); writer.write("cd ./" + name + " && \\\n"); writer.write("git fast-import < ../" + name + ".txt && \\\n"); writer.write("git checkout -f\n"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index 00a3760e21..80bd689084 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -1182,7 +1182,8 @@ public class DfsGarbageCollectorTest { DfsReader reader = odb.newReader(); DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); assertTrue(gcPack.hasObjectSizeIndex(reader)); - assertEquals(12, gcPack.getIndexedObjectSize(reader, headsBlob)); + assertEquals(12, gcPack.getIndexedObjectSize(reader, + gcPack.findIdxPosition(reader, headsBlob))); } @Test @@ -1203,7 +1204,8 @@ public class DfsGarbageCollectorTest { DfsReader reader = odb.newReader(); DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); assertTrue(gcPack.hasObjectSizeIndex(reader)); - assertEquals(-1, gcPack.getIndexedObjectSize(reader, tooSmallBlob)); + assertEquals(-1, gcPack.getIndexedObjectSize(reader, + gcPack.findIdxPosition(reader, tooSmallBlob))); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java index 0b558edf2c..efa98de549 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java @@ -214,7 +214,7 @@ public class DfsInserterTest { } @Test - public void testNoCheckExisting() throws IOException { + public void testNoDuplicates() throws IOException { byte[] contents = Constants.encode("foo"); ObjectId fooId; try (ObjectInserter ins = db.newObjectInserter()) { @@ -224,21 +224,20 @@ public class DfsInserterTest { assertEquals(1, db.getObjectDatabase().listPacks().size()); try (ObjectInserter ins = db.newObjectInserter()) { - ((DfsInserter) ins).checkExisting(false); + ins.insert(Constants.OBJ_BLOB, Constants.encode("bar")); assertEquals(fooId, ins.insert(Constants.OBJ_BLOB, contents)); ins.flush(); } assertEquals(2, db.getObjectDatabase().listPacks().size()); - // Verify that we have a foo in both INSERT packs. + // Newer packs are first. Verify that foo is only in the second pack try (DfsReader reader = new DfsReader(db.getObjectDatabase())) { DfsPackFile packs[] = db.getObjectDatabase().getPacks(); - assertEquals(2, packs.length); DfsPackFile p1 = packs[0]; assertEquals(PackSource.INSERT, p1.getPackDescription().getPackSource()); - assertTrue(p1.hasObject(reader, fooId)); + assertFalse(p1.hasObject(reader, fooId)); DfsPackFile p2 = packs[1]; assertEquals(PackSource.INSERT, @@ -310,7 +309,8 @@ public class DfsInserterTest { assertEquals(PackSource.INSERT, insertPack.getPackDescription().getPackSource()); assertTrue(insertPack.hasObjectSizeIndex(reader)); - assertEquals(contents.length, insertPack.getIndexedObjectSize(reader, fooId)); + assertEquals(contents.length, insertPack.getIndexedObjectSize(reader, + insertPack.findIdxPosition(reader, fooId))); } private static String readString(ObjectLoader loader) throws IOException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java index 9680019f88..f2129fd3c5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java @@ -131,7 +131,8 @@ public class DfsPackFileTest { DfsReader reader = db.getObjectDatabase().newReader(); DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; assertTrue(pack.hasObjectSizeIndex(reader)); - assertEquals(800, pack.getIndexedObjectSize(reader, blobId)); + assertEquals(800, pack.getIndexedObjectSize(reader, + pack.findIdxPosition(reader, blobId))); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java index c1cd231c66..9d26978d66 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java @@ -65,6 +65,7 @@ public class DfsPackParserTest { DfsReader reader = repo.getObjectDatabase().newReader(); PackList packList = repo.getObjectDatabase().getPackList(); assertEquals(1, packList.packs.length); - assertEquals(1, packList.packs[0].getIndexedObjectSize(reader, blobA)); + assertEquals(1, packList.packs[0].getIndexedObjectSize(reader, + packList.packs[0].findIdxPosition(reader, blobA))); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java index 92d7465376..cd73c6ae83 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -702,13 +702,29 @@ public class BasePackWriterTest extends SampleDataRepositoryTestCase { } @Test - public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet() + public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSetTrue() throws Exception { + totalPackFilesScanWhenSearchForReuseTimeoutNotSet(true); + } + + @Test + public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSetFalse() + throws Exception { + totalPackFilesScanWhenSearchForReuseTimeoutNotSet(false); + } + + public void totalPackFilesScanWhenSearchForReuseTimeoutNotSet(boolean doReturn) throws Exception { FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + int numberOfPackFiles = (int) new GC(fileRepository).getStatistics().numberOfPackFiles; + int objectsInMultiplePacks = 2; + int objectsInOnePacks = 1; + int expectedSelectCalls = objectsInMultiplePacks * (doReturn ? numberOfPackFiles : 1) + + objectsInOnePacks; + PackWriter mockedPackWriter = Mockito .spy(new PackWriter(config, fileRepository.newObjectReader())); - doNothing().when(mockedPackWriter).select(any(), any()); + doReturn(doReturn).when(mockedPackWriter).select(any(), any()); try (FileOutputStream packOS = new FileOutputStream( getPackFileToWrite(fileRepository, mockedPackWriter))) { @@ -716,27 +732,37 @@ public class BasePackWriterTest extends SampleDataRepositoryTestCase { NullProgressMonitor.INSTANCE, packOS); } - long numberOfPackFiles = new GC(fileRepository) - .getStatistics().numberOfPackFiles; - int expectedSelectCalls = - // Objects contained in multiple packfiles * number of packfiles - 2 * (int) numberOfPackFiles + - // Objects in single packfile - 1; verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), any()); } @Test - public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck() + public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheckTrue() throws Exception { + totalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck(true); + } + + @Test + public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheckFalse() + throws Exception { + totalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck(false); + } + + public void totalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck( + boolean doReturn) throws Exception { FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + int numberOfPackFiles = (int) new GC(fileRepository).getStatistics().numberOfPackFiles; + int objectsInMultiplePacks = 2; + int objectsInOnePacks = 1; + int expectedSelectCalls = objectsInMultiplePacks * (doReturn ? numberOfPackFiles : 1) + + objectsInOnePacks; + PackConfig packConfig = new PackConfig(); packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); PackWriter mockedPackWriter = Mockito.spy( new PackWriter(packConfig, fileRepository.newObjectReader())); - doNothing().when(mockedPackWriter).select(any(), any()); + doReturn(doReturn).when(mockedPackWriter).select(any(), any()); try (FileOutputStream packOS = new FileOutputStream( getPackFileToWrite(fileRepository, mockedPackWriter))) { @@ -744,28 +770,31 @@ public class BasePackWriterTest extends SampleDataRepositoryTestCase { NullProgressMonitor.INSTANCE, packOS); } - long numberOfPackFiles = new GC(fileRepository) - .getStatistics().numberOfPackFiles; - int expectedSelectCalls = - // Objects contained in multiple packfiles * number of packfiles - 2 * (int) numberOfPackFiles + - // Objects contained in single packfile - 1; verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), any()); } @Test - public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck() + public void partialPackFilesScanWhenDoingSearchForReuseTimeoutCheck() throws Exception { + int objectsInMultiplePacks = 2; + int objectsInOnePacks = 1; + int expectedSelectCalls = objectsInMultiplePacks + objectsInOnePacks; + testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck(true, expectedSelectCalls); + testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck(false, expectedSelectCalls); + } + + public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck( + boolean doReturn, int expectedSelectCalls) throws Exception { FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackConfig packConfig = new PackConfig(); packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); PackWriter mockedPackWriter = Mockito.spy( new PackWriter(packConfig, fileRepository.newObjectReader())); mockedPackWriter.enableSearchForReuseTimeout(); - doNothing().when(mockedPackWriter).select(any(), any()); + doReturn(doReturn).when(mockedPackWriter).select(any(), any()); try (FileOutputStream packOS = new FileOutputStream( getPackFileToWrite(fileRepository, mockedPackWriter))) { @@ -773,7 +802,6 @@ public class BasePackWriterTest extends SampleDataRepositoryTestCase { NullProgressMonitor.INSTANCE, packOS); } - int expectedSelectCalls = 3; // Objects in packfiles verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), any()); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java index 6c7992716c..e8363ce21d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java @@ -81,8 +81,7 @@ public class FileReftableStackTest { } public void testCompaction(int N) throws Exception { - try (FileReftableStack stack = new FileReftableStack( - new File(reftableDir, "refs"), reftableDir, null, + try (FileReftableStack stack = new FileReftableStack(reftableDir, null, () -> new Config())) { writeBranches(stack, "refs/heads/branch%d", 0, N); MergedReftable table = stack.getMergedReftable(); @@ -124,8 +123,7 @@ public class FileReftableStackTest { // Can't delete in-use files on Windows. assumeFalse(SystemReader.getInstance().isWindows()); - try (FileReftableStack stack = new FileReftableStack( - new File(reftableDir, "refs"), reftableDir, null, + try (FileReftableStack stack = new FileReftableStack(reftableDir, null, () -> new Config())) { outer: for (int i = 0; i < 10; i++) { final long next = stack.getMergedReftable().maxUpdateIndex() @@ -152,8 +150,8 @@ public class FileReftableStackTest { } } assertThrows(FileNotFoundException.class, - () -> new FileReftableStack(new File(reftableDir, "refs"), - reftableDir, null, () -> new Config())); + () -> new FileReftableStack(reftableDir, null, + () -> new Config())); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java new file mode 100644 index 0000000000..1a05d88583 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcObjectSizeIndexTest.java @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2025, Google LLC. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class GcObjectSizeIndexTest extends GcTestCase { + + @Test + public void gc_2commits_noSizeLimit_blobsInIndex() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(4, stats.numberOfSizeIndexedObjects); + + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + Pack pack = getOnlyPack(repo); + assertEquals(7, pack.getIndexedObjectSize(blobA1)); + assertEquals(11, pack.getIndexedObjectSize(blobA2)); + assertEquals(1, pack.getIndexedObjectSize(blobB1)); + assertEquals(2, pack.getIndexedObjectSize(blobB2)); + } + + @Test + public void gc_2commits_sizeLimit_biggerBlobsInIndex() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 5); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(2, stats.numberOfSizeIndexedObjects); + + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + Pack pack = getOnlyPack(repo); + assertEquals(7, pack.getIndexedObjectSize(blobA1)); + assertEquals(11, pack.getIndexedObjectSize(blobA2)); + assertEquals(-1, pack.getIndexedObjectSize(blobB1)); + assertEquals(-1, pack.getIndexedObjectSize(blobB2)); + } + + @Test + public void gc_2commits_disableSizeIdx_noIdx() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevBlob blobA1 = tr.blob("7-bytes"); + RevBlob blobA2 = tr.blob("11-bytes xx"); + RevBlob blobB1 = tr.blob("B"); + RevBlob blobB2 = tr.blob("B2"); + bb.commit().add("A", blobA1).add("B", blobB1).create(); + bb.commit().add("A", blobA2).add("B", blobB2).create(); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, -1); + gc.gc().get(); + + + stats = gc.getStatistics(); + assertEquals(1, stats.numberOfPackFiles); + assertEquals(0, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_alreadyPacked_noChanges() + throws Exception { + tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B") + .create(); + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + assertEquals(2, stats.numberOfSizeIndexedObjects); + + // Do the gc again and check that it hasn't changed anything + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + assertTrue(getOnlyPack(repo).hasObjectSizeIndex()); + assertEquals(2, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_twoReachableCommits_oneUnreachable_twoPacks() + throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + RevCommit first = bb.commit().add("A", "A").add("B", "B").create(); + bb.commit().add("A", "A2").add("B", "B2").create(); + tr.update("refs/heads/master", first); + + stats = gc.getStatistics(); + assertEquals(8, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, 0); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(8, stats.numberOfPackedObjects); + assertEquals(2, stats.numberOfPackFiles); + assertEquals(4, stats.numberOfSizeIndexedObjects); + } + + @Test + public void gc_preserved_objSizeIdxIsPreserved() throws Exception { + Collection<Pack> oldPacks = preserveOldPacks(); + assertEquals(1, oldPacks.size()); + PackFile preserved = oldPacks.iterator().next().getPackFile() + .create(PackExt.OBJECT_SIZE_INDEX) + .createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preserved.exists()); + } + + @Test + public void gc_preserved_prune_noPreserves() throws Exception { + preserveOldPacks(); + configureGc(gc, 0).setPrunePreserved(true); + gc.gc().get(); + + assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists()); + } + + private Collection<Pack> preserveOldPacks() throws Exception { + TestRepository<FileRepository>.BranchBuilder bb = tr + .branch("refs/heads/master"); + bb.commit().message("P").add("P", "P").create(); + + // pack loose object into packfile + configureGc(gc, 0); + gc.setExpireAgeMillis(0); + gc.gc().get(); + Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase() + .getPacks(); + PackFile oldPackfile = oldPacks.iterator().next().getPackFile(); + assertTrue(oldPackfile.exists()); + + fsTick(); + bb.commit().message("B").add("B", "Q").create(); + + // repack again but now without a grace period for packfiles. We should + // end up with a new packfile and the old one should be placed in the + // preserved directory + gc.setPackExpireAgeMillis(0); + configureGc(gc, 0).setPreserveOldPacks(true); + gc.gc().get(); + + File preservedPackFile = oldPackfile.createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preservedPackFile.exists()); + return oldPacks; + } + + @Ignore + public void testPruneAndRestoreOldPacks() throws Exception { + String tempRef = "refs/heads/soon-to-be-unreferenced"; + TestRepository<FileRepository>.BranchBuilder bb = tr.branch(tempRef); + bb.commit().add("A", "A").add("B", "B").create(); + + // Verify setup conditions + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + + // Force all referenced objects into packs (to avoid having loose objects) + configureGc(gc, 0); + gc.setExpireAgeMillis(0); + gc.setPackExpireAgeMillis(0); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + + // Delete the temp ref, orphaning its commit + RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setForceUpdate(true); + ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it! + RefUpdate.Result result = update.delete(); + assertEquals(RefUpdate.Result.FORCED, result); + + fsTick(); + + // Repack with only orphaned commit, so packfile will be pruned + configureGc(gc, 0).setPreserveOldPacks(true); + gc.gc().get(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + assertEquals(0, stats.numberOfPackFiles); + + // Restore the temp ref to the deleted commit, should restore old-packs! + update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setNewObjectId(objectId); + update.setExpectedOldObjectId(null); + result = update.update(); + assertEquals(RefUpdate.Result.NEW, result); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + } + + private PackConfig configureGc(GC myGc, int minSize) { + PackConfig pconfig = new PackConfig(repo); + pconfig.setMinBytesForObjSizeIndex(minSize); + myGc.setPackConfig(pconfig); + return pconfig; + } + + private Pack getOnlyPack(FileRepository fileRepo) + throws IOException { + Collection<Pack> packs = fileRepo.getObjectDatabase().getPacks(); + if (packs.size() != 1) { + throw new IOException("More than one pack"); + } + + return packs.iterator().next(); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 85043034aa..cc43d3c2bb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -53,6 +53,7 @@ import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; @@ -77,12 +78,14 @@ import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.util.IO; @@ -489,6 +492,38 @@ public class PackInserterTest extends RepositoryTestCase { } } + @Test + public void createsObjectSizeIndex() throws Exception { + FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig(); + jGitConfig.setInt( + ConfigConstants.CONFIG_PACK_SECTION, + null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 10); + jGitConfig.save(); + byte[] oneBlob = Constants.encode("a blob with some content"); + byte[] anotherBlob = Constants.encode("some more contents"); + byte[] streamMeBlob = Constants.encode("some more content to write"); + + ObjectId oneBlobOid, anotherBlobOid, streamMeBlobOid; + try (PackInserter ins = newInserter()) { + oneBlobOid = ins.insert(OBJ_BLOB, oneBlob); + anotherBlobOid = ins.insert(OBJ_BLOB, anotherBlob); + streamMeBlobOid = ins.insert(OBJ_BLOB, streamMeBlob.length, + new ByteArrayInputStream(streamMeBlob)); + ins.flush(); + } + + List<Pack> listPacks = listPacks(db); + assertEquals(1, listPacks.size()); + Pack thePack = listPacks.get(0); + assertTrue(thePack.hasObjectSizeIndex()); + assertEquals(oneBlob.length, thePack.getIndexedObjectSize(oneBlobOid)); + assertEquals(anotherBlob.length, + thePack.getIndexedObjectSize(anotherBlobOid)); + assertEquals(streamMeBlob.length, + thePack.getIndexedObjectSize(streamMeBlobOid)); + } + private List<Pack> listPacks() throws Exception { List<Pack> fromOpenDb = listPacks(db); List<Pack> reopened; @@ -549,7 +584,8 @@ public class PackInserterTest extends RepositoryTestCase { } private void assertPacksOnly() throws Exception { - new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")) + new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx") + && !f.endsWith(".objsize")) .assertNoBadFiles(db.getObjectDatabase().getDirectory()); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index e1509456e5..016a6afd70 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import java.util.zip.Deflater; import org.eclipse.jgit.errors.LargeObjectException; @@ -39,6 +41,7 @@ import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRng; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -47,6 +50,7 @@ import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.transport.PackParser; import org.eclipse.jgit.transport.PackedObjectInfo; @@ -295,6 +299,29 @@ public class PackTest extends LocalDiskRepositoryTestCase { } } + @Test + public void testObjectSize() throws Exception { + byte[] data = getRng().nextBytes(300); + RevBlob aBlob = tr.blob(data); + RevCommit aCommit = tr.branch("master").commit().add("A", aBlob).create(); + repo.getConfig().setInt(CONFIG_PACK_SECTION, null, ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0); + tr.packAndPrune(); + + List<Pack> packs = repo.getObjectDatabase().getPacks().stream().collect(Collectors.toList()); + assertEquals(1, packs.size()); + // Indexed object + assertEquals(300, packs.get(0).getIndexedObjectSize(aBlob)); + assertEquals(300, packs.get(0).getObjectSize(wc, aBlob)); + // Non indexed object + assertEquals(-1, packs.get(0).getIndexedObjectSize(aCommit)); + assertEquals(168, packs.get(0).getObjectSize(wc, aCommit)); + // Object not in pack + assertEquals(-1, packs.get(0).getObjectSize(wc, + ObjectId.fromString("1111111111111111111111111111111111111111"))); + assertEquals(-1, packs.get(0).getIndexedObjectSize( + ObjectId.fromString("1111111111111111111111111111111111111111"))); + } + private static byte[] clone(int first, byte[] base) { byte[] r = new byte[base.length]; System.arraycopy(base, 1, r, 1, r.length - 1); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java index da015bf930..ab452854b2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexTest.java @@ -13,16 +13,21 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.junit.FakeIndexFactory; import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.junit.Test; @@ -67,8 +72,7 @@ public class MultiPackIndexTest { assertNotNull(midx); assertArrayEquals(packNames, midx.getPackNames()); - MultiPackIndex.PackOffset oo = midx - .find(ObjectId.fromString(knownOid)); + MultiPackIndex.PackOffset oo = midx.find(ObjectId.fromString(knownOid)); assertEquals(knowOffset, oo.getOffset()); assertEquals(knownPackId, oo.getPackId()); @@ -133,7 +137,7 @@ public class MultiPackIndexTest { "0000000000000000000000000000000000000005", 12))); PackIndex idxTwo = FakeIndexFactory.indexOf(List.of( new FakeIndexFactory.IndexObject( - "0000000000000000000000000000000000000002", (1L<< 35)), + "0000000000000000000000000000000000000002", (1L << 35)), new FakeIndexFactory.IndexObject( "0000000000000000000000000000000000000003", 13))); Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); @@ -144,9 +148,11 @@ public class MultiPackIndexTest { MultiPackIndex midx = MultiPackIndexLoader .read(new ByteArrayInputStream(out.toByteArray())); assertEquals(2, midx.getPackNames().length); - assertInIndex(midx, 0, "0000000000000000000000000000000000000001", (1L << 34)); + assertInIndex(midx, 0, "0000000000000000000000000000000000000001", + (1L << 34)); assertInIndex(midx, 0, "0000000000000000000000000000000000000005", 12); - assertInIndex(midx, 1, "0000000000000000000000000000000000000002", (1L << 35)); + assertInIndex(midx, 1, "0000000000000000000000000000000000000002", + (1L << 35)); } @Test @@ -155,7 +161,8 @@ public class MultiPackIndexTest { // Most significant bit to 1 is still valid offset PackIndex idxOne = FakeIndexFactory.indexOf(List.of( new FakeIndexFactory.IndexObject( - "0000000000000000000000000000000000000001", 0xff00_0000), + "0000000000000000000000000000000000000001", + 0xff00_0000), new FakeIndexFactory.IndexObject( "0000000000000000000000000000000000000005", 12))); PackIndex idxTwo = FakeIndexFactory.indexOf(List.of( @@ -171,9 +178,161 @@ public class MultiPackIndexTest { MultiPackIndex midx = MultiPackIndexLoader .read(new ByteArrayInputStream(out.toByteArray())); assertEquals(2, midx.getPackNames().length); - assertInIndex(midx, 0, "0000000000000000000000000000000000000001", 0xff00_0000L); + assertInIndex(midx, 0, "0000000000000000000000000000000000000001", + 0xff00_0000L); assertInIndex(midx, 0, "0000000000000000000000000000000000000005", 12); } + + @Test + public void jgit_resolve() throws IOException { + AbbreviatedObjectId abbrev = AbbreviatedObjectId + .fromString("32fe829a1c"); + + PackIndex idxOne = indexWith( + // Noise + "0000000000000000000000000000000000000001", + "3000000000000000000000000000000000000005", + // One before abbrev + "32fe829a1b000000000000000000000000000001", + // matches + "32fe829a1c000000000000000000000000000001", + "32fe829a1c000000000000000000000000000100", + // One after abbrev + "32fe829a1d000000000000000000000000000000"); + PackIndex idxTwo = indexWith( + // Noise + "8888880000000000000000000000000000000002", + "bbbbbb0000000000000000000000000000000003", + // Match + "32fe829a1c000000000000000000000000000010"); + + Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + + + Set<ObjectId> results = new HashSet<>(); + midx.resolve(results, abbrev, 100); + + assertEquals(3, results.size()); + assertTrue(results.contains(ObjectId + .fromString("32fe829a1c000000000000000000000000000001"))); + assertTrue(results.contains(ObjectId + .fromString("32fe829a1c000000000000000000000000000010"))); + assertTrue(results.contains(ObjectId + .fromString("32fe829a1c000000000000000000000000000100"))); + + } + + @Test + public void jgit_resolve_matchLimit() throws IOException { + AbbreviatedObjectId abbrev = AbbreviatedObjectId + .fromString("32fe829a1c"); + + PackIndex idxOne = indexWith( + // Noise + "0000000000000000000000000000000000000001", + "3000000000000000000000000000000000000005", + // One before abbrev + "32fe829a1b000000000000000000000000000001", + // matches + "32fe829a1c000000000000000000000000000001", + "32fe829a1c000000000000000000000000000100", + // One after abbrev + "32fe829a1d000000000000000000000000000000"); + PackIndex idxTwo = indexWith( + // Noise + "8888880000000000000000000000000000000002", + "bbbbbb0000000000000000000000000000000003", + // Match + "32fe829a1c000000000000000000000000000010"); + + Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + + + Set<ObjectId> results = new HashSet<>(); + midx.resolve(results, abbrev, 2); + + assertEquals(2, results.size()); + assertTrue(results.contains(ObjectId + .fromString("32fe829a1c000000000000000000000000000001"))); + assertTrue(results.contains(ObjectId + .fromString("32fe829a1c000000000000000000000000000010"))); + } + + @Test + public void jgit_resolve_noMatches() throws IOException { + AbbreviatedObjectId abbrev = AbbreviatedObjectId + .fromString("4400000000"); + + PackIndex idxOne = indexWith( + "0000000000000000000000000000000000000001", + "3000000000000000000000000000000000000005", + "32fe829a1b000000000000000000000000000001", + "32fe829a1c000000000000000000000000000001", + "32fe829a1c000000000000000000000000000100", + "32fe829a1d000000000000000000000000000000"); + PackIndex idxTwo = indexWith( + // Noise + "8888880000000000000000000000000000000002", + "bbbbbb0000000000000000000000000000000003", + "32fe829a1c000000000000000000000000000010"); + + Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + + + Set<ObjectId> results = new HashSet<>(); + midx.resolve(results, abbrev, 200); + + assertEquals(0, results.size()); + } + + @Test + public void jgit_resolve_empty() throws IOException { + AbbreviatedObjectId abbrev = AbbreviatedObjectId + .fromString("4400000000"); + + PackIndex idxOne = FakeIndexFactory.indexOf(List.of()); + PackIndex idxTwo = FakeIndexFactory.indexOf(List.of()); + + Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + MultiPackIndex midx = MultiPackIndexLoader + .read(new ByteArrayInputStream(out.toByteArray())); + + + Set<ObjectId> results = new HashSet<>(); + midx.resolve(results, abbrev, 200); + + assertEquals(0, results.size()); + } + + private static PackIndex indexWith(String... oids) { + List<FakeIndexFactory.IndexObject> idxObjs = new ArrayList<>( + oids.length); + int offset = 12; + for (String oid : oids) { + idxObjs.add(new FakeIndexFactory.IndexObject(oid, offset)); + offset += 10; + } + return FakeIndexFactory.indexOf(idxObjs); + } + private static void assertInIndex(MultiPackIndex midx, int expectedPackId, String oid, long expectedOffset) { MultiPackIndex.PackOffset packOffset = midx diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java index 82f3eb1e08..8b57a2dcb4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -139,6 +139,24 @@ public class MultiPackIndexWriterTest { assertEquals(5, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); } + @Test + public void jgit_emptyMidx() throws IOException { + PackIndex idxOne = FakeIndexFactory.indexOf(List.of()); + PackIndex idxTwo = FakeIndexFactory.indexOf(List.of()); + Map<String, PackIndex> packs = Map.of("p1", idxOne, "p2", idxTwo); + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, packs); + List<Integer> chunkIds = readChunkIds(out); + assertEquals(1134, out.size()); + assertEquals(5, chunkIds.size()); + assertEquals(0, chunkIds.indexOf(MIDX_CHUNKID_OIDFANOUT)); + assertEquals(1, chunkIds.indexOf(MIDX_CHUNKID_OIDLOOKUP)); + assertEquals(2, chunkIds.indexOf(MIDX_CHUNKID_OBJECTOFFSETS)); + assertEquals(3, chunkIds.indexOf(MIDX_CHUNKID_REVINDEX)); + assertEquals(4, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); + } + private List<Integer> readChunkIds(ByteArrayOutputStream out) { List<Integer> chunkIds = new ArrayList<>(); byte[] raw = out.toByteArray(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java index 1d8bde0f44..8218cbc20d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java index 917288a899..0b3ccacfc1 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java new file mode 100644 index 0000000000..b17c577087 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ObjectDirectoryPackParserTest.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2021, Google LLC. and others + * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com> + * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.util.zip.Deflater; + +import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; +import org.eclipse.jgit.internal.storage.file.Pack; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Pack parsing is covered in {@link PackParserTest}. + * + * Here we test ObjectDirectoryPackParser specific parts. e.g. that is creates + * the object-size index. + */ +public class ObjectDirectoryPackParserTest extends RepositoryTestCase { + + @Before + public void setup() throws IOException { + FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig(); + jGitConfig.setInt(ConfigConstants.CONFIG_PACK_SECTION, null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 7); + jGitConfig.save(); + } + + /** + * Test indexing one of the test packs in the egit repo. It has deltas. + * + * @throws IOException + */ + @Test + public void testGitPack() throws IOException { + File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); + try (InputStream is = new FileInputStream(packFile)) { + ObjectDirectoryPackParser p = index(is); + p.parse(NullProgressMonitor.INSTANCE); + + Pack pack = p.getPack(); + assertTrue(pack.hasObjectSizeIndex()); + + // Only blobs in the pack + ObjectId blob1 = ObjectId + .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); + ObjectId blob2 = ObjectId + .fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"); + assertEquals(18787, pack.getIndexedObjectSize(blob1)); + assertEquals(18009, pack.getIndexedObjectSize(blob2)); + + // Indexed sizes match object db sizes + assertEquals(db.getObjectDatabase().open(blob1).getSize(), + pack.getIndexedObjectSize(blob1)); + assertEquals(db.getObjectDatabase().open(blob2).getSize(), + pack.getIndexedObjectSize(blob2)); + + } + } + + /** + * This is just another pack. It so happens that we have two convenient pack to + * test with in the repository. + * + * @throws IOException + */ + @Test + public void testAnotherGitPack() throws IOException { + File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack"); + try (InputStream is = new FileInputStream(packFile)) { + ObjectDirectoryPackParser p = index(is); + p.parse(NullProgressMonitor.INSTANCE); + Pack pack = p.getPack(); + + // Blob smaller than threshold: + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); + + // Blob bigger than threshold + assertEquals(10, pack.getIndexedObjectSize(ObjectId + .fromString("8230f48330e0055d9e0bc5a2a77718f6dd9324b8"))); + + // A commit (not indexed) + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("d0114ab8ac326bab30e3a657a0397578c5a1af88"))); + + // Object not in pack + assertEquals(-1, pack.getIndexedObjectSize(ObjectId + .fromString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))); + } + } + + @Test + public void testTinyThinPack() throws Exception { + // less than 16 bytes, so its length fits in a single byte later + String base = "abcdefghijklmn"; + RevBlob a; + try (TestRepository d = new TestRepository<Repository>(db)) { + a = d.blob(base); + } + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + + packHeader(pack, 1); + + pack.write(Constants.OBJ_REF_DELTA << 4 | 4); + a.copyRawTo(pack); + deflate(pack, new byte[] { (byte) base.length(), // size of the base + (byte) (base.length() + 1), // size after reconstruction + 0x1, 'b' }); // append one byte + + digest(pack); + + ObjectDirectoryPackParser p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.parse(NullProgressMonitor.INSTANCE); + + Pack writtenPack = p.getPack(); + // base + assertEquals(base.length(), writtenPack.getIndexedObjectSize(a)); + // undeltified blob + assertEquals(base.length() + 1, + writtenPack.getIndexedObjectSize(ObjectId.fromString( + "f177875498138143c9657cc52b049ad4d20d5223"))); + } + + @Test + public void testPackWithDuplicateBlob() throws Exception { + final byte[] data = Constants.encode("0123456789abcdefg"); + RevBlob blob; + try (TestRepository<Repository> d = new TestRepository<>(db)) { + blob = d.blob(data); + assertTrue(db.getObjectDatabase().has(blob)); + } + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + packHeader(pack, 1); + pack.write(Constants.OBJ_BLOB << 4 | 0x80 | 1); + pack.write(1); + deflate(pack, data); + digest(pack); + + ObjectDirectoryPackParser p = index( + new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(false); + p.parse(NullProgressMonitor.INSTANCE); + + assertEquals(data.length, p.getPack().getIndexedObjectSize(blob)); + } + + private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt) + throws IOException { + final byte[] hdr = new byte[8]; + NB.encodeInt32(hdr, 0, 2); + NB.encodeInt32(hdr, 4, cnt); + + tinyPack.write(Constants.PACK_SIGNATURE); + tinyPack.write(hdr, 0, 8); + } + + private static void deflate(TemporaryBuffer.Heap tinyPack, + final byte[] content) + throws IOException { + final Deflater deflater = new Deflater(); + final byte[] buf = new byte[128]; + deflater.setInput(content, 0, content.length); + deflater.finish(); + do { + final int n = deflater.deflate(buf, 0, buf.length); + if (n > 0) + tinyPack.write(buf, 0, n); + } while (!deflater.finished()); + } + + private static void digest(TemporaryBuffer.Heap buf) throws IOException { + MessageDigest md = Constants.newMessageDigest(); + md.update(buf.toByteArray()); + buf.write(md.digest()); + } + + private ObjectInserter inserter; + + @After + public void release() { + if (inserter != null) { + inserter.close(); + } + } + + private ObjectDirectoryPackParser index(InputStream in) throws IOException { + if (inserter == null) + inserter = db.newObjectInserter(); + return (ObjectDirectoryPackParser) inserter.newPackParser(in); + } +} |