From efb154fc24fbf416ae3513942fa720128358b31b Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 10 Feb 2021 22:42:41 -0700 Subject: Rename PackFile to Pack Pack better represents the purpose of the object and paves the way to add a PackFile object that extends File. Change-Id: I39b4f697902d395e9b6df5e8ce53078ce72fcea3 Signed-off-by: Nasser Grainawi --- .../internal/storage/file/GcBasicPackingTest.java | 4 +- .../internal/storage/file/GcConcurrentTest.java | 8 +- .../internal/storage/file/GcKeepFilesTest.java | 6 +- .../storage/file/PackFileSnapshotTest.java | 38 +-- .../jgit/internal/storage/file/PackFileTest.java | 374 --------------------- .../internal/storage/file/PackInserterTest.java | 28 +- .../jgit/internal/storage/file/PackTest.java | 374 +++++++++++++++++++++ .../jgit/internal/storage/file/PackWriterTest.java | 4 +- .../storage/file/T0004_PackReaderTest.java | 4 +- .../org/eclipse/jgit/transport/PackParserTest.java | 50 +-- 10 files changed, 445 insertions(+), 445 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java (limited to 'org.eclipse.jgit.test') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index d007dd4511..42e4238451 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -157,7 +157,7 @@ public class GcBasicPackingTest extends GcTestCase { .create(); tr.update("refs/tags/t1", second); - Collection oldPacks = tr.getRepository().getObjectDatabase() + Collection oldPacks = tr.getRepository().getObjectDatabase() .getPacks(); assertEquals(0, oldPacks.size()); stats = gc.getStatistics(); @@ -171,7 +171,7 @@ public class GcBasicPackingTest extends GcTestCase { stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); - List packs = new ArrayList<>( + List packs = new ArrayList<>( repo.getObjectDatabase().getPacks()); assertEquals(11, packs.get(0).getObjectCount()); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java index bb8455f515..5cac1e3429 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java @@ -156,8 +156,8 @@ public class GcConcurrentTest extends GcTestCase { } } - PackFile getSinglePack(FileRepository r) { - Collection packs = r.getObjectDatabase().getPacks(); + Pack getSinglePack(FileRepository r) { + Collection packs = r.getObjectDatabase().getPacks(); assertEquals(1, packs.size()); return packs.iterator().next(); } @@ -206,11 +206,11 @@ public class GcConcurrentTest extends GcTestCase { SampleDataRepositoryTestCase.copyCGitTestPacks(repo); ExecutorService executor = Executors.newSingleThreadExecutor(); final CountDownLatch latch = new CountDownLatch(1); - Future> result = executor.submit(() -> { + Future> result = executor.submit(() -> { long start = System.currentTimeMillis(); System.out.println("starting gc"); latch.countDown(); - Collection r = gc.gc(); + Collection r = gc.gc(); System.out.println( "gc took " + (System.currentTimeMillis() - start) + " ms"); return r; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java index e1559584fd..8472983d5b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java @@ -36,9 +36,9 @@ public class GcKeepFilesTest extends GcTestCase { assertEquals(4, stats.numberOfPackedObjects); assertEquals(1, stats.numberOfPackFiles); - Iterator packIt = repo.getObjectDatabase().getPacks() + Iterator packIt = repo.getObjectDatabase().getPacks() .iterator(); - PackFile singlePack = packIt.next(); + Pack singlePack = packIt.next(); assertFalse(packIt.hasNext()); String packFileName = singlePack.getPackFile().getPath(); String keepFileName = packFileName.substring(0, @@ -58,7 +58,7 @@ public class GcKeepFilesTest extends GcTestCase { assertEquals(2, stats.numberOfPackFiles); // check that no object is packed twice - Iterator packs = repo.getObjectDatabase().getPacks() + Iterator packs = repo.getObjectDatabase().getPacks() .iterator(); PackIndex ind1 = packs.next().getIndex(); assertEquals(4, ind1.getObjectCount()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java index 1f1e094385..7c32ce7cea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java @@ -72,14 +72,14 @@ public class PackFileSnapshotTest extends RepositoryTestCase { c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); c.save(); - Collection packs = gc(Deflater.NO_COMPRESSION); + Collection packs = gc(Deflater.NO_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p1 = packs.iterator().next(); + Pack p1 = packs.iterator().next(); PackFileSnapshot snapshot = p1.getFileSnapshot(); packs = gc(Deflater.BEST_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p2 = packs.iterator().next(); + Pack p2 = packs.iterator().next(); File pf = p2.getPackFile(); // changing compression level with aggressive gc may change size, @@ -153,11 +153,11 @@ public class PackFileSnapshotTest extends RepositoryTestCase { createTestRepo(testDataSeed, testDataLength); // repack to create initial packfile - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); FS fs = db.getFS(); Instant m1 = fs.lastModifiedInstant(packFilePath); @@ -207,16 +207,16 @@ public class PackFileSnapshotTest extends RepositoryTestCase { createTestRepo(testDataSeed, testDataLength); // Repack to create initial packfile. Make a copy of it - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); Path fn = packFilePath.getFileName(); assertNotNull(fn); String packFileName = fn.toString(); Path packFileBasePath = packFilePath .resolveSibling(packFileName.replaceAll(".pack", "")); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); copyPack(packFileBasePath, "", ".copy1"); // Repack to create second packfile. Make a copy of it @@ -280,10 +280,10 @@ public class PackFileSnapshotTest extends RepositoryTestCase { Paths.get(base + ".pack" + dstSuffix)); } - private PackFile repackAndCheck(int compressionLevel, String oldName, + private Pack repackAndCheck(int compressionLevel, String oldName, Long oldLength, AnyObjectId oldChkSum) throws IOException, ParseException { - PackFile p = getSinglePack(gc(compressionLevel)); + Pack p = getSinglePack(gc(compressionLevel)); File pf = p.getPackFile(); // The following two assumptions should not cause the test to fail. If // on a certain platform we get packfiles (containing the same git @@ -298,14 +298,14 @@ public class PackFileSnapshotTest extends RepositoryTestCase { return p; } - private PackFile getSinglePack(Collection packs) { - Iterator pIt = packs.iterator(); - PackFile p = pIt.next(); + private Pack getSinglePack(Collection packs) { + Iterator pIt = packs.iterator(); + Pack p = pIt.next(); assertFalse(pIt.hasNext()); return p; } - private Collection gc(int compressionLevel) + private Collection gc(int compressionLevel) throws IOException, ParseException { GC gc = new GC(db); PackConfig pc = new PackConfig(db.getConfig()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java deleted file mode 100644 index 97a86e249e..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (C) 2010, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.security.MessageDigest; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.zip.Deflater; - -import org.eclipse.jgit.errors.LargeObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.pack.DeltaEncoder; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.junit.TestRng; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectLoader; -import org.eclipse.jgit.lib.ObjectStream; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevBlob; -import org.eclipse.jgit.storage.file.WindowCacheConfig; -import org.eclipse.jgit.transport.PackParser; -import org.eclipse.jgit.transport.PackedObjectInfo; -import org.eclipse.jgit.util.IO; -import org.eclipse.jgit.util.NB; -import org.eclipse.jgit.util.TemporaryBuffer; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class PackFileTest extends LocalDiskRepositoryTestCase { - private int streamThreshold = 16 * 1024; - - private TestRng rng; - - private FileRepository repo; - - private TestRepository tr; - - private WindowCursor wc; - - private TestRng getRng() { - if (rng == null) - rng = new TestRng(JGitTestUtil.getName()); - return rng; - } - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - - WindowCacheConfig cfg = new WindowCacheConfig(); - cfg.setStreamFileThreshold(streamThreshold); - cfg.install(); - - repo = createBareRepository(); - tr = new TestRepository<>(repo); - wc = (WindowCursor) repo.newObjectReader(); - } - - @Override - @After - public void tearDown() throws Exception { - if (wc != null) - wc.close(); - new WindowCacheConfig().install(); - super.tearDown(); - } - - @Test - public void testWhole_SmallObject() throws Exception { - final int type = Constants.OBJ_BLOB; - byte[] data = getRng().nextBytes(300); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - assertNotNull("created loader", ol); - assertEquals(type, ol.getType()); - assertEquals(data.length, ol.getSize()); - assertFalse("is not large", ol.isLarge()); - assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(type, in.getType()); - assertEquals(data.length, in.getSize()); - byte[] data2 = new byte[data.length]; - IO.readFully(in, data2, 0, data.length); - assertTrue("same content", Arrays.equals(data2, data)); - assertEquals("stream at EOF", -1, in.read()); - } - } - - @Test - public void testWhole_LargeObject() throws Exception { - final int type = Constants.OBJ_BLOB; - byte[] data = getRng().nextBytes(streamThreshold + 5); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - assertNotNull("created loader", ol); - assertEquals(type, ol.getType()); - assertEquals(data.length, ol.getSize()); - assertTrue("is large", ol.isLarge()); - try { - ol.getCachedBytes(); - fail("Should have thrown LargeObjectException"); - } catch (LargeObjectException tooBig) { - assertEquals(MessageFormat.format( - JGitText.get().largeObjectException, id.name()), tooBig - .getMessage()); - } - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(type, in.getType()); - assertEquals(data.length, in.getSize()); - byte[] data2 = new byte[data.length]; - IO.readFully(in, data2, 0, data.length); - assertTrue("same content", Arrays.equals(data2, data)); - assertEquals("stream at EOF", -1, in.read()); - } - } - - @Test - public void testDelta_SmallObjectChain() throws Exception { - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - byte[] data0 = new byte[512]; - Arrays.fill(data0, (byte) 0xf3); - ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); - - TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); - packHeader(pack, 4); - objectHeader(pack, Constants.OBJ_BLOB, data0.length); - deflate(pack, data0); - - byte[] data1 = clone(0x01, data0); - byte[] delta1 = delta(data0, data1); - ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); - id0.copyRawTo(pack); - deflate(pack, delta1); - - byte[] data2 = clone(0x02, data1); - byte[] delta2 = delta(data1, data2); - ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); - id1.copyRawTo(pack); - deflate(pack, delta2); - - byte[] data3 = clone(0x03, data2); - byte[] delta3 = delta(data2, data3); - ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); - id2.copyRawTo(pack); - deflate(pack, delta3); - - digest(pack); - PackParser ip = index(pack.toByteArray()); - ip.setAllowThin(true); - ip.parse(NullProgressMonitor.INSTANCE); - - assertTrue("has blob", wc.has(id3)); - - ObjectLoader ol = wc.open(id3); - assertNotNull("created loader", ol); - assertEquals(Constants.OBJ_BLOB, ol.getType()); - assertEquals(data3.length, ol.getSize()); - assertFalse("is large", ol.isLarge()); - assertNotNull(ol.getCachedBytes()); - assertArrayEquals(data3, ol.getCachedBytes()); - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(Constants.OBJ_BLOB, in.getType()); - assertEquals(data3.length, in.getSize()); - byte[] act = new byte[data3.length]; - IO.readFully(in, act, 0, data3.length); - assertTrue("same content", Arrays.equals(act, data3)); - assertEquals("stream at EOF", -1, in.read()); - } - } - } - - @Test - public void testDelta_FailsOver2GiB() throws Exception { - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - byte[] base = new byte[] { 'a' }; - ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base); - ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' }); - - PackedObjectInfo a = new PackedObjectInfo(idA); - PackedObjectInfo b = new PackedObjectInfo(idB); - - TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); - packHeader(pack, 2); - a.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_BLOB, base.length); - deflate(pack, base); - - ByteArrayOutputStream tmp = new ByteArrayOutputStream(); - DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); - de.copy(0, 1); - byte[] delta = tmp.toByteArray(); - b.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length); - idA.copyRawTo(pack); - deflate(pack, delta); - byte[] footer = digest(pack); - - File dir = new File(repo.getObjectDatabase().getDirectory(), - "pack"); - File packName = new File(dir, idA.name() + ".pack"); - File idxName = new File(dir, idA.name() + ".idx"); - - try (FileOutputStream f = new FileOutputStream(packName)) { - f.write(pack.toByteArray()); - } - - try (FileOutputStream f = new FileOutputStream(idxName)) { - List list = new ArrayList<>(); - list.add(a); - list.add(b); - Collections.sort(list); - new PackIndexWriterV1(f).write(list, footer); - } - - PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit()); - try { - packFile.get(wc, b); - fail("expected LargeObjectException.ExceedsByteArrayLimit"); - } catch (LargeObjectException.ExceedsByteArrayLimit bad) { - assertNull(bad.getObjectId()); - } finally { - packFile.close(); - } - } - } - - @Test - public void testConfigurableStreamFileThreshold() throws Exception { - byte[] data = getRng().nextBytes(300); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - try (ObjectStream in = ol.openStream()) { - assertTrue(in instanceof ObjectStream.SmallStream); - assertEquals(300, in.available()); - } - - wc.setStreamFileThreshold(299); - ol = wc.open(id); - try (ObjectStream in = ol.openStream()) { - assertTrue(in instanceof ObjectStream.Filter); - assertEquals(1, in.available()); - } - } - - private static byte[] clone(int first, byte[] base) { - byte[] r = new byte[base.length]; - System.arraycopy(base, 1, r, 1, r.length - 1); - r[0] = (byte) first; - return r; - } - - private static byte[] delta(byte[] base, byte[] dest) throws IOException { - ByteArrayOutputStream tmp = new ByteArrayOutputStream(); - DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length); - de.insert(dest, 0, 1); - de.copy(1, base.length - 1); - return tmp.toByteArray(); - } - - private static void packHeader(TemporaryBuffer.Heap pack, int cnt) - throws IOException { - final byte[] hdr = new byte[8]; - NB.encodeInt32(hdr, 0, 2); - NB.encodeInt32(hdr, 4, cnt); - pack.write(Constants.PACK_SIGNATURE); - pack.write(hdr, 0, 8); - } - - private static void objectHeader(TemporaryBuffer.Heap pack, int type, int sz) - throws IOException { - byte[] buf = new byte[8]; - int nextLength = sz >>> 4; - buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F)); - sz = nextLength; - int n = 1; - while (sz > 0) { - nextLength >>>= 7; - buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F)); - sz = nextLength; - } - pack.write(buf, 0, n); - } - - private static void deflate(TemporaryBuffer.Heap pack, byte[] content) - throws IOException { - final Deflater deflater = new Deflater(); - final byte[] buf = new byte[128]; - deflater.setInput(content, 0, content.length); - deflater.finish(); - do { - final int n = deflater.deflate(buf, 0, buf.length); - if (n > 0) - pack.write(buf, 0, n); - } while (!deflater.finished()); - deflater.end(); - } - - private static byte[] digest(TemporaryBuffer.Heap buf) - throws IOException { - MessageDigest md = Constants.newMessageDigest(); - md.update(buf.toByteArray()); - byte[] footer = md.digest(); - buf.write(footer); - return footer; - } - - private ObjectInserter inserter; - - @After - public void release() { - if (inserter != null) { - inserter.close(); - } - } - - private PackParser index(byte[] raw) throws IOException { - if (inserter == null) - inserter = repo.newObjectInserter(); - return inserter.newPackParser(new ByteArrayInputStream(raw)); - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 8c56480fe1..85043034aa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -160,7 +160,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(3, packs.get(0).getObjectCount()); @@ -193,7 +193,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(2, packs.size()); assertEquals(1, packs.get(0).getObjectCount()); assertEquals(1, packs.get(1).getObjectCount()); @@ -216,9 +216,9 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - Collection packs = listPacks(); + Collection packs = listPacks(); assertEquals(1, packs.size()); - PackFile p = packs.iterator().next(); + Pack p = packs.iterator().next(); assertEquals(1, p.getObjectCount()); try (ObjectReader reader = db.newObjectReader()) { @@ -237,9 +237,9 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); - PackFile pack = packs.get(0); + Pack pack = packs.get(0); assertEquals(1, pack.getObjectCount()); String inode = getInode(pack.getPackFile()); @@ -372,7 +372,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(2, packs.get(0).getObjectCount()); @@ -489,16 +489,16 @@ public class PackInserterTest extends RepositoryTestCase { } } - private List listPacks() throws Exception { - List fromOpenDb = listPacks(db); - List reopened; + private List listPacks() throws Exception { + List fromOpenDb = listPacks(db); + List reopened; try (FileRepository db2 = new FileRepository(db.getDirectory())) { reopened = listPacks(db2); } assertEquals(fromOpenDb.size(), reopened.size()); for (int i = 0 ; i < fromOpenDb.size(); i++) { - PackFile a = fromOpenDb.get(i); - PackFile b = reopened.get(i); + Pack a = fromOpenDb.get(i); + Pack b = reopened.get(i); assertEquals(a.getPackName(), b.getPackName()); assertEquals( a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); @@ -508,9 +508,9 @@ public class PackInserterTest extends RepositoryTestCase { return fromOpenDb; } - private static List listPacks(FileRepository db) throws Exception { + private static List listPacks(FileRepository db) throws Exception { return db.getObjectDatabase().getPacks().stream() - .sorted(comparing(PackFile::getPackName)).collect(toList()); + .sorted(comparing(Pack::getPackName)).collect(toList()); } private PackInserter newInserter() { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java new file mode 100644 index 0000000000..182e422650 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -0,0 +1,374 @@ +/* + * Copyright (C) 2010, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.security.MessageDigest; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.zip.Deflater; + +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.DeltaEncoder; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.junit.TestRng; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.transport.PackParser; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class PackTest extends LocalDiskRepositoryTestCase { + private int streamThreshold = 16 * 1024; + + private TestRng rng; + + private FileRepository repo; + + private TestRepository tr; + + private WindowCursor wc; + + private TestRng getRng() { + if (rng == null) + rng = new TestRng(JGitTestUtil.getName()); + return rng; + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + WindowCacheConfig cfg = new WindowCacheConfig(); + cfg.setStreamFileThreshold(streamThreshold); + cfg.install(); + + repo = createBareRepository(); + tr = new TestRepository<>(repo); + wc = (WindowCursor) repo.newObjectReader(); + } + + @Override + @After + public void tearDown() throws Exception { + if (wc != null) + wc.close(); + new WindowCacheConfig().install(); + super.tearDown(); + } + + @Test + public void testWhole_SmallObject() throws Exception { + final int type = Constants.OBJ_BLOB; + byte[] data = getRng().nextBytes(300); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + assertNotNull("created loader", ol); + assertEquals(type, ol.getType()); + assertEquals(data.length, ol.getSize()); + assertFalse("is not large", ol.isLarge()); + assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(type, in.getType()); + assertEquals(data.length, in.getSize()); + byte[] data2 = new byte[data.length]; + IO.readFully(in, data2, 0, data.length); + assertTrue("same content", Arrays.equals(data2, data)); + assertEquals("stream at EOF", -1, in.read()); + } + } + + @Test + public void testWhole_LargeObject() throws Exception { + final int type = Constants.OBJ_BLOB; + byte[] data = getRng().nextBytes(streamThreshold + 5); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + assertNotNull("created loader", ol); + assertEquals(type, ol.getType()); + assertEquals(data.length, ol.getSize()); + assertTrue("is large", ol.isLarge()); + try { + ol.getCachedBytes(); + fail("Should have thrown LargeObjectException"); + } catch (LargeObjectException tooBig) { + assertEquals(MessageFormat.format( + JGitText.get().largeObjectException, id.name()), tooBig + .getMessage()); + } + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(type, in.getType()); + assertEquals(data.length, in.getSize()); + byte[] data2 = new byte[data.length]; + IO.readFully(in, data2, 0, data.length); + assertTrue("same content", Arrays.equals(data2, data)); + assertEquals("stream at EOF", -1, in.read()); + } + } + + @Test + public void testDelta_SmallObjectChain() throws Exception { + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + byte[] data0 = new byte[512]; + Arrays.fill(data0, (byte) 0xf3); + ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); + packHeader(pack, 4); + objectHeader(pack, Constants.OBJ_BLOB, data0.length); + deflate(pack, data0); + + byte[] data1 = clone(0x01, data0); + byte[] delta1 = delta(data0, data1); + ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); + id0.copyRawTo(pack); + deflate(pack, delta1); + + byte[] data2 = clone(0x02, data1); + byte[] delta2 = delta(data1, data2); + ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); + id1.copyRawTo(pack); + deflate(pack, delta2); + + byte[] data3 = clone(0x03, data2); + byte[] delta3 = delta(data2, data3); + ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); + id2.copyRawTo(pack); + deflate(pack, delta3); + + digest(pack); + PackParser ip = index(pack.toByteArray()); + ip.setAllowThin(true); + ip.parse(NullProgressMonitor.INSTANCE); + + assertTrue("has blob", wc.has(id3)); + + ObjectLoader ol = wc.open(id3); + assertNotNull("created loader", ol); + assertEquals(Constants.OBJ_BLOB, ol.getType()); + assertEquals(data3.length, ol.getSize()); + assertFalse("is large", ol.isLarge()); + assertNotNull(ol.getCachedBytes()); + assertArrayEquals(data3, ol.getCachedBytes()); + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(Constants.OBJ_BLOB, in.getType()); + assertEquals(data3.length, in.getSize()); + byte[] act = new byte[data3.length]; + IO.readFully(in, act, 0, data3.length); + assertTrue("same content", Arrays.equals(act, data3)); + assertEquals("stream at EOF", -1, in.read()); + } + } + } + + @Test + public void testDelta_FailsOver2GiB() throws Exception { + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + byte[] base = new byte[] { 'a' }; + ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base); + ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' }); + + PackedObjectInfo a = new PackedObjectInfo(idA); + PackedObjectInfo b = new PackedObjectInfo(idB); + + TemporaryBuffer.Heap packContents = new TemporaryBuffer.Heap(64 * 1024); + packHeader(packContents, 2); + a.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_BLOB, base.length); + deflate(packContents, base); + + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); + de.copy(0, 1); + byte[] delta = tmp.toByteArray(); + b.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_REF_DELTA, delta.length); + idA.copyRawTo(packContents); + deflate(packContents, delta); + byte[] footer = digest(packContents); + + File dir = new File(repo.getObjectDatabase().getDirectory(), + "pack"); + File packName = new File(dir, idA.name() + ".pack"); + File idxName = new File(dir, idA.name() + ".idx"); + + try (FileOutputStream f = new FileOutputStream(packName)) { + f.write(packContents.toByteArray()); + } + + try (FileOutputStream f = new FileOutputStream(idxName)) { + List list = new ArrayList<>(); + list.add(a); + list.add(b); + Collections.sort(list); + new PackIndexWriterV1(f).write(list, footer); + } + + Pack pack = new Pack(packName, PackExt.INDEX.getBit()); + try { + pack.get(wc, b); + fail("expected LargeObjectException.ExceedsByteArrayLimit"); + } catch (LargeObjectException.ExceedsByteArrayLimit bad) { + assertNull(bad.getObjectId()); + } finally { + pack.close(); + } + } + } + + @Test + public void testConfigurableStreamFileThreshold() throws Exception { + byte[] data = getRng().nextBytes(300); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + try (ObjectStream in = ol.openStream()) { + assertTrue(in instanceof ObjectStream.SmallStream); + assertEquals(300, in.available()); + } + + wc.setStreamFileThreshold(299); + ol = wc.open(id); + try (ObjectStream in = ol.openStream()) { + assertTrue(in instanceof ObjectStream.Filter); + assertEquals(1, in.available()); + } + } + + private static byte[] clone(int first, byte[] base) { + byte[] r = new byte[base.length]; + System.arraycopy(base, 1, r, 1, r.length - 1); + r[0] = (byte) first; + return r; + } + + private static byte[] delta(byte[] base, byte[] dest) throws IOException { + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length); + de.insert(dest, 0, 1); + de.copy(1, base.length - 1); + return tmp.toByteArray(); + } + + private static void packHeader(TemporaryBuffer.Heap pack, int cnt) + throws IOException { + final byte[] hdr = new byte[8]; + NB.encodeInt32(hdr, 0, 2); + NB.encodeInt32(hdr, 4, cnt); + pack.write(Constants.PACK_SIGNATURE); + pack.write(hdr, 0, 8); + } + + private static void objectHeader(TemporaryBuffer.Heap pack, int type, int sz) + throws IOException { + byte[] buf = new byte[8]; + int nextLength = sz >>> 4; + buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F)); + sz = nextLength; + int n = 1; + while (sz > 0) { + nextLength >>>= 7; + buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F)); + sz = nextLength; + } + pack.write(buf, 0, n); + } + + private static void deflate(TemporaryBuffer.Heap pack, byte[] content) + throws IOException { + final Deflater deflater = new Deflater(); + final byte[] buf = new byte[128]; + deflater.setInput(content, 0, content.length); + deflater.finish(); + do { + final int n = deflater.deflate(buf, 0, buf.length); + if (n > 0) + pack.write(buf, 0, n); + } while (!deflater.finished()); + deflater.end(); + } + + private static byte[] digest(TemporaryBuffer.Heap buf) + throws IOException { + MessageDigest md = Constants.newMessageDigest(); + md.update(buf.toByteArray()); + byte[] footer = md.digest(); + buf.write(footer); + return footer; + } + + private ObjectInserter inserter; + + @After + public void release() { + if (inserter != null) { + inserter.close(); + } + } + + private PackParser index(byte[] raw) throws IOException { + if (inserter == null) + inserter = repo.newObjectInserter(); + return inserter.newPackParser(new ByteArrayInputStream(raw)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index c90310e079..214ddb9893 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -72,7 +72,7 @@ public class PackWriterTest extends SampleDataRepositoryTestCase { private ByteArrayOutputStream os; - private PackFile pack; + private Pack pack; private ObjectInserter inserter; @@ -840,7 +840,7 @@ public class PackWriterTest extends SampleDataRepositoryTestCase { p.setAllowThin(thin); p.setIndexVersion(2); p.parse(NullProgressMonitor.INSTANCE); - pack = p.getPackFile(); + pack = p.getPack(); assertNotNull("have PackFile after parsing", pack); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java index ee4c9b1dc7..8f1371e09c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java @@ -32,8 +32,8 @@ public class T0004_PackReaderTest extends SampleDataRepositoryTestCase { final ObjectId id; final ObjectLoader or; - PackFile pr = null; - for (PackFile p : db.getObjectDatabase().getPacks()) { + Pack pr = null; + for (Pack p : db.getObjectDatabase().getPacks()) { if (PACK_NAME.equals(p.getPackName())) { pr = p; break; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java index 07c236daba..60b8098b31 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java @@ -29,7 +29,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.TooLargeObjectInPackException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; @@ -63,16 +63,16 @@ public class PackParserTest extends RepositoryTestCase { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); - - assertTrue(file.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); - assertTrue(file.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); - assertTrue(file.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); - assertTrue(file.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); - assertTrue(file.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); - assertTrue(file.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); - assertTrue(file.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); - assertTrue(file.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); + Pack pack = p.getPack(); + + assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); + assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); + assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); + assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); + assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); + assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); + assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); + assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); } } @@ -88,20 +88,20 @@ public class PackParserTest extends RepositoryTestCase { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); - - assertTrue(file.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); - assertTrue(file.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); - assertTrue(file.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); - assertTrue(file.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); - assertTrue(file.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); - assertTrue(file.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); - assertTrue(file.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); - assertTrue(file.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); - assertTrue(file.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); - assertTrue(file.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); - assertTrue(file.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); - assertTrue(file.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); + Pack pack = p.getPack(); + + assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); + assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); + assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); + assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); + assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); + assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); + assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); + assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); + assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); + assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); + assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); + assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); // and lots more... } } -- cgit v1.2.3