From 9034c7d42349dc995c6215dfb55dcafb65d9e951 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 2 Dec 2020 15:57:16 +0100 Subject: Prepare 5.11.0-SNAPSHOT builds Change-Id: I91e5532526775191fbd34f81e2ef777cba605e3b Signed-off-by: Matthias Sohn --- org.eclipse.jgit.junit/META-INF/MANIFEST.MF | 40 +++++++++++----------- org.eclipse.jgit.junit/META-INF/SOURCE-MANIFEST.MF | 4 +-- org.eclipse.jgit.junit/pom.xml | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) (limited to 'org.eclipse.jgit.junit') diff --git a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF index 3fed43b911..2295bf4ca8 100644 --- a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF @@ -3,35 +3,35 @@ Bundle-ManifestVersion: 2 Bundle-Name: %Bundle-Name Automatic-Module-Name: org.eclipse.jgit.junit Bundle-SymbolicName: org.eclipse.jgit.junit -Bundle-Version: 5.10.0.qualifier +Bundle-Version: 5.11.0.qualifier Bundle-Localization: plugin Bundle-Vendor: %Bundle-Vendor Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.eclipse.jgit.annotations;version="[5.10.0,5.11.0)", - org.eclipse.jgit.api;version="[5.10.0,5.11.0)", - org.eclipse.jgit.api.errors;version="[5.10.0,5.11.0)", - org.eclipse.jgit.dircache;version="[5.10.0,5.11.0)", - org.eclipse.jgit.errors;version="[5.10.0,5.11.0)", - org.eclipse.jgit.internal.storage.file;version="[5.10.0,5.11.0)", - org.eclipse.jgit.internal.storage.pack;version="[5.10.0,5.11.0)", - org.eclipse.jgit.lib;version="[5.10.0,5.11.0)", - org.eclipse.jgit.merge;version="[5.10.0,5.11.0)", - org.eclipse.jgit.revwalk;version="[5.10.0,5.11.0)", - org.eclipse.jgit.storage.file;version="[5.10.0,5.11.0)", - org.eclipse.jgit.transport;version="5.10.0", - org.eclipse.jgit.treewalk;version="[5.10.0,5.11.0)", - org.eclipse.jgit.treewalk.filter;version="[5.10.0,5.11.0)", - org.eclipse.jgit.util;version="[5.10.0,5.11.0)", - org.eclipse.jgit.util.io;version="[5.10.0,5.11.0)", - org.eclipse.jgit.util.time;version="[5.10.0,5.11.0)", +Import-Package: org.eclipse.jgit.annotations;version="[5.11.0,5.12.0)", + org.eclipse.jgit.api;version="[5.11.0,5.12.0)", + org.eclipse.jgit.api.errors;version="[5.11.0,5.12.0)", + org.eclipse.jgit.dircache;version="[5.11.0,5.12.0)", + org.eclipse.jgit.errors;version="[5.11.0,5.12.0)", + org.eclipse.jgit.internal.storage.file;version="[5.11.0,5.12.0)", + org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", + org.eclipse.jgit.lib;version="[5.11.0,5.12.0)", + org.eclipse.jgit.merge;version="[5.11.0,5.12.0)", + org.eclipse.jgit.revwalk;version="[5.11.0,5.12.0)", + org.eclipse.jgit.storage.file;version="[5.11.0,5.12.0)", + org.eclipse.jgit.transport;version="5.11.0", + org.eclipse.jgit.treewalk;version="[5.11.0,5.12.0)", + org.eclipse.jgit.treewalk.filter;version="[5.11.0,5.12.0)", + org.eclipse.jgit.util;version="[5.11.0,5.12.0)", + org.eclipse.jgit.util.io;version="[5.11.0,5.12.0)", + org.eclipse.jgit.util.time;version="[5.11.0,5.12.0)", org.junit;version="[4.13,5.0.0)", org.junit.rules;version="[4.13,5.0.0)", org.junit.runner;version="[4.13,5.0.0)", org.junit.runners;version="[4.13,5.0.0)", org.junit.runners.model;version="[4.13,5.0.0)", org.slf4j;version="[1.7.0,2.0.0)" -Export-Package: org.eclipse.jgit.junit;version="5.10.0"; +Export-Package: org.eclipse.jgit.junit;version="5.11.0"; uses:="org.eclipse.jgit.dircache, org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, @@ -44,4 +44,4 @@ Export-Package: org.eclipse.jgit.junit;version="5.10.0"; org.junit.runners.model, org.junit.runner, org.eclipse.jgit.util.time", - org.eclipse.jgit.junit.time;version="5.10.0";uses:="org.eclipse.jgit.util.time" + org.eclipse.jgit.junit.time;version="5.11.0";uses:="org.eclipse.jgit.util.time" diff --git a/org.eclipse.jgit.junit/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/SOURCE-MANIFEST.MF index cc88d53183..778226932e 100644 --- a/org.eclipse.jgit.junit/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit.junit/META-INF/SOURCE-MANIFEST.MF @@ -3,5 +3,5 @@ Bundle-ManifestVersion: 2 Bundle-Name: org.eclipse.jgit.junit - Sources Bundle-SymbolicName: org.eclipse.jgit.junit.source Bundle-Vendor: Eclipse.org - JGit -Bundle-Version: 5.10.0.qualifier -Eclipse-SourceBundle: org.eclipse.jgit.junit;version="5.10.0.qualifier";roots="." +Bundle-Version: 5.11.0.qualifier +Eclipse-SourceBundle: org.eclipse.jgit.junit;version="5.11.0.qualifier";roots="." diff --git a/org.eclipse.jgit.junit/pom.xml b/org.eclipse.jgit.junit/pom.xml index c08a73a394..26331c36c0 100644 --- a/org.eclipse.jgit.junit/pom.xml +++ b/org.eclipse.jgit.junit/pom.xml @@ -19,7 +19,7 @@ org.eclipse.jgit org.eclipse.jgit-parent - 5.10.0-SNAPSHOT + 5.11.0-SNAPSHOT org.eclipse.jgit.junit -- cgit v1.2.3 From 9299df41cb941ab21b6c4bd835510305b7fd5382 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sat, 26 Dec 2020 02:28:03 +0100 Subject: Fix SeparateClassloaderTestRunner on Java 9 or higher Since Java 9 the SystemClassLoader is no longer a URLClassLoader. Change-Id: I3aa834f1075e611c86fc4684fda6a50c684b3729 Signed-off-by: Matthias Sohn --- .../eclipse/jgit/junit/SeparateClassloaderTestRunner.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit.junit') diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java index b982787e75..4a4dc92c43 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java @@ -9,10 +9,10 @@ */ package org.eclipse.jgit.junit; -import static java.lang.ClassLoader.getSystemClassLoader; - +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Paths; import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.InitializationError; @@ -40,7 +40,13 @@ public class SeparateClassloaderTestRunner extends BlockJUnit4ClassRunner { private static Class loadNewClass(Class klass) throws InitializationError { try { - URL[] urls = ((URLClassLoader) getSystemClassLoader()).getURLs(); + String pathSeparator = System.getProperty("path.separator"); + String[] classPathEntries = System.getProperty("java.class.path") + .split(pathSeparator); + URL[] urls = new URL[classPathEntries.length]; + for (int i = 0; i < classPathEntries.length; i++) { + urls[i] = Paths.get(classPathEntries[i]).toUri().toURL(); + } ClassLoader testClassLoader = new URLClassLoader(urls) { @Override @@ -54,7 +60,7 @@ public class SeparateClassloaderTestRunner extends BlockJUnit4ClassRunner { } }; return Class.forName(klass.getName(), true, testClassLoader); - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | MalformedURLException e) { throw new InitializationError(e); } } -- cgit v1.2.3 From efb154fc24fbf416ae3513942fa720128358b31b Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 10 Feb 2021 22:42:41 -0700 Subject: Rename PackFile to Pack Pack better represents the purpose of the object and paves the way to add a PackFile object that extends File. Change-Id: I39b4f697902d395e9b6df5e8ce53078ce72fcea3 Signed-off-by: Nasser Grainawi --- .../eclipse/jgit/http/server/InfoPacksServlet.java | 4 +- .../src/org/eclipse/jgit/junit/TestRepository.java | 6 +- .../internal/storage/file/GcBasicPackingTest.java | 4 +- .../internal/storage/file/GcConcurrentTest.java | 8 +- .../internal/storage/file/GcKeepFilesTest.java | 6 +- .../storage/file/PackFileSnapshotTest.java | 38 +- .../jgit/internal/storage/file/PackFileTest.java | 374 ------ .../internal/storage/file/PackInserterTest.java | 28 +- .../jgit/internal/storage/file/PackTest.java | 374 ++++++ .../jgit/internal/storage/file/PackWriterTest.java | 4 +- .../storage/file/T0004_PackReaderTest.java | 4 +- .../org/eclipse/jgit/transport/PackParserTest.java | 50 +- .../jgit/errors/NoPackSignatureException.java | 3 +- .../eclipse/jgit/errors/PackInvalidException.java | 2 +- .../eclipse/jgit/errors/PackMismatchException.java | 2 +- .../errors/UnsupportedPackVersionException.java | 2 +- .../internal/storage/file/ByteArrayWindow.java | 2 +- .../internal/storage/file/ByteBufferWindow.java | 2 +- .../jgit/internal/storage/file/ByteWindow.java | 10 +- .../storage/file/CachedObjectDirectory.java | 4 +- .../jgit/internal/storage/file/DeltaBaseCache.java | 6 +- .../internal/storage/file/FileObjectDatabase.java | 4 +- .../org/eclipse/jgit/internal/storage/file/GC.java | 62 +- .../storage/file/LargePackedWholeObject.java | 4 +- .../internal/storage/file/LocalCachedPack.java | 20 +- .../storage/file/LocalObjectRepresentation.java | 24 +- .../internal/storage/file/LocalObjectToPack.java | 2 +- .../internal/storage/file/ObjectDirectory.java | 10 +- .../storage/file/ObjectDirectoryPackParser.java | 6 +- .../eclipse/jgit/internal/storage/file/Pack.java | 1208 ++++++++++++++++++++ .../jgit/internal/storage/file/PackDirectory.java | 75 +- .../jgit/internal/storage/file/PackFile.java | 1208 -------------------- .../jgit/internal/storage/file/PackIndex.java | 2 +- .../internal/storage/file/PackIndexWriter.java | 2 +- .../internal/storage/file/PackInputStream.java | 4 +- .../jgit/internal/storage/file/PackLock.java | 2 +- .../internal/storage/file/PackReverseIndex.java | 2 +- .../jgit/internal/storage/file/WindowCache.java | 73 +- .../jgit/internal/storage/file/WindowCursor.java | 14 +- 39 files changed, 1829 insertions(+), 1826 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java (limited to 'org.eclipse.jgit.junit') diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java index c3d72552a5..e90580b75f 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java @@ -20,7 +20,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.lib.ObjectDatabase; /** Sends the current list of pack files, sorted most recent first. */ @@ -38,7 +38,7 @@ class InfoPacksServlet extends HttpServlet { final StringBuilder out = new StringBuilder(); final ObjectDatabase db = getRepository(req).getObjectDatabase(); if (db instanceof ObjectDirectory) { - for (PackFile pack : ((ObjectDirectory) db).getPacks()) { + for (Pack pack : ((ObjectDirectory) db).getPacks()) { out.append("P "); out.append(pack.getPackFile().getName()); out.append('\n'); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index a5b3b1f3ac..e3eb2c5367 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -43,7 +43,7 @@ import org.eclipse.jgit.errors.ObjectWritingException; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; @@ -773,7 +773,7 @@ public class TestRepository implements AutoCloseable { rw.writeInfoRefs(); final StringBuilder w = new StringBuilder(); - for (PackFile p : fr.getObjectDatabase().getPacks()) { + for (Pack p : fr.getObjectDatabase().getPacks()) { w.append("P "); w.append(p.getPackFile().getName()); w.append('\n'); @@ -954,7 +954,7 @@ public class TestRepository implements AutoCloseable { } private static void prunePacked(ObjectDirectory odb) throws IOException { - for (PackFile p : odb.getPacks()) { + for (Pack p : odb.getPacks()) { for (MutableEntry e : p) FileUtils.delete(odb.fileFor(e.toObjectId())); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index d007dd4511..42e4238451 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -157,7 +157,7 @@ public class GcBasicPackingTest extends GcTestCase { .create(); tr.update("refs/tags/t1", second); - Collection oldPacks = tr.getRepository().getObjectDatabase() + Collection oldPacks = tr.getRepository().getObjectDatabase() .getPacks(); assertEquals(0, oldPacks.size()); stats = gc.getStatistics(); @@ -171,7 +171,7 @@ public class GcBasicPackingTest extends GcTestCase { stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); - List packs = new ArrayList<>( + List packs = new ArrayList<>( repo.getObjectDatabase().getPacks()); assertEquals(11, packs.get(0).getObjectCount()); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java index bb8455f515..5cac1e3429 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java @@ -156,8 +156,8 @@ public class GcConcurrentTest extends GcTestCase { } } - PackFile getSinglePack(FileRepository r) { - Collection packs = r.getObjectDatabase().getPacks(); + Pack getSinglePack(FileRepository r) { + Collection packs = r.getObjectDatabase().getPacks(); assertEquals(1, packs.size()); return packs.iterator().next(); } @@ -206,11 +206,11 @@ public class GcConcurrentTest extends GcTestCase { SampleDataRepositoryTestCase.copyCGitTestPacks(repo); ExecutorService executor = Executors.newSingleThreadExecutor(); final CountDownLatch latch = new CountDownLatch(1); - Future> result = executor.submit(() -> { + Future> result = executor.submit(() -> { long start = System.currentTimeMillis(); System.out.println("starting gc"); latch.countDown(); - Collection r = gc.gc(); + Collection r = gc.gc(); System.out.println( "gc took " + (System.currentTimeMillis() - start) + " ms"); return r; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java index e1559584fd..8472983d5b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java @@ -36,9 +36,9 @@ public class GcKeepFilesTest extends GcTestCase { assertEquals(4, stats.numberOfPackedObjects); assertEquals(1, stats.numberOfPackFiles); - Iterator packIt = repo.getObjectDatabase().getPacks() + Iterator packIt = repo.getObjectDatabase().getPacks() .iterator(); - PackFile singlePack = packIt.next(); + Pack singlePack = packIt.next(); assertFalse(packIt.hasNext()); String packFileName = singlePack.getPackFile().getPath(); String keepFileName = packFileName.substring(0, @@ -58,7 +58,7 @@ public class GcKeepFilesTest extends GcTestCase { assertEquals(2, stats.numberOfPackFiles); // check that no object is packed twice - Iterator packs = repo.getObjectDatabase().getPacks() + Iterator packs = repo.getObjectDatabase().getPacks() .iterator(); PackIndex ind1 = packs.next().getIndex(); assertEquals(4, ind1.getObjectCount()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java index 1f1e094385..7c32ce7cea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java @@ -72,14 +72,14 @@ public class PackFileSnapshotTest extends RepositoryTestCase { c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); c.save(); - Collection packs = gc(Deflater.NO_COMPRESSION); + Collection packs = gc(Deflater.NO_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p1 = packs.iterator().next(); + Pack p1 = packs.iterator().next(); PackFileSnapshot snapshot = p1.getFileSnapshot(); packs = gc(Deflater.BEST_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p2 = packs.iterator().next(); + Pack p2 = packs.iterator().next(); File pf = p2.getPackFile(); // changing compression level with aggressive gc may change size, @@ -153,11 +153,11 @@ public class PackFileSnapshotTest extends RepositoryTestCase { createTestRepo(testDataSeed, testDataLength); // repack to create initial packfile - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); FS fs = db.getFS(); Instant m1 = fs.lastModifiedInstant(packFilePath); @@ -207,16 +207,16 @@ public class PackFileSnapshotTest extends RepositoryTestCase { createTestRepo(testDataSeed, testDataLength); // Repack to create initial packfile. Make a copy of it - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); Path fn = packFilePath.getFileName(); assertNotNull(fn); String packFileName = fn.toString(); Path packFileBasePath = packFilePath .resolveSibling(packFileName.replaceAll(".pack", "")); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); copyPack(packFileBasePath, "", ".copy1"); // Repack to create second packfile. Make a copy of it @@ -280,10 +280,10 @@ public class PackFileSnapshotTest extends RepositoryTestCase { Paths.get(base + ".pack" + dstSuffix)); } - private PackFile repackAndCheck(int compressionLevel, String oldName, + private Pack repackAndCheck(int compressionLevel, String oldName, Long oldLength, AnyObjectId oldChkSum) throws IOException, ParseException { - PackFile p = getSinglePack(gc(compressionLevel)); + Pack p = getSinglePack(gc(compressionLevel)); File pf = p.getPackFile(); // The following two assumptions should not cause the test to fail. If // on a certain platform we get packfiles (containing the same git @@ -298,14 +298,14 @@ public class PackFileSnapshotTest extends RepositoryTestCase { return p; } - private PackFile getSinglePack(Collection packs) { - Iterator pIt = packs.iterator(); - PackFile p = pIt.next(); + private Pack getSinglePack(Collection packs) { + Iterator pIt = packs.iterator(); + Pack p = pIt.next(); assertFalse(pIt.hasNext()); return p; } - private Collection gc(int compressionLevel) + private Collection gc(int compressionLevel) throws IOException, ParseException { GC gc = new GC(db); PackConfig pc = new PackConfig(db.getConfig()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java deleted file mode 100644 index 97a86e249e..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (C) 2010, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.security.MessageDigest; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.zip.Deflater; - -import org.eclipse.jgit.errors.LargeObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.pack.DeltaEncoder; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.junit.TestRng; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectLoader; -import org.eclipse.jgit.lib.ObjectStream; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevBlob; -import org.eclipse.jgit.storage.file.WindowCacheConfig; -import org.eclipse.jgit.transport.PackParser; -import org.eclipse.jgit.transport.PackedObjectInfo; -import org.eclipse.jgit.util.IO; -import org.eclipse.jgit.util.NB; -import org.eclipse.jgit.util.TemporaryBuffer; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class PackFileTest extends LocalDiskRepositoryTestCase { - private int streamThreshold = 16 * 1024; - - private TestRng rng; - - private FileRepository repo; - - private TestRepository tr; - - private WindowCursor wc; - - private TestRng getRng() { - if (rng == null) - rng = new TestRng(JGitTestUtil.getName()); - return rng; - } - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - - WindowCacheConfig cfg = new WindowCacheConfig(); - cfg.setStreamFileThreshold(streamThreshold); - cfg.install(); - - repo = createBareRepository(); - tr = new TestRepository<>(repo); - wc = (WindowCursor) repo.newObjectReader(); - } - - @Override - @After - public void tearDown() throws Exception { - if (wc != null) - wc.close(); - new WindowCacheConfig().install(); - super.tearDown(); - } - - @Test - public void testWhole_SmallObject() throws Exception { - final int type = Constants.OBJ_BLOB; - byte[] data = getRng().nextBytes(300); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - assertNotNull("created loader", ol); - assertEquals(type, ol.getType()); - assertEquals(data.length, ol.getSize()); - assertFalse("is not large", ol.isLarge()); - assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(type, in.getType()); - assertEquals(data.length, in.getSize()); - byte[] data2 = new byte[data.length]; - IO.readFully(in, data2, 0, data.length); - assertTrue("same content", Arrays.equals(data2, data)); - assertEquals("stream at EOF", -1, in.read()); - } - } - - @Test - public void testWhole_LargeObject() throws Exception { - final int type = Constants.OBJ_BLOB; - byte[] data = getRng().nextBytes(streamThreshold + 5); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - assertNotNull("created loader", ol); - assertEquals(type, ol.getType()); - assertEquals(data.length, ol.getSize()); - assertTrue("is large", ol.isLarge()); - try { - ol.getCachedBytes(); - fail("Should have thrown LargeObjectException"); - } catch (LargeObjectException tooBig) { - assertEquals(MessageFormat.format( - JGitText.get().largeObjectException, id.name()), tooBig - .getMessage()); - } - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(type, in.getType()); - assertEquals(data.length, in.getSize()); - byte[] data2 = new byte[data.length]; - IO.readFully(in, data2, 0, data.length); - assertTrue("same content", Arrays.equals(data2, data)); - assertEquals("stream at EOF", -1, in.read()); - } - } - - @Test - public void testDelta_SmallObjectChain() throws Exception { - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - byte[] data0 = new byte[512]; - Arrays.fill(data0, (byte) 0xf3); - ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); - - TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); - packHeader(pack, 4); - objectHeader(pack, Constants.OBJ_BLOB, data0.length); - deflate(pack, data0); - - byte[] data1 = clone(0x01, data0); - byte[] delta1 = delta(data0, data1); - ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); - id0.copyRawTo(pack); - deflate(pack, delta1); - - byte[] data2 = clone(0x02, data1); - byte[] delta2 = delta(data1, data2); - ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); - id1.copyRawTo(pack); - deflate(pack, delta2); - - byte[] data3 = clone(0x03, data2); - byte[] delta3 = delta(data2, data3); - ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); - id2.copyRawTo(pack); - deflate(pack, delta3); - - digest(pack); - PackParser ip = index(pack.toByteArray()); - ip.setAllowThin(true); - ip.parse(NullProgressMonitor.INSTANCE); - - assertTrue("has blob", wc.has(id3)); - - ObjectLoader ol = wc.open(id3); - assertNotNull("created loader", ol); - assertEquals(Constants.OBJ_BLOB, ol.getType()); - assertEquals(data3.length, ol.getSize()); - assertFalse("is large", ol.isLarge()); - assertNotNull(ol.getCachedBytes()); - assertArrayEquals(data3, ol.getCachedBytes()); - - try (ObjectStream in = ol.openStream()) { - assertNotNull("have stream", in); - assertEquals(Constants.OBJ_BLOB, in.getType()); - assertEquals(data3.length, in.getSize()); - byte[] act = new byte[data3.length]; - IO.readFully(in, act, 0, data3.length); - assertTrue("same content", Arrays.equals(act, data3)); - assertEquals("stream at EOF", -1, in.read()); - } - } - } - - @Test - public void testDelta_FailsOver2GiB() throws Exception { - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - byte[] base = new byte[] { 'a' }; - ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base); - ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' }); - - PackedObjectInfo a = new PackedObjectInfo(idA); - PackedObjectInfo b = new PackedObjectInfo(idB); - - TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); - packHeader(pack, 2); - a.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_BLOB, base.length); - deflate(pack, base); - - ByteArrayOutputStream tmp = new ByteArrayOutputStream(); - DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); - de.copy(0, 1); - byte[] delta = tmp.toByteArray(); - b.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length); - idA.copyRawTo(pack); - deflate(pack, delta); - byte[] footer = digest(pack); - - File dir = new File(repo.getObjectDatabase().getDirectory(), - "pack"); - File packName = new File(dir, idA.name() + ".pack"); - File idxName = new File(dir, idA.name() + ".idx"); - - try (FileOutputStream f = new FileOutputStream(packName)) { - f.write(pack.toByteArray()); - } - - try (FileOutputStream f = new FileOutputStream(idxName)) { - List list = new ArrayList<>(); - list.add(a); - list.add(b); - Collections.sort(list); - new PackIndexWriterV1(f).write(list, footer); - } - - PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit()); - try { - packFile.get(wc, b); - fail("expected LargeObjectException.ExceedsByteArrayLimit"); - } catch (LargeObjectException.ExceedsByteArrayLimit bad) { - assertNull(bad.getObjectId()); - } finally { - packFile.close(); - } - } - } - - @Test - public void testConfigurableStreamFileThreshold() throws Exception { - byte[] data = getRng().nextBytes(300); - RevBlob id = tr.blob(data); - tr.branch("master").commit().add("A", id).create(); - tr.packAndPrune(); - assertTrue("has blob", wc.has(id)); - - ObjectLoader ol = wc.open(id); - try (ObjectStream in = ol.openStream()) { - assertTrue(in instanceof ObjectStream.SmallStream); - assertEquals(300, in.available()); - } - - wc.setStreamFileThreshold(299); - ol = wc.open(id); - try (ObjectStream in = ol.openStream()) { - assertTrue(in instanceof ObjectStream.Filter); - assertEquals(1, in.available()); - } - } - - private static byte[] clone(int first, byte[] base) { - byte[] r = new byte[base.length]; - System.arraycopy(base, 1, r, 1, r.length - 1); - r[0] = (byte) first; - return r; - } - - private static byte[] delta(byte[] base, byte[] dest) throws IOException { - ByteArrayOutputStream tmp = new ByteArrayOutputStream(); - DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length); - de.insert(dest, 0, 1); - de.copy(1, base.length - 1); - return tmp.toByteArray(); - } - - private static void packHeader(TemporaryBuffer.Heap pack, int cnt) - throws IOException { - final byte[] hdr = new byte[8]; - NB.encodeInt32(hdr, 0, 2); - NB.encodeInt32(hdr, 4, cnt); - pack.write(Constants.PACK_SIGNATURE); - pack.write(hdr, 0, 8); - } - - private static void objectHeader(TemporaryBuffer.Heap pack, int type, int sz) - throws IOException { - byte[] buf = new byte[8]; - int nextLength = sz >>> 4; - buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F)); - sz = nextLength; - int n = 1; - while (sz > 0) { - nextLength >>>= 7; - buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F)); - sz = nextLength; - } - pack.write(buf, 0, n); - } - - private static void deflate(TemporaryBuffer.Heap pack, byte[] content) - throws IOException { - final Deflater deflater = new Deflater(); - final byte[] buf = new byte[128]; - deflater.setInput(content, 0, content.length); - deflater.finish(); - do { - final int n = deflater.deflate(buf, 0, buf.length); - if (n > 0) - pack.write(buf, 0, n); - } while (!deflater.finished()); - deflater.end(); - } - - private static byte[] digest(TemporaryBuffer.Heap buf) - throws IOException { - MessageDigest md = Constants.newMessageDigest(); - md.update(buf.toByteArray()); - byte[] footer = md.digest(); - buf.write(footer); - return footer; - } - - private ObjectInserter inserter; - - @After - public void release() { - if (inserter != null) { - inserter.close(); - } - } - - private PackParser index(byte[] raw) throws IOException { - if (inserter == null) - inserter = repo.newObjectInserter(); - return inserter.newPackParser(new ByteArrayInputStream(raw)); - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 8c56480fe1..85043034aa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -160,7 +160,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(3, packs.get(0).getObjectCount()); @@ -193,7 +193,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(2, packs.size()); assertEquals(1, packs.get(0).getObjectCount()); assertEquals(1, packs.get(1).getObjectCount()); @@ -216,9 +216,9 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - Collection packs = listPacks(); + Collection packs = listPacks(); assertEquals(1, packs.size()); - PackFile p = packs.iterator().next(); + Pack p = packs.iterator().next(); assertEquals(1, p.getObjectCount()); try (ObjectReader reader = db.newObjectReader()) { @@ -237,9 +237,9 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); - PackFile pack = packs.get(0); + Pack pack = packs.get(0); assertEquals(1, pack.getObjectCount()); String inode = getInode(pack.getPackFile()); @@ -372,7 +372,7 @@ public class PackInserterTest extends RepositoryTestCase { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(2, packs.get(0).getObjectCount()); @@ -489,16 +489,16 @@ public class PackInserterTest extends RepositoryTestCase { } } - private List listPacks() throws Exception { - List fromOpenDb = listPacks(db); - List reopened; + private List listPacks() throws Exception { + List fromOpenDb = listPacks(db); + List reopened; try (FileRepository db2 = new FileRepository(db.getDirectory())) { reopened = listPacks(db2); } assertEquals(fromOpenDb.size(), reopened.size()); for (int i = 0 ; i < fromOpenDb.size(); i++) { - PackFile a = fromOpenDb.get(i); - PackFile b = reopened.get(i); + Pack a = fromOpenDb.get(i); + Pack b = reopened.get(i); assertEquals(a.getPackName(), b.getPackName()); assertEquals( a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); @@ -508,9 +508,9 @@ public class PackInserterTest extends RepositoryTestCase { return fromOpenDb; } - private static List listPacks(FileRepository db) throws Exception { + private static List listPacks(FileRepository db) throws Exception { return db.getObjectDatabase().getPacks().stream() - .sorted(comparing(PackFile::getPackName)).collect(toList()); + .sorted(comparing(Pack::getPackName)).collect(toList()); } private PackInserter newInserter() { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java new file mode 100644 index 0000000000..182e422650 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -0,0 +1,374 @@ +/* + * Copyright (C) 2010, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.security.MessageDigest; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.zip.Deflater; + +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.DeltaEncoder; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.junit.TestRng; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.transport.PackParser; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class PackTest extends LocalDiskRepositoryTestCase { + private int streamThreshold = 16 * 1024; + + private TestRng rng; + + private FileRepository repo; + + private TestRepository tr; + + private WindowCursor wc; + + private TestRng getRng() { + if (rng == null) + rng = new TestRng(JGitTestUtil.getName()); + return rng; + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + WindowCacheConfig cfg = new WindowCacheConfig(); + cfg.setStreamFileThreshold(streamThreshold); + cfg.install(); + + repo = createBareRepository(); + tr = new TestRepository<>(repo); + wc = (WindowCursor) repo.newObjectReader(); + } + + @Override + @After + public void tearDown() throws Exception { + if (wc != null) + wc.close(); + new WindowCacheConfig().install(); + super.tearDown(); + } + + @Test + public void testWhole_SmallObject() throws Exception { + final int type = Constants.OBJ_BLOB; + byte[] data = getRng().nextBytes(300); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + assertNotNull("created loader", ol); + assertEquals(type, ol.getType()); + assertEquals(data.length, ol.getSize()); + assertFalse("is not large", ol.isLarge()); + assertTrue("same content", Arrays.equals(data, ol.getCachedBytes())); + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(type, in.getType()); + assertEquals(data.length, in.getSize()); + byte[] data2 = new byte[data.length]; + IO.readFully(in, data2, 0, data.length); + assertTrue("same content", Arrays.equals(data2, data)); + assertEquals("stream at EOF", -1, in.read()); + } + } + + @Test + public void testWhole_LargeObject() throws Exception { + final int type = Constants.OBJ_BLOB; + byte[] data = getRng().nextBytes(streamThreshold + 5); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + assertNotNull("created loader", ol); + assertEquals(type, ol.getType()); + assertEquals(data.length, ol.getSize()); + assertTrue("is large", ol.isLarge()); + try { + ol.getCachedBytes(); + fail("Should have thrown LargeObjectException"); + } catch (LargeObjectException tooBig) { + assertEquals(MessageFormat.format( + JGitText.get().largeObjectException, id.name()), tooBig + .getMessage()); + } + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(type, in.getType()); + assertEquals(data.length, in.getSize()); + byte[] data2 = new byte[data.length]; + IO.readFully(in, data2, 0, data.length); + assertTrue("same content", Arrays.equals(data2, data)); + assertEquals("stream at EOF", -1, in.read()); + } + } + + @Test + public void testDelta_SmallObjectChain() throws Exception { + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + byte[] data0 = new byte[512]; + Arrays.fill(data0, (byte) 0xf3); + ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0); + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); + packHeader(pack, 4); + objectHeader(pack, Constants.OBJ_BLOB, data0.length); + deflate(pack, data0); + + byte[] data1 = clone(0x01, data0); + byte[] delta1 = delta(data0, data1); + ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length); + id0.copyRawTo(pack); + deflate(pack, delta1); + + byte[] data2 = clone(0x02, data1); + byte[] delta2 = delta(data1, data2); + ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length); + id1.copyRawTo(pack); + deflate(pack, delta2); + + byte[] data3 = clone(0x03, data2); + byte[] delta3 = delta(data2, data3); + ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3); + objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length); + id2.copyRawTo(pack); + deflate(pack, delta3); + + digest(pack); + PackParser ip = index(pack.toByteArray()); + ip.setAllowThin(true); + ip.parse(NullProgressMonitor.INSTANCE); + + assertTrue("has blob", wc.has(id3)); + + ObjectLoader ol = wc.open(id3); + assertNotNull("created loader", ol); + assertEquals(Constants.OBJ_BLOB, ol.getType()); + assertEquals(data3.length, ol.getSize()); + assertFalse("is large", ol.isLarge()); + assertNotNull(ol.getCachedBytes()); + assertArrayEquals(data3, ol.getCachedBytes()); + + try (ObjectStream in = ol.openStream()) { + assertNotNull("have stream", in); + assertEquals(Constants.OBJ_BLOB, in.getType()); + assertEquals(data3.length, in.getSize()); + byte[] act = new byte[data3.length]; + IO.readFully(in, act, 0, data3.length); + assertTrue("same content", Arrays.equals(act, data3)); + assertEquals("stream at EOF", -1, in.read()); + } + } + } + + @Test + public void testDelta_FailsOver2GiB() throws Exception { + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + byte[] base = new byte[] { 'a' }; + ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base); + ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' }); + + PackedObjectInfo a = new PackedObjectInfo(idA); + PackedObjectInfo b = new PackedObjectInfo(idB); + + TemporaryBuffer.Heap packContents = new TemporaryBuffer.Heap(64 * 1024); + packHeader(packContents, 2); + a.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_BLOB, base.length); + deflate(packContents, base); + + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); + de.copy(0, 1); + byte[] delta = tmp.toByteArray(); + b.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_REF_DELTA, delta.length); + idA.copyRawTo(packContents); + deflate(packContents, delta); + byte[] footer = digest(packContents); + + File dir = new File(repo.getObjectDatabase().getDirectory(), + "pack"); + File packName = new File(dir, idA.name() + ".pack"); + File idxName = new File(dir, idA.name() + ".idx"); + + try (FileOutputStream f = new FileOutputStream(packName)) { + f.write(packContents.toByteArray()); + } + + try (FileOutputStream f = new FileOutputStream(idxName)) { + List list = new ArrayList<>(); + list.add(a); + list.add(b); + Collections.sort(list); + new PackIndexWriterV1(f).write(list, footer); + } + + Pack pack = new Pack(packName, PackExt.INDEX.getBit()); + try { + pack.get(wc, b); + fail("expected LargeObjectException.ExceedsByteArrayLimit"); + } catch (LargeObjectException.ExceedsByteArrayLimit bad) { + assertNull(bad.getObjectId()); + } finally { + pack.close(); + } + } + } + + @Test + public void testConfigurableStreamFileThreshold() throws Exception { + byte[] data = getRng().nextBytes(300); + RevBlob id = tr.blob(data); + tr.branch("master").commit().add("A", id).create(); + tr.packAndPrune(); + assertTrue("has blob", wc.has(id)); + + ObjectLoader ol = wc.open(id); + try (ObjectStream in = ol.openStream()) { + assertTrue(in instanceof ObjectStream.SmallStream); + assertEquals(300, in.available()); + } + + wc.setStreamFileThreshold(299); + ol = wc.open(id); + try (ObjectStream in = ol.openStream()) { + assertTrue(in instanceof ObjectStream.Filter); + assertEquals(1, in.available()); + } + } + + private static byte[] clone(int first, byte[] base) { + byte[] r = new byte[base.length]; + System.arraycopy(base, 1, r, 1, r.length - 1); + r[0] = (byte) first; + return r; + } + + private static byte[] delta(byte[] base, byte[] dest) throws IOException { + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length); + de.insert(dest, 0, 1); + de.copy(1, base.length - 1); + return tmp.toByteArray(); + } + + private static void packHeader(TemporaryBuffer.Heap pack, int cnt) + throws IOException { + final byte[] hdr = new byte[8]; + NB.encodeInt32(hdr, 0, 2); + NB.encodeInt32(hdr, 4, cnt); + pack.write(Constants.PACK_SIGNATURE); + pack.write(hdr, 0, 8); + } + + private static void objectHeader(TemporaryBuffer.Heap pack, int type, int sz) + throws IOException { + byte[] buf = new byte[8]; + int nextLength = sz >>> 4; + buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F)); + sz = nextLength; + int n = 1; + while (sz > 0) { + nextLength >>>= 7; + buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F)); + sz = nextLength; + } + pack.write(buf, 0, n); + } + + private static void deflate(TemporaryBuffer.Heap pack, byte[] content) + throws IOException { + final Deflater deflater = new Deflater(); + final byte[] buf = new byte[128]; + deflater.setInput(content, 0, content.length); + deflater.finish(); + do { + final int n = deflater.deflate(buf, 0, buf.length); + if (n > 0) + pack.write(buf, 0, n); + } while (!deflater.finished()); + deflater.end(); + } + + private static byte[] digest(TemporaryBuffer.Heap buf) + throws IOException { + MessageDigest md = Constants.newMessageDigest(); + md.update(buf.toByteArray()); + byte[] footer = md.digest(); + buf.write(footer); + return footer; + } + + private ObjectInserter inserter; + + @After + public void release() { + if (inserter != null) { + inserter.close(); + } + } + + private PackParser index(byte[] raw) throws IOException { + if (inserter == null) + inserter = repo.newObjectInserter(); + return inserter.newPackParser(new ByteArrayInputStream(raw)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index c90310e079..214ddb9893 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -72,7 +72,7 @@ public class PackWriterTest extends SampleDataRepositoryTestCase { private ByteArrayOutputStream os; - private PackFile pack; + private Pack pack; private ObjectInserter inserter; @@ -840,7 +840,7 @@ public class PackWriterTest extends SampleDataRepositoryTestCase { p.setAllowThin(thin); p.setIndexVersion(2); p.parse(NullProgressMonitor.INSTANCE); - pack = p.getPackFile(); + pack = p.getPack(); assertNotNull("have PackFile after parsing", pack); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java index ee4c9b1dc7..8f1371e09c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java @@ -32,8 +32,8 @@ public class T0004_PackReaderTest extends SampleDataRepositoryTestCase { final ObjectId id; final ObjectLoader or; - PackFile pr = null; - for (PackFile p : db.getObjectDatabase().getPacks()) { + Pack pr = null; + for (Pack p : db.getObjectDatabase().getPacks()) { if (PACK_NAME.equals(p.getPackName())) { pr = p; break; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java index 07c236daba..60b8098b31 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java @@ -29,7 +29,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.TooLargeObjectInPackException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; @@ -63,16 +63,16 @@ public class PackParserTest extends RepositoryTestCase { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); - - assertTrue(file.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); - assertTrue(file.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); - assertTrue(file.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); - assertTrue(file.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); - assertTrue(file.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); - assertTrue(file.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); - assertTrue(file.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); - assertTrue(file.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); + Pack pack = p.getPack(); + + assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); + assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); + assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); + assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); + assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); + assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); + assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); + assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); } } @@ -88,20 +88,20 @@ public class PackParserTest extends RepositoryTestCase { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); - - assertTrue(file.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); - assertTrue(file.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); - assertTrue(file.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); - assertTrue(file.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); - assertTrue(file.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); - assertTrue(file.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); - assertTrue(file.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); - assertTrue(file.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); - assertTrue(file.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); - assertTrue(file.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); - assertTrue(file.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); - assertTrue(file.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); + Pack pack = p.getPack(); + + assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); + assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); + assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); + assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); + assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); + assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); + assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); + assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); + assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); + assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); + assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); + assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); // and lots more... } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java index c3b1df9928..a37b8bee24 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java @@ -13,8 +13,7 @@ package org.eclipse.jgit.errors; import java.io.IOException; /** - * Thrown when a PackFile is found not to contain the pack signature defined by - * git. + * Thrown when a Pack is found not to contain the pack signature defined by git. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index c484984f82..1fd80867b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java @@ -17,7 +17,7 @@ import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile previously failed and is known to be unusable + * Thrown when a Pack previously failed and is known to be unusable */ public class PackInvalidException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java index ad5664ceb2..44b8e0193c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java @@ -13,7 +13,7 @@ package org.eclipse.jgit.errors; import java.io.IOException; /** - * Thrown when a PackFile no longer matches the PackIndex. + * Thrown when a Pack no longer matches the PackIndex. */ public class PackMismatchException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java index 7538229950..07aa7564dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java @@ -16,7 +16,7 @@ import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile uses a pack version not supported by JGit. + * Thrown when a Pack uses a pack version not supported by JGit. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java index 45d9c85c8c..1036535423 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; final class ByteArrayWindow extends ByteWindow { private final byte[] array; - ByteArrayWindow(PackFile pack, long o, byte[] b) { + ByteArrayWindow(Pack pack, long o, byte[] b) { super(pack, o, b.length); array = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java index 8703216322..b6877578c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java @@ -27,7 +27,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; final class ByteBufferWindow extends ByteWindow { private final ByteBuffer buffer; - ByteBufferWindow(PackFile pack, long o, ByteBuffer b) { + ByteBufferWindow(Pack pack, long o, ByteBuffer b) { super(pack, o, b.capacity()); buffer = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java index 159f31c971..31e7eadd8a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java @@ -27,7 +27,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; *

*/ abstract class ByteWindow { - protected final PackFile pack; + protected final Pack pack; protected final long start; @@ -37,13 +37,13 @@ abstract class ByteWindow { * Constructor for ByteWindow. * * @param p - * a {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * a {@link org.eclipse.jgit.internal.storage.file.Pack}. * @param s * where the byte window starts in the pack file * @param n * size of the byte window */ - protected ByteWindow(PackFile p, long s, int n) { + protected ByteWindow(Pack p, long s, int n) { pack = p; start = s; end = start + n; @@ -53,8 +53,8 @@ abstract class ByteWindow { return (int) (end - start); } - final boolean contains(PackFile neededFile, long neededPos) { - return pack == neededFile && start <= neededPos && neededPos < end; + final boolean contains(Pack neededPack, long neededPos) { + return pack == neededPack && start <= neededPos && neededPos < end; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java index 9c7a2e7111..7dedeb57ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java @@ -239,7 +239,7 @@ class CachedObjectDirectory extends FileObjectDatabase { } @Override - PackFile openPack(File pack) throws IOException { + Pack openPack(File pack) throws IOException { return wrapped.openPack(pack); } @@ -250,7 +250,7 @@ class CachedObjectDirectory extends FileObjectDatabase { } @Override - Collection getPacks() { + Collection getPacks() { return wrapped.getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java index cef5a330f2..69cebadf1e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java @@ -49,7 +49,7 @@ class DeltaBaseCache { cache = new Slot[CACHE_SZ]; } - Entry get(PackFile pack, long position) { + Entry get(Pack pack, long position) { Slot e = cache[hash(position)]; if (e == null) return null; @@ -63,7 +63,7 @@ class DeltaBaseCache { return null; } - void store(final PackFile pack, final long position, + void store(final Pack pack, final long position, final byte[] data, final int objectType) { if (data.length > maxByteCount) return; // Too large to cache. @@ -146,7 +146,7 @@ class DeltaBaseCache { Slot lruNext; - PackFile provider; + Pack provider; long position; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java index 11ed10c90a..01dd27d9fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java @@ -71,7 +71,7 @@ abstract class FileObjectDatabase extends ObjectDatabase { abstract InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException; - abstract PackFile openPack(File pack) throws IOException; + abstract Pack openPack(File pack) throws IOException; - abstract Collection getPacks(); + abstract Collection getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 324075269a..75de3be89e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -205,16 +205,16 @@ public class GC { * gc.log. * * @return the collection of - * {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which + * {@link org.eclipse.jgit.internal.storage.file.Pack}'s which * are newly created * @throws java.io.IOException * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed */ - // TODO(ms): change signature and return Future> + // TODO(ms): change signature and return Future> @SuppressWarnings("FutureReturnValueIgnored") - public Collection gc() throws IOException, ParseException { + public Collection gc() throws IOException, ParseException { if (!background) { return doGc(); } @@ -224,9 +224,9 @@ public class GC { return Collections.emptyList(); } - Callable> gcTask = () -> { + Callable> gcTask = () -> { try { - Collection newPacks = doGc(); + Collection newPacks = doGc(); if (automatic && tooManyLooseObjects()) { String message = JGitText.get().gcTooManyUnpruned; gcLog.write(message); @@ -258,14 +258,14 @@ public class GC { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection doGc() throws IOException, ParseException { + private Collection doGc() throws IOException, ParseException { if (automatic && !needGc()) { return Collections.emptyList(); } pm.start(6 /* tasks */); packRefs(); // TODO: implement reflog_expire(pm, repo); - Collection newPacks = repack(); + Collection newPacks = repack(); prune(Collections.emptySet()); // TODO: implement rerere_gc(pm); return newPacks; @@ -281,7 +281,7 @@ public class GC { * @param existing * @throws IOException */ - private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet existing) + private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet existing) throws IOException { for (PackIndex.MutableEntry entry : pack) { ObjectId oid = entry.toObjectId(); @@ -313,10 +313,10 @@ public class GC { * @throws ParseException * @throws IOException */ - private void deleteOldPacks(Collection oldPacks, - Collection newPacks) throws ParseException, IOException { + private void deleteOldPacks(Collection oldPacks, + Collection newPacks) throws ParseException, IOException { HashSet ids = new HashSet<>(); - for (PackFile pack : newPacks) { + for (Pack pack : newPacks) { for (PackIndex.MutableEntry entry : pack) { ids.add(entry.toObjectId()); } @@ -329,12 +329,12 @@ public class GC { prunePreserved(); long packExpireDate = getPackExpireDate(); - oldPackLoop: for (PackFile oldPack : oldPacks) { + oldPackLoop: for (Pack oldPack : oldPacks) { checkCancelled(); String oldName = oldPack.getPackName(); // check whether an old pack file is also among the list of new // pack files. Then we must not delete it. - for (PackFile newPack : newPacks) + for (Pack newPack : newPacks) if (oldName.equals(newPack.getPackName())) continue oldPackLoop; @@ -438,7 +438,7 @@ public class GC { */ public void prunePacked() throws IOException { ObjectDirectory objdb = repo.getObjectDatabase(); - Collection packs = objdb.getPacks(); + Collection packs = objdb.getPacks(); File objects = repo.getObjectsDirectory(); String[] fanout = objects.list(); @@ -466,7 +466,7 @@ public class GC { continue; } boolean found = false; - for (PackFile p : packs) { + for (Pack p : packs) { checkCancelled(); if (p.hasObject(id)) { found = true; @@ -788,8 +788,8 @@ public class GC { * reflog-entries or during writing to the packfiles * {@link java.io.IOException} occurs */ - public Collection repack() throws IOException { - Collection toBeDeleted = repo.getObjectDatabase().getPacks(); + public Collection repack() throws IOException { + Collection toBeDeleted = repo.getObjectDatabase().getPacks(); long time = System.currentTimeMillis(); Collection refsBefore = getAllRefs(); @@ -821,10 +821,10 @@ public class GC { } List excluded = new LinkedList<>(); - for (PackFile f : repo.getObjectDatabase().getPacks()) { + for (Pack p : repo.getObjectDatabase().getPacks()) { checkCancelled(); - if (f.shouldBeKept()) - excluded.add(f.getIndex()); + if (p.shouldBeKept()) + excluded.add(p.getIndex()); } // Don't exclude tags that are also branch tips @@ -842,8 +842,8 @@ public class GC { nonHeads.clear(); } - List ret = new ArrayList<>(2); - PackFile heads = null; + List ret = new ArrayList<>(2); + Pack heads = null; if (!allHeadsAndTags.isEmpty()) { heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, tagTargets, excluded); @@ -853,13 +853,13 @@ public class GC { } } if (!nonHeads.isEmpty()) { - PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, + Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, tagTargets, excluded); if (rest != null) ret.add(rest); } if (!txnHeads.isEmpty()) { - PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, + Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, null, excluded); if (txn != null) ret.add(txn); @@ -1129,7 +1129,7 @@ public class GC { } } - private PackFile writePack(@NonNull Set want, + private Pack writePack(@NonNull Set want, @NonNull Set have, @NonNull Set tags, Set tagTargets, List excludeObjects) throws IOException { @@ -1356,13 +1356,13 @@ public class GC { */ public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); - Collection packs = repo.getObjectDatabase().getPacks(); - for (PackFile f : packs) { - ret.numberOfPackedObjects += f.getIndex().getObjectCount(); + Collection packs = repo.getObjectDatabase().getPacks(); + for (Pack p : packs) { + ret.numberOfPackedObjects += p.getIndex().getObjectCount(); ret.numberOfPackFiles++; - ret.sizeOfPackedObjects += f.getPackFile().length(); - if (f.getBitmapIndex() != null) - ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount(); + ret.sizeOfPackedObjects += p.getPackFile().length(); + if (p.getBitmapIndex() != null) + ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java index ee4bbc1964..e2fbd7a0b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java @@ -30,12 +30,12 @@ class LargePackedWholeObject extends ObjectLoader { private final int headerLength; - private final PackFile pack; + private final Pack pack; private final FileObjectDatabase db; LargePackedWholeObject(int type, long size, long objectOffset, - int headerLength, PackFile pack, FileObjectDatabase db) { + int headerLength, Pack pack, FileObjectDatabase db) { this.type = type; this.size = size; this.objectOffset = objectOffset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index 9d04062e37..ae5bce6985 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -25,31 +25,31 @@ class LocalCachedPack extends CachedPack { private final String[] packNames; - private PackFile[] packs; + private Pack[] packs; LocalCachedPack(ObjectDirectory odb, List packNames) { this.odb = odb; this.packNames = packNames.toArray(new String[0]); } - LocalCachedPack(List packs) { + LocalCachedPack(List packs) { odb = null; packNames = null; - this.packs = packs.toArray(new PackFile[0]); + this.packs = packs.toArray(new Pack[0]); } /** {@inheritDoc} */ @Override public long getObjectCount() throws IOException { long cnt = 0; - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) cnt += pack.getObjectCount(); return cnt; } void copyAsIs(PackOutputStream out, WindowCursor wc) throws IOException { - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) pack.copyPackAsIs(out, wc); } @@ -58,7 +58,7 @@ class LocalCachedPack extends CachedPack { public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { try { LocalObjectRepresentation local = (LocalObjectRepresentation) rep; - for (PackFile pack : getPacks()) { + for (Pack pack : getPacks()) { if (local.pack == pack) return true; } @@ -68,9 +68,9 @@ class LocalCachedPack extends CachedPack { } } - private PackFile[] getPacks() throws FileNotFoundException { + private Pack[] getPacks() throws FileNotFoundException { if (packs == null) { - PackFile[] p = new PackFile[packNames.length]; + Pack[] p = new Pack[packNames.length]; for (int i = 0; i < packNames.length; i++) p[i] = getPackFile(packNames[i]); packs = p; @@ -78,8 +78,8 @@ class LocalCachedPack extends CachedPack { return packs; } - private PackFile getPackFile(String packName) throws FileNotFoundException { - for (PackFile pack : odb.getPacks()) { + private Pack getPackFile(String packName) throws FileNotFoundException { + for (Pack pack : odb.getPacks()) { if (packName.equals(pack.getPackName())) return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java index 3950dde4a5..559718af3a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java @@ -16,40 +16,40 @@ import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; import org.eclipse.jgit.lib.ObjectId; class LocalObjectRepresentation extends StoredObjectRepresentation { - static LocalObjectRepresentation newWhole(PackFile f, long p, long length) { + static LocalObjectRepresentation newWhole(Pack pack, long offset, long length) { LocalObjectRepresentation r = new LocalObjectRepresentation() { @Override public int getFormat() { return PACK_WHOLE; } }; - r.pack = f; - r.offset = p; + r.pack = pack; + r.offset = offset; r.length = length; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, ObjectId base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseId = base; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, long base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseOffset = base; return r; } - PackFile pack; + Pack pack; long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java index 4a0ac1fd84..ac6cd212d5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java @@ -17,7 +17,7 @@ import org.eclipse.jgit.lib.AnyObjectId; /** {@link ObjectToPack} for {@link ObjectDirectory}. */ class LocalObjectToPack extends ObjectToPack { /** Pack to reuse compressed data from, otherwise null. */ - PackFile pack; + Pack pack; /** Offset of the object's header in {@link #pack}. */ long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 4a40db68dd..e71a960603 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -51,7 +51,7 @@ import org.eclipse.jgit.util.FileUtils; * This is the classical object database representation for a Git repository, * where objects are stored loose by hashing them into directories by their * {@link org.eclipse.jgit.lib.ObjectId}, or are stored in compressed containers - * known as {@link org.eclipse.jgit.internal.storage.file.PackFile}s. + * known as {@link org.eclipse.jgit.internal.storage.file.Pack}s. *

* Optionally an object database can reference one or more alternates; other * ObjectDatabase instances that are searched in addition to the current @@ -206,7 +206,7 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override - public Collection getPacks() { + public Collection getPacks() { return packed.getPacks(); } @@ -216,7 +216,7 @@ public class ObjectDirectory extends FileObjectDatabase { * Add a single existing pack to the list of available pack files. */ @Override - public PackFile openPack(File pack) + public Pack openPack(File pack) throws IOException { final String p = pack.getName(); if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ @@ -235,7 +235,7 @@ public class ObjectDirectory extends FileObjectDatabase { } } - PackFile res = new PackFile(pack, extensions); + Pack res = new Pack(pack, extensions); packed.insert(res); return res; } @@ -509,7 +509,7 @@ public class ObjectDirectory extends FileObjectDatabase { // PackConfig) then make sure we get rid of all handles on the file. // Windows will not allow for rename otherwise. if (packFile.exists()) { - for (PackFile p : packed.getPacks()) { + for (Pack p : packed.getPacks()) { if (packFile.getPath().equals(p.getPackFile().getPath())) { p.close(); break; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index e27518690b..04d2ff8ab2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -88,7 +88,7 @@ public class ObjectDirectoryPackParser extends PackParser { private Deflater def; /** The pack that was created, if parsing was successful. */ - private PackFile newPack; + private Pack newPack; private PackConfig pconfig; @@ -129,14 +129,14 @@ public class ObjectDirectoryPackParser extends PackParser { } /** - * Get the imported {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * Get the imported {@link org.eclipse.jgit.internal.storage.file.Pack}. *

* This method is supplied only to support testing; applications shouldn't * be using it directly to access the imported data. * * @return the imported PackFile, if parsing was successful. */ - public PackFile getPackFile() { + public Pack getPack() { return newPack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java new file mode 100644 index 0000000000..d928633a73 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -0,0 +1,1208 @@ +/* + * Copyright (C) 2008-2009, Google Inc. + * Copyright (C) 2007, Robin Rosenberg + * Copyright (C) 2006-2008, Shawn O. Pearce and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; + +import java.io.EOFException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel.MapMode; +import java.nio.file.AccessDeniedException; +import java.nio.file.NoSuchFileException; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.NoPackSignatureException; +import org.eclipse.jgit.errors.PackInvalidException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; +import org.eclipse.jgit.errors.UnpackException; +import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException; +import org.eclipse.jgit.errors.UnsupportedPackVersionException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.BinaryDelta; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackOutputStream; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.LongList; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.RawParseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Git version 2 pack file representation. A pack file contains Git objects in + * delta packed format yielding high compression of lots of object where some + * objects are similar. + */ +public class Pack implements Iterable { + private static final Logger LOG = LoggerFactory.getLogger(Pack.class); + + /** + * Sorts PackFiles to be most recently created to least recently created. + */ + public static final Comparator SORT = (a, b) -> b.packLastModified + .compareTo(a.packLastModified); + + private final File packFile; + + private final int extensions; + + private File keepFile; + + private volatile String packName; + + final int hash; + + private RandomAccessFile fd; + + /** Serializes reads performed against {@link #fd}. */ + private final Object readLock = new Object(); + + long length; + + private int activeWindows; + + private int activeCopyRawData; + + Instant packLastModified; + + private PackFileSnapshot fileSnapshot; + + private volatile boolean invalid; + + private volatile Exception invalidatingCause; + + private boolean invalidBitmap; + + private AtomicInteger transientErrorCount = new AtomicInteger(); + + private byte[] packChecksum; + + private volatile PackIndex loadedIdx; + + private PackReverseIndex reverseIdx; + + private PackBitmapIndex bitmapIdx; + + /** + * Objects we have tried to read, and discovered to be corrupt. + *

+ * The list is allocated after the first corruption is found, and filled in + * as more entries are discovered. Typically this list is never used, as + * pack files do not usually contain corrupt objects. + */ + private volatile LongList corruptObjects; + + /** + * Construct a reader for an existing, pre-indexed packfile. + * + * @param packFile + * path of the .pack file holding the data. + * @param extensions + * additional pack file extensions with the same base as the pack + */ + public Pack(File packFile, int extensions) { + this.packFile = packFile; + this.fileSnapshot = PackFileSnapshot.save(packFile); + this.packLastModified = fileSnapshot.lastModifiedInstant(); + this.extensions = extensions; + + // Multiply by 31 here so we can more directly combine with another + // value in WindowCache.hash(), without doing the multiply there. + // + hash = System.identityHashCode(this) * 31; + length = Long.MAX_VALUE; + } + + private PackIndex idx() throws IOException { + PackIndex idx = loadedIdx; + if (idx == null) { + synchronized (this) { + idx = loadedIdx; + if (idx == null) { + if (invalid) { + throw new PackInvalidException(packFile, invalidatingCause); + } + try { + long start = System.currentTimeMillis(); + idx = PackIndex.open(extFile(INDEX)); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + extFile(INDEX).getAbsolutePath(), + Float.valueOf(extFile(INDEX).length() + / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() + - start))); + } + + if (packChecksum == null) { + packChecksum = idx.packChecksum; + fileSnapshot.setChecksum( + ObjectId.fromRaw(packChecksum)); + } else if (!Arrays.equals(packChecksum, + idx.packChecksum)) { + throw new PackMismatchException(MessageFormat + .format(JGitText.get().packChecksumMismatch, + packFile.getPath(), + ObjectId.fromRaw(packChecksum) + .name(), + ObjectId.fromRaw(idx.packChecksum) + .name())); + } + loadedIdx = idx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + throw e; + } catch (IOException e) { + invalid = true; + invalidatingCause = e; + throw e; + } + } + } + } + return idx; + } + /** + * Get the File object which locates this pack on disk. + * + * @return the File object which locates this pack on disk. + */ + public File getPackFile() { + return packFile; + } + + /** + * Get the index for this pack file. + * + * @return the index for this pack file. + * @throws java.io.IOException + */ + public PackIndex getIndex() throws IOException { + return idx(); + } + + /** + * Get name extracted from {@code pack-*.pack} pattern. + * + * @return name extracted from {@code pack-*.pack} pattern. + */ + public String getPackName() { + String name = packName; + if (name == null) { + name = getPackFile().getName(); + if (name.startsWith("pack-")) //$NON-NLS-1$ + name = name.substring("pack-".length()); //$NON-NLS-1$ + if (name.endsWith(".pack")) //$NON-NLS-1$ + name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$ + packName = name; + } + return name; + } + + /** + * Determine if an object is contained within the pack file. + *

+ * For performance reasons only the index file is searched; the main pack + * content is ignored entirely. + *

+ * + * @param id + * the object to look for. Must not be null. + * @return true if the object is in this pack; false otherwise. + * @throws java.io.IOException + * the index file cannot be loaded into memory. + */ + public boolean hasObject(AnyObjectId id) throws IOException { + final long offset = idx().findOffset(id); + return 0 < offset && !isCorrupt(offset); + } + + /** + * Determines whether a .keep file exists for this pack file. + * + * @return true if a .keep file exist. + */ + public boolean shouldBeKept() { + if (keepFile == null) + keepFile = extFile(KEEP); + return keepFile.exists(); + } + + /** + * Get an object from this pack. + * + * @param curs + * temporary working space associated with the calling thread. + * @param id + * the object to obtain from the pack. Must not be null. + * @return the object loader for the requested object if it is contained in + * this pack; null if the object was not found. + * @throws IOException + * the pack file or the index could not be read. + */ + ObjectLoader get(WindowCursor curs, AnyObjectId id) + throws IOException { + final long offset = idx().findOffset(id); + return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null; + } + + void resolve(Set matches, AbbreviatedObjectId id, int matchLimit) + throws IOException { + idx().resolve(matches, id, matchLimit); + } + + /** + * Close the resources utilized by this repository + */ + public void close() { + WindowCache.purge(this); + synchronized (this) { + loadedIdx = null; + reverseIdx = null; + } + } + + /** + * {@inheritDoc} + *

+ * Provide iterator over entries in associated pack index, that should also + * exist in this pack file. Objects returned by such iterator are mutable + * during iteration. + *

+ * Iterator returns objects in SHA-1 lexicographical order. + *

+ * + * @see PackIndex#iterator() + */ + @Override + public Iterator iterator() { + try { + return idx().iterator(); + } catch (IOException e) { + return Collections. emptyList().iterator(); + } + } + + /** + * Obtain the total number of objects available in this pack. This method + * relies on pack index, giving number of effectively available objects. + * + * @return number of objects in index of this pack, likewise in this pack + * @throws IOException + * the index file cannot be loaded into memory. + */ + long getObjectCount() throws IOException { + return idx().getObjectCount(); + } + + /** + * Search for object id with the specified start offset in associated pack + * (reverse) index. + * + * @param offset + * start offset of object to find + * @return object id for this offset, or null if no object was found + * @throws IOException + * the index file cannot be loaded into memory. + */ + ObjectId findObjectForOffset(long offset) throws IOException { + return getReverseIdx().findObject(offset); + } + + /** + * Return the @{@link FileSnapshot} associated to the underlying packfile + * that has been used when the object was created. + * + * @return the packfile @{@link FileSnapshot} that the object is loaded from. + */ + PackFileSnapshot getFileSnapshot() { + return fileSnapshot; + } + + AnyObjectId getPackChecksum() { + return ObjectId.fromRaw(packChecksum); + } + + private final byte[] decompress(final long position, final int sz, + final WindowCursor curs) throws IOException, DataFormatException { + byte[] dstbuf; + try { + dstbuf = new byte[sz]; + } catch (OutOfMemoryError noMemory) { + // The size may be larger than our heap allows, return null to + // let the caller know allocation isn't possible and it should + // use the large object streaming approach instead. + // + // For example, this can occur when sz is 640 MB, and JRE + // maximum heap size is only 256 MB. Even if the JRE has + // 200 MB free, it cannot allocate a 640 MB byte array. + return null; + } + + if (curs.inflate(this, position, dstbuf, false) != sz) + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + Long.valueOf(position))); + return dstbuf; + } + + void copyPackAsIs(PackOutputStream out, WindowCursor curs) + throws IOException { + // Pin the first window, this ensures the length is accurate. + curs.pin(this, 0); + curs.copyPackAsIs(this, length, out); + } + + final void copyAsIs(PackOutputStream out, LocalObjectToPack src, + boolean validate, WindowCursor curs) throws IOException, + StoredObjectRepresentationNotAvailableException { + beginCopyAsIs(src); + try { + copyAsIs2(out, src, validate, curs); + } finally { + endCopyAsIs(); + } + } + + private void copyAsIs2(PackOutputStream out, LocalObjectToPack src, + boolean validate, WindowCursor curs) throws IOException, + StoredObjectRepresentationNotAvailableException { + final CRC32 crc1 = validate ? new CRC32() : null; + final CRC32 crc2 = validate ? new CRC32() : null; + final byte[] buf = out.getCopyBuffer(); + + // Rip apart the header so we can discover the size. + // + readFully(src.offset, buf, 0, 20, curs); + int c = buf[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long inflatedLength = c & 15; + int shift = 4; + int headerCnt = 1; + while ((c & 0x80) != 0) { + c = buf[headerCnt++] & 0xff; + inflatedLength += ((long) (c & 0x7f)) << shift; + shift += 7; + } + + if (typeCode == Constants.OBJ_OFS_DELTA) { + do { + c = buf[headerCnt++] & 0xff; + } while ((c & 128) != 0); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + } else if (typeCode == Constants.OBJ_REF_DELTA) { + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + + readFully(src.offset + headerCnt, buf, 0, 20, curs); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, 20); + crc2.update(buf, 0, 20); + } + headerCnt += 20; + } else if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + + final long dataOffset = src.offset + headerCnt; + final long dataLength = src.length; + final long expectedCRC; + final ByteArrayWindow quickCopy; + + // Verify the object isn't corrupt before sending. If it is, + // we report it missing instead. + // + try { + quickCopy = curs.quickCopy(this, dataOffset, dataLength); + + if (validate && idx().hasCRC32Support()) { + assert(crc1 != null); + // Index has the CRC32 code cached, validate the object. + // + expectedCRC = idx().findCRC32(src); + if (quickCopy != null) { + quickCopy.crc32(crc1, dataOffset, (int) dataLength); + } else { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + pos += n; + cnt -= n; + } + } + if (crc1.getValue() != expectedCRC) { + setCorrupt(src.offset); + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } + } else if (validate) { + // We don't have a CRC32 code in the index, so compute it + // now while inflating the raw data to get zlib to tell us + // whether or not the data is safe. + // + Inflater inf = curs.inflater(); + byte[] tmp = new byte[1024]; + if (quickCopy != null) { + quickCopy.check(inf, tmp, dataOffset, (int) dataLength); + } else { + assert(crc1 != null); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + inf.setInput(buf, 0, n); + while (inf.inflate(tmp, 0, tmp.length) > 0) + continue; + pos += n; + cnt -= n; + } + } + if (!inf.finished() || inf.getBytesRead() != dataLength) { + setCorrupt(src.offset); + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + Long.valueOf(src.offset))); + } + assert(crc1 != null); + expectedCRC = crc1.getValue(); + } else { + expectedCRC = -1; + } + } catch (DataFormatException dataFormat) { + setCorrupt(src.offset); + + CorruptObjectException corruptObject = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile()), + dataFormat); + + throw new StoredObjectRepresentationNotAvailableException(src, + corruptObject); + + } catch (IOException ioError) { + throw new StoredObjectRepresentationNotAvailableException(src, + ioError); + } + + if (quickCopy != null) { + // The entire object fits into a single byte array window slice, + // and we have it pinned. Write this out without copying. + // + out.writeHeader(src, inflatedLength); + quickCopy.write(out, dataOffset, (int) dataLength); + + } else if (dataLength <= buf.length) { + // Tiny optimization: Lots of objects are very small deltas or + // deflated commits that are likely to fit in the copy buffer. + // + if (!validate) { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + pos += n; + cnt -= n; + } + } + out.writeHeader(src, inflatedLength); + out.write(buf, 0, (int) dataLength); + } else { + // Now we are committed to sending the object. As we spool it out, + // check its CRC32 code to make sure there wasn't corruption between + // the verification we did above, and us actually outputting it. + // + out.writeHeader(src, inflatedLength); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + if (validate) { + assert(crc2 != null); + crc2.update(buf, 0, n); + } + out.write(buf, 0, n); + pos += n; + cnt -= n; + } + if (validate) { + assert(crc2 != null); + if (crc2.getValue() != expectedCRC) { + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } + } + } + } + + boolean invalid() { + return invalid; + } + + void setInvalid() { + invalid = true; + } + + int incrementTransientErrorCount() { + return transientErrorCount.incrementAndGet(); + } + + void resetTransientErrorCount() { + transientErrorCount.set(0); + } + + private void readFully(final long position, final byte[] dstbuf, + int dstoff, final int cnt, final WindowCursor curs) + throws IOException { + if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt) + throw new EOFException(); + } + + private synchronized void beginCopyAsIs(ObjectToPack otp) + throws StoredObjectRepresentationNotAvailableException { + if (++activeCopyRawData == 1 && activeWindows == 0) { + try { + doOpen(); + } catch (IOException thisPackNotValid) { + throw new StoredObjectRepresentationNotAvailableException(otp, + thisPackNotValid); + } + } + } + + private synchronized void endCopyAsIs() { + if (--activeCopyRawData == 0 && activeWindows == 0) + doClose(); + } + + synchronized boolean beginWindowCache() throws IOException { + if (++activeWindows == 1) { + if (activeCopyRawData == 0) + doOpen(); + return true; + } + return false; + } + + synchronized boolean endWindowCache() { + final boolean r = --activeWindows == 0; + if (r && activeCopyRawData == 0) + doClose(); + return r; + } + + private void doOpen() throws IOException { + if (invalid) { + openFail(true, invalidatingCause); + throw new PackInvalidException(packFile, invalidatingCause); + } + try { + synchronized (readLock) { + fd = new RandomAccessFile(packFile, "r"); //$NON-NLS-1$ + length = fd.length(); + onOpenPack(); + } + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from another thread + openFail(false, e); + throw e; + } catch (FileNotFoundException fn) { + // don't invalidate the pack if opening an existing file failed + // since it may be related to a temporary lack of resources (e.g. + // max open files) + openFail(!packFile.exists(), fn); + throw fn; + } catch (EOFException | AccessDeniedException | NoSuchFileException + | CorruptObjectException | NoPackSignatureException + | PackMismatchException | UnpackException + | UnsupportedPackIndexVersionException + | UnsupportedPackVersionException pe) { + // exceptions signaling permanent problems with a pack + openFail(true, pe); + throw pe; + } catch (IOException | RuntimeException ge) { + // generic exceptions could be transient so we should not mark the + // pack invalid to avoid false MissingObjectExceptions + openFail(false, ge); + throw ge; + } + } + + private void openFail(boolean invalidate, Exception cause) { + activeWindows = 0; + activeCopyRawData = 0; + invalid = invalidate; + invalidatingCause = cause; + doClose(); + } + + private void doClose() { + synchronized (readLock) { + if (fd != null) { + try { + fd.close(); + } catch (IOException err) { + // Ignore a close event. We had it open only for reading. + // There should not be errors related to network buffers + // not flushed, etc. + } + fd = null; + } + } + } + + ByteArrayWindow read(long pos, int size) throws IOException { + synchronized (readLock) { + if (invalid || fd == null) { + // Due to concurrency between a read and another packfile invalidation thread + // one thread could come up to this point and then fail with NPE. + // Detect the situation and throw a proper exception so that can be properly + // managed by the main packfile search loop and the Git client won't receive + // any failures. + throw new PackInvalidException(packFile, invalidatingCause); + } + if (length < pos + size) + size = (int) (length - pos); + final byte[] buf = new byte[size]; + fd.seek(pos); + fd.readFully(buf, 0, size); + return new ByteArrayWindow(this, pos, buf); + } + } + + ByteWindow mmap(long pos, int size) throws IOException { + synchronized (readLock) { + if (length < pos + size) + size = (int) (length - pos); + + MappedByteBuffer map; + try { + map = fd.getChannel().map(MapMode.READ_ONLY, pos, size); + } catch (IOException ioe1) { + // The most likely reason this failed is the JVM has run out + // of virtual memory. We need to discard quickly, and try to + // force the GC to finalize and release any existing mappings. + // + System.gc(); + System.runFinalization(); + map = fd.getChannel().map(MapMode.READ_ONLY, pos, size); + } + + if (map.hasArray()) + return new ByteArrayWindow(this, pos, map.array()); + return new ByteBufferWindow(this, pos, map); + } + } + + private void onOpenPack() throws IOException { + final PackIndex idx = idx(); + final byte[] buf = new byte[20]; + + fd.seek(0); + fd.readFully(buf, 0, 12); + if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) { + throw new NoPackSignatureException(JGitText.get().notAPACKFile); + } + final long vers = NB.decodeUInt32(buf, 4); + final long packCnt = NB.decodeUInt32(buf, 8); + if (vers != 2 && vers != 3) { + throw new UnsupportedPackVersionException(vers); + } + + if (packCnt != idx.getObjectCount()) { + throw new PackMismatchException(MessageFormat.format( + JGitText.get().packObjectCountMismatch, + Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()), + getPackFile())); + } + + fd.seek(length - 20); + fd.readFully(buf, 0, 20); + if (!Arrays.equals(buf, packChecksum)) { + throw new PackMismatchException(MessageFormat.format( + JGitText.get().packChecksumMismatch, + getPackFile(), + ObjectId.fromRaw(buf).name(), + ObjectId.fromRaw(idx.packChecksum).name())); + } + } + + ObjectLoader load(WindowCursor curs, long pos) + throws IOException, LargeObjectException { + try { + final byte[] ib = curs.tempId; + Delta delta = null; + byte[] data = null; + int type = Constants.OBJ_BAD; + boolean cached = false; + + SEARCH: for (;;) { + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long sz = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = ib[p++] & 0xff; + sz += ((long) (c & 0x7f)) << shift; + shift += 7; + } + + switch (typeCode) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: { + if (delta != null || sz < curs.getStreamFileThreshold()) { + data = decompress(pos + p, (int) sz, curs); + } + + if (delta != null) { + type = typeCode; + break SEARCH; + } + + if (data != null) { + return new ObjectLoader.SmallObject(typeCode, data); + } + return new LargePackedWholeObject(typeCode, sz, pos, p, + this, curs.db); + } + + case Constants.OBJ_OFS_DELTA: { + c = ib[p++] & 0xff; + long base = c & 127; + while ((c & 128) != 0) { + base += 1; + c = ib[p++] & 0xff; + base <<= 7; + base += (c & 127); + } + base = pos - base; + delta = new Delta(delta, pos, (int) sz, p, base); + if (sz != delta.deltaSize) + break SEARCH; + + DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); + if (e != null) { + type = e.type; + data = e.data; + cached = true; + break SEARCH; + } + pos = base; + continue SEARCH; + } + + case Constants.OBJ_REF_DELTA: { + readFully(pos + p, ib, 0, 20, curs); + long base = findDeltaBase(ObjectId.fromRaw(ib)); + delta = new Delta(delta, pos, (int) sz, p + 20, base); + if (sz != delta.deltaSize) + break SEARCH; + + DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); + if (e != null) { + type = e.type; + data = e.data; + cached = true; + break SEARCH; + } + pos = base; + continue SEARCH; + } + + default: + throw new IOException(MessageFormat.format( + JGitText.get().unknownObjectType, + Integer.valueOf(typeCode))); + } + } + + // At this point there is at least one delta to apply to data. + // (Whole objects with no deltas to apply return early above.) + + if (data == null) + throw new IOException(JGitText.get().inMemoryBufferLimitExceeded); + + assert(delta != null); + do { + // Cache only the base immediately before desired object. + if (cached) + cached = false; + else if (delta.next == null) + curs.getDeltaBaseCache().store(this, delta.basePos, data, type); + + pos = delta.deltaPos; + + final byte[] cmds = decompress(pos + delta.hdrLen, + delta.deltaSize, curs); + if (cmds == null) { + data = null; // Discard base in case of OutOfMemoryError + throw new LargeObjectException.OutOfMemory(new OutOfMemoryError()); + } + + final long sz = BinaryDelta.getResultSize(cmds); + if (Integer.MAX_VALUE <= sz) + throw new LargeObjectException.ExceedsByteArrayLimit(); + + final byte[] result; + try { + result = new byte[(int) sz]; + } catch (OutOfMemoryError tooBig) { + data = null; // Discard base in case of OutOfMemoryError + throw new LargeObjectException.OutOfMemory(tooBig); + } + + BinaryDelta.apply(data, cmds, result); + data = result; + delta = delta.next; + } while (delta != null); + + return new ObjectLoader.SmallObject(type, data); + + } catch (DataFormatException dfe) { + throw new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(pos), getPackFile()), + dfe); + } + } + + private long findDeltaBase(ObjectId baseId) throws IOException, + MissingObjectException { + long ofs = idx().findOffset(baseId); + if (ofs < 0) + throw new MissingObjectException(baseId, + JGitText.get().missingDeltaBase); + return ofs; + } + + private static class Delta { + /** Child that applies onto this object. */ + final Delta next; + + /** Offset of the delta object. */ + final long deltaPos; + + /** Size of the inflated delta stream. */ + final int deltaSize; + + /** Total size of the delta's pack entry header (including base). */ + final int hdrLen; + + /** Offset of the base object this delta applies onto. */ + final long basePos; + + Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) { + this.next = next; + this.deltaPos = ofs; + this.deltaSize = sz; + this.hdrLen = hdrLen; + this.basePos = baseOffset; + } + } + + byte[] getDeltaHeader(WindowCursor wc, long pos) + throws IOException, DataFormatException { + // The delta stream starts as two variable length integers. If we + // assume they are 64 bits each, we need 16 bytes to encode them, + // plus 2 extra bytes for the variable length overhead. So 18 is + // the longest delta instruction header. + // + final byte[] hdr = new byte[18]; + wc.inflate(this, pos, hdr, true /* headerOnly */); + return hdr; + } + + int getObjectType(WindowCursor curs, long pos) throws IOException { + final byte[] ib = curs.tempId; + for (;;) { + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + final int type = (c >> 4) & 7; + + switch (type) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + return type; + + case Constants.OBJ_OFS_DELTA: { + int p = 1; + while ((c & 0x80) != 0) + c = ib[p++] & 0xff; + c = ib[p++] & 0xff; + long ofs = c & 127; + while ((c & 128) != 0) { + ofs += 1; + c = ib[p++] & 0xff; + ofs <<= 7; + ofs += (c & 127); + } + pos = pos - ofs; + continue; + } + + case Constants.OBJ_REF_DELTA: { + int p = 1; + while ((c & 0x80) != 0) + c = ib[p++] & 0xff; + readFully(pos + p, ib, 0, 20, curs); + pos = findDeltaBase(ObjectId.fromRaw(ib)); + continue; + } + + default: + throw new IOException( + MessageFormat.format(JGitText.get().unknownObjectType, + Integer.valueOf(type))); + } + } + } + + long getObjectSize(WindowCursor curs, AnyObjectId id) + throws IOException { + final long offset = idx().findOffset(id); + return 0 < offset ? getObjectSize(curs, offset) : -1; + } + + long getObjectSize(WindowCursor curs, long pos) + throws IOException { + final byte[] ib = curs.tempId; + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + final int type = (c >> 4) & 7; + long sz = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = ib[p++] & 0xff; + sz += ((long) (c & 0x7f)) << shift; + shift += 7; + } + + long deltaAt; + switch (type) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + return sz; + + case Constants.OBJ_OFS_DELTA: + c = ib[p++] & 0xff; + while ((c & 128) != 0) + c = ib[p++] & 0xff; + deltaAt = pos + p; + break; + + case Constants.OBJ_REF_DELTA: + deltaAt = pos + p + 20; + break; + + default: + throw new IOException(MessageFormat.format( + JGitText.get().unknownObjectType, Integer.valueOf(type))); + } + + try { + return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt)); + } catch (DataFormatException e) { + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos), + getPackFile()), e); + } + } + + LocalObjectRepresentation representation(final WindowCursor curs, + final AnyObjectId objectId) throws IOException { + final long pos = idx().findOffset(objectId); + if (pos < 0) + return null; + + final byte[] ib = curs.tempId; + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + int p = 1; + final int typeCode = (c >> 4) & 7; + while ((c & 0x80) != 0) + c = ib[p++] & 0xff; + + long len = (findEndOffset(pos) - pos); + switch (typeCode) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + return LocalObjectRepresentation.newWhole(this, pos, len - p); + + case Constants.OBJ_OFS_DELTA: { + c = ib[p++] & 0xff; + long ofs = c & 127; + while ((c & 128) != 0) { + ofs += 1; + c = ib[p++] & 0xff; + ofs <<= 7; + ofs += (c & 127); + } + ofs = pos - ofs; + return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs); + } + + case Constants.OBJ_REF_DELTA: { + len -= p; + len -= Constants.OBJECT_ID_LENGTH; + readFully(pos + p, ib, 0, 20, curs); + ObjectId id = ObjectId.fromRaw(ib); + return LocalObjectRepresentation.newDelta(this, pos, len, id); + } + + default: + throw new IOException( + MessageFormat.format(JGitText.get().unknownObjectType, + Integer.valueOf(typeCode))); + } + } + + private long findEndOffset(long startOffset) + throws IOException, CorruptObjectException { + final long maxOffset = length - 20; + return getReverseIdx().findNextOffset(startOffset, maxOffset); + } + + synchronized PackBitmapIndex getBitmapIndex() throws IOException { + if (invalid || invalidBitmap) + return null; + if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { + final PackBitmapIndex idx; + try { + idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(), + getReverseIdx()); + } catch (FileNotFoundException e) { + // Once upon a time this bitmap file existed. Now it + // has been removed. Most likely an external gc has + // removed this packfile and the bitmap + invalidBitmap = true; + return null; + } + + // At this point, idx() will have set packChecksum. + if (Arrays.equals(packChecksum, idx.packChecksum)) + bitmapIdx = idx; + else + invalidBitmap = true; + } + return bitmapIdx; + } + + private synchronized PackReverseIndex getReverseIdx() throws IOException { + if (reverseIdx == null) + reverseIdx = new PackReverseIndex(idx()); + return reverseIdx; + } + + private boolean isCorrupt(long offset) { + LongList list = corruptObjects; + if (list == null) + return false; + synchronized (list) { + return list.contains(offset); + } + } + + private void setCorrupt(long offset) { + LongList list = corruptObjects; + if (list == null) { + synchronized (readLock) { + list = corruptObjects; + if (list == null) { + list = new LongList(); + corruptObjects = list; + } + } + } + synchronized (list) { + list.add(offset); + } + } + + private File extFile(PackExt ext) { + String p = packFile.getName(); + int dot = p.lastIndexOf('.'); + String b = (dot < 0) ? p : p.substring(0, dot); + return new File(packFile.getParentFile(), b + '.' + ext.getExtension()); + } + + private boolean hasExt(PackExt ext) { + return (extensions & ext.getBit()) != 0; + } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "Pack [packFileName=" + packFile.getName() + ", length=" + + packFile.length() + ", packChecksum=" + + ObjectId.fromRaw(packChecksum).name() + "]"; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index fd9da7c6c4..b2ba36bf91 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -47,16 +47,17 @@ import org.slf4j.LoggerFactory; /** * Traditional file system packed objects directory handler. *

- * This is the {@code PackFile}s object representation for a Git object - * database, where objects are stored in compressed containers known as - * {@link org.eclipse.jgit.internal.storage.file.PackFile}s. + * This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object + * representation for a Git object database, where objects are stored in + * compressed containers known as + * {@link org.eclipse.jgit.internal.storage.file.Pack}s. */ class PackDirectory { private final static Logger LOG = LoggerFactory .getLogger(PackDirectory.class); private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, - new PackFile[0]); + new Pack[0]); private final Config config; @@ -94,18 +95,18 @@ class PackDirectory { void close() { PackList packs = packList.get(); if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (PackFile p : packs.packs) { + for (Pack p : packs.packs) { p.close(); } } } - Collection getPacks() { + Collection getPacks() { PackList list = packList.get(); if (list == NO_PACKS) { list = scanPacks(list); } - PackFile[] packs = list.packs; + Pack[] packs = list.packs; return Collections.unmodifiableCollection(Arrays.asList(packs)); } @@ -126,7 +127,7 @@ class PackDirectory { PackList pList; do { pList = packList.get(); - for (PackFile p : pList.packs) { + for (Pack p : pList.packs) { try { if (p.hasObject(objectId)) { return true; @@ -167,7 +168,7 @@ class PackDirectory { PackList pList; do { pList = packList.get(); - for (PackFile p : pList.packs) { + for (Pack p : pList.packs) { try { p.resolve(matches, id, matchLimit); p.resetTransientErrorCount(); @@ -187,7 +188,7 @@ class PackDirectory { do { SEARCH: for (;;) { pList = packList.get(); - for (PackFile p : pList.packs) { + for (Pack p : pList.packs) { try { ObjectLoader ldr = p.get(curs, objectId); p.resetTransientErrorCount(); @@ -213,7 +214,7 @@ class PackDirectory { do { SEARCH: for (;;) { pList = packList.get(); - for (PackFile p : pList.packs) { + for (Pack p : pList.packs) { try { long len = p.getObjectSize(curs, id); p.resetTransientErrorCount(); @@ -239,7 +240,7 @@ class PackDirectory { WindowCursor curs) { PackList pList = packList.get(); SEARCH: for (;;) { - for (PackFile p : pList.packs) { + for (Pack p : pList.packs) { try { LocalObjectRepresentation rep = p.representation(curs, otp); p.resetTransientErrorCount(); @@ -259,7 +260,7 @@ class PackDirectory { } } - private void handlePackError(IOException e, PackFile p) { + private void handlePackError(IOException e, Pack p) { String warnTmpl = null; int transientErrorCount = 0; String errTmpl = JGitText.get().exceptionWhileReadingPack; @@ -322,7 +323,7 @@ class PackDirectory { && old != scanPacks(old); } - void insert(PackFile pf) { + void insert(Pack pack) { PackList o, n; do { o = packList.get(); @@ -331,33 +332,33 @@ class PackDirectory { // (picked up by a concurrent thread that did a scan?) we // do not want to insert it a second time. // - final PackFile[] oldList = o.packs; - final String name = pf.getPackFile().getName(); - for (PackFile p : oldList) { + final Pack[] oldList = o.packs; + final String name = pack.getPackFile().getName(); + for (Pack p : oldList) { if (name.equals(p.getPackFile().getName())) { return; } } - final PackFile[] newList = new PackFile[1 + oldList.length]; - newList[0] = pf; + final Pack[] newList = new Pack[1 + oldList.length]; + newList[0] = pack; System.arraycopy(oldList, 0, newList, 1, oldList.length); n = new PackList(o.snapshot, newList); } while (!packList.compareAndSet(o, n)); } - private void remove(PackFile deadPack) { + private void remove(Pack deadPack) { PackList o, n; do { o = packList.get(); - final PackFile[] oldList = o.packs; + final Pack[] oldList = o.packs; final int j = indexOf(oldList, deadPack); if (j < 0) { break; } - final PackFile[] newList = new PackFile[oldList.length - 1]; + final Pack[] newList = new Pack[oldList.length - 1]; System.arraycopy(oldList, 0, newList, 0, j); System.arraycopy(oldList, j + 1, newList, j, newList.length - j); n = new PackList(o.snapshot, newList); @@ -365,7 +366,7 @@ class PackDirectory { deadPack.close(); } - private static int indexOf(PackFile[] list, PackFile pack) { + private static int indexOf(Pack[] list, Pack pack) { for (int i = 0; i < list.length; i++) { if (list[i] == pack) { return i; @@ -395,10 +396,10 @@ class PackDirectory { } private PackList scanPacksImpl(PackList old) { - final Map forReuse = reuseMap(old); + final Map forReuse = reuseMap(old); final FileSnapshot snapshot = FileSnapshot.save(directory); final Set names = listPackDirectory(); - final List list = new ArrayList<>(names.size() >> 2); + final List list = new ArrayList<>(names.size() >> 2); boolean foundNew = false; for (String indexName : names) { // Must match "pack-[0-9a-f]{40}.idx" to be an index. @@ -425,7 +426,7 @@ class PackDirectory { final String packName = base + PACK.getExtension(); final File packFile = new File(directory, packName); - final PackFile oldPack = forReuse.get(packName); + final Pack oldPack = forReuse.get(packName); if (oldPack != null && !oldPack.getFileSnapshot().isModified(packFile)) { forReuse.remove(packName); @@ -433,7 +434,7 @@ class PackDirectory { continue; } - list.add(new PackFile(packFile, extensions)); + list.add(new Pack(packFile, extensions)); foundNew = true; } @@ -447,7 +448,7 @@ class PackDirectory { return old; } - for (PackFile p : forReuse.values()) { + for (Pack p : forReuse.values()) { p.close(); } @@ -455,14 +456,14 @@ class PackDirectory { return new PackList(snapshot, NO_PACKS.packs); } - final PackFile[] r = list.toArray(new PackFile[0]); - Arrays.sort(r, PackFile.SORT); + final Pack[] r = list.toArray(new Pack[0]); + Arrays.sort(r, Pack.SORT); return new PackList(snapshot, r); } - private static Map reuseMap(PackList old) { - final Map forReuse = new HashMap<>(); - for (PackFile p : old.packs) { + private static Map reuseMap(PackList old) { + final Map forReuse = new HashMap<>(); + for (Pack p : old.packs) { if (p.invalid()) { // The pack instance is corrupted, and cannot be safely used // again. Do not include it in our reuse map. @@ -471,7 +472,7 @@ class PackDirectory { continue; } - final PackFile prior = forReuse.put(p.getPackFile().getName(), p); + final Pack prior = forReuse.put(p.getPackFile().getName(), p); if (prior != null) { // This should never occur. It should be impossible for us // to have two pack files with the same name, as all of them @@ -504,10 +505,10 @@ class PackDirectory { /** State just before reading the pack directory. */ final FileSnapshot snapshot; - /** All known packs, sorted by {@link PackFile#SORT}. */ - final PackFile[] packs; + /** All known packs, sorted by {@link Pack#SORT}. */ + final Pack[] packs; - PackList(FileSnapshot monitor, PackFile[] packs) { + PackList(FileSnapshot monitor, Pack[] packs) { this.snapshot = monitor; this.packs = packs; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java deleted file mode 100644 index e112fe7444..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ /dev/null @@ -1,1208 +0,0 @@ -/* - * Copyright (C) 2008-2009, Google Inc. - * Copyright (C) 2007, Robin Rosenberg - * Copyright (C) 2006-2008, Shawn O. Pearce and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; -import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; - -import java.io.EOFException; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.RandomAccessFile; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel.MapMode; -import java.nio.file.AccessDeniedException; -import java.nio.file.NoSuchFileException; -import java.text.MessageFormat; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.CRC32; -import java.util.zip.DataFormatException; -import java.util.zip.Inflater; - -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.LargeObjectException; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.errors.NoPackSignatureException; -import org.eclipse.jgit.errors.PackInvalidException; -import org.eclipse.jgit.errors.PackMismatchException; -import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; -import org.eclipse.jgit.errors.UnpackException; -import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException; -import org.eclipse.jgit.errors.UnsupportedPackVersionException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.pack.BinaryDelta; -import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.pack.PackOutputStream; -import org.eclipse.jgit.lib.AbbreviatedObjectId; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectLoader; -import org.eclipse.jgit.util.LongList; -import org.eclipse.jgit.util.NB; -import org.eclipse.jgit.util.RawParseUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Git version 2 pack file representation. A pack file contains Git objects in - * delta packed format yielding high compression of lots of object where some - * objects are similar. - */ -public class PackFile implements Iterable { - private static final Logger LOG = LoggerFactory.getLogger(PackFile.class); - - /** - * Sorts PackFiles to be most recently created to least recently created. - */ - public static final Comparator SORT = (a, b) -> b.packLastModified - .compareTo(a.packLastModified); - - private final File packFile; - - private final int extensions; - - private File keepFile; - - private volatile String packName; - - final int hash; - - private RandomAccessFile fd; - - /** Serializes reads performed against {@link #fd}. */ - private final Object readLock = new Object(); - - long length; - - private int activeWindows; - - private int activeCopyRawData; - - Instant packLastModified; - - private PackFileSnapshot fileSnapshot; - - private volatile boolean invalid; - - private volatile Exception invalidatingCause; - - private boolean invalidBitmap; - - private AtomicInteger transientErrorCount = new AtomicInteger(); - - private byte[] packChecksum; - - private volatile PackIndex loadedIdx; - - private PackReverseIndex reverseIdx; - - private PackBitmapIndex bitmapIdx; - - /** - * Objects we have tried to read, and discovered to be corrupt. - *

- * The list is allocated after the first corruption is found, and filled in - * as more entries are discovered. Typically this list is never used, as - * pack files do not usually contain corrupt objects. - */ - private volatile LongList corruptObjects; - - /** - * Construct a reader for an existing, pre-indexed packfile. - * - * @param packFile - * path of the .pack file holding the data. - * @param extensions - * additional pack file extensions with the same base as the pack - */ - public PackFile(File packFile, int extensions) { - this.packFile = packFile; - this.fileSnapshot = PackFileSnapshot.save(packFile); - this.packLastModified = fileSnapshot.lastModifiedInstant(); - this.extensions = extensions; - - // Multiply by 31 here so we can more directly combine with another - // value in WindowCache.hash(), without doing the multiply there. - // - hash = System.identityHashCode(this) * 31; - length = Long.MAX_VALUE; - } - - private PackIndex idx() throws IOException { - PackIndex idx = loadedIdx; - if (idx == null) { - synchronized (this) { - idx = loadedIdx; - if (idx == null) { - if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); - } - try { - long start = System.currentTimeMillis(); - idx = PackIndex.open(extFile(INDEX)); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format( - "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - extFile(INDEX).getAbsolutePath(), - Float.valueOf(extFile(INDEX).length() - / (1024f * 1024)), - Long.valueOf(System.currentTimeMillis() - - start))); - } - - if (packChecksum == null) { - packChecksum = idx.packChecksum; - fileSnapshot.setChecksum( - ObjectId.fromRaw(packChecksum)); - } else if (!Arrays.equals(packChecksum, - idx.packChecksum)) { - throw new PackMismatchException(MessageFormat - .format(JGitText.get().packChecksumMismatch, - packFile.getPath(), - ObjectId.fromRaw(packChecksum) - .name(), - ObjectId.fromRaw(idx.packChecksum) - .name())); - } - loadedIdx = idx; - } catch (InterruptedIOException e) { - // don't invalidate the pack, we are interrupted from - // another thread - throw e; - } catch (IOException e) { - invalid = true; - invalidatingCause = e; - throw e; - } - } - } - } - return idx; - } - /** - * Get the File object which locates this pack on disk. - * - * @return the File object which locates this pack on disk. - */ - public File getPackFile() { - return packFile; - } - - /** - * Get the index for this pack file. - * - * @return the index for this pack file. - * @throws java.io.IOException - */ - public PackIndex getIndex() throws IOException { - return idx(); - } - - /** - * Get name extracted from {@code pack-*.pack} pattern. - * - * @return name extracted from {@code pack-*.pack} pattern. - */ - public String getPackName() { - String name = packName; - if (name == null) { - name = getPackFile().getName(); - if (name.startsWith("pack-")) //$NON-NLS-1$ - name = name.substring("pack-".length()); //$NON-NLS-1$ - if (name.endsWith(".pack")) //$NON-NLS-1$ - name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$ - packName = name; - } - return name; - } - - /** - * Determine if an object is contained within the pack file. - *

- * For performance reasons only the index file is searched; the main pack - * content is ignored entirely. - *

- * - * @param id - * the object to look for. Must not be null. - * @return true if the object is in this pack; false otherwise. - * @throws java.io.IOException - * the index file cannot be loaded into memory. - */ - public boolean hasObject(AnyObjectId id) throws IOException { - final long offset = idx().findOffset(id); - return 0 < offset && !isCorrupt(offset); - } - - /** - * Determines whether a .keep file exists for this pack file. - * - * @return true if a .keep file exist. - */ - public boolean shouldBeKept() { - if (keepFile == null) - keepFile = extFile(KEEP); - return keepFile.exists(); - } - - /** - * Get an object from this pack. - * - * @param curs - * temporary working space associated with the calling thread. - * @param id - * the object to obtain from the pack. Must not be null. - * @return the object loader for the requested object if it is contained in - * this pack; null if the object was not found. - * @throws IOException - * the pack file or the index could not be read. - */ - ObjectLoader get(WindowCursor curs, AnyObjectId id) - throws IOException { - final long offset = idx().findOffset(id); - return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null; - } - - void resolve(Set matches, AbbreviatedObjectId id, int matchLimit) - throws IOException { - idx().resolve(matches, id, matchLimit); - } - - /** - * Close the resources utilized by this repository - */ - public void close() { - WindowCache.purge(this); - synchronized (this) { - loadedIdx = null; - reverseIdx = null; - } - } - - /** - * {@inheritDoc} - *

- * Provide iterator over entries in associated pack index, that should also - * exist in this pack file. Objects returned by such iterator are mutable - * during iteration. - *

- * Iterator returns objects in SHA-1 lexicographical order. - *

- * - * @see PackIndex#iterator() - */ - @Override - public Iterator iterator() { - try { - return idx().iterator(); - } catch (IOException e) { - return Collections. emptyList().iterator(); - } - } - - /** - * Obtain the total number of objects available in this pack. This method - * relies on pack index, giving number of effectively available objects. - * - * @return number of objects in index of this pack, likewise in this pack - * @throws IOException - * the index file cannot be loaded into memory. - */ - long getObjectCount() throws IOException { - return idx().getObjectCount(); - } - - /** - * Search for object id with the specified start offset in associated pack - * (reverse) index. - * - * @param offset - * start offset of object to find - * @return object id for this offset, or null if no object was found - * @throws IOException - * the index file cannot be loaded into memory. - */ - ObjectId findObjectForOffset(long offset) throws IOException { - return getReverseIdx().findObject(offset); - } - - /** - * Return the @{@link FileSnapshot} associated to the underlying packfile - * that has been used when the object was created. - * - * @return the packfile @{@link FileSnapshot} that the object is loaded from. - */ - PackFileSnapshot getFileSnapshot() { - return fileSnapshot; - } - - AnyObjectId getPackChecksum() { - return ObjectId.fromRaw(packChecksum); - } - - private final byte[] decompress(final long position, final int sz, - final WindowCursor curs) throws IOException, DataFormatException { - byte[] dstbuf; - try { - dstbuf = new byte[sz]; - } catch (OutOfMemoryError noMemory) { - // The size may be larger than our heap allows, return null to - // let the caller know allocation isn't possible and it should - // use the large object streaming approach instead. - // - // For example, this can occur when sz is 640 MB, and JRE - // maximum heap size is only 256 MB. Even if the JRE has - // 200 MB free, it cannot allocate a 640 MB byte array. - return null; - } - - if (curs.inflate(this, position, dstbuf, false) != sz) - throw new EOFException(MessageFormat.format( - JGitText.get().shortCompressedStreamAt, - Long.valueOf(position))); - return dstbuf; - } - - void copyPackAsIs(PackOutputStream out, WindowCursor curs) - throws IOException { - // Pin the first window, this ensures the length is accurate. - curs.pin(this, 0); - curs.copyPackAsIs(this, length, out); - } - - final void copyAsIs(PackOutputStream out, LocalObjectToPack src, - boolean validate, WindowCursor curs) throws IOException, - StoredObjectRepresentationNotAvailableException { - beginCopyAsIs(src); - try { - copyAsIs2(out, src, validate, curs); - } finally { - endCopyAsIs(); - } - } - - private void copyAsIs2(PackOutputStream out, LocalObjectToPack src, - boolean validate, WindowCursor curs) throws IOException, - StoredObjectRepresentationNotAvailableException { - final CRC32 crc1 = validate ? new CRC32() : null; - final CRC32 crc2 = validate ? new CRC32() : null; - final byte[] buf = out.getCopyBuffer(); - - // Rip apart the header so we can discover the size. - // - readFully(src.offset, buf, 0, 20, curs); - int c = buf[0] & 0xff; - final int typeCode = (c >> 4) & 7; - long inflatedLength = c & 15; - int shift = 4; - int headerCnt = 1; - while ((c & 0x80) != 0) { - c = buf[headerCnt++] & 0xff; - inflatedLength += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - if (typeCode == Constants.OBJ_OFS_DELTA) { - do { - c = buf[headerCnt++] & 0xff; - } while ((c & 128) != 0); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - } else if (typeCode == Constants.OBJ_REF_DELTA) { - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - - readFully(src.offset + headerCnt, buf, 0, 20, curs); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, 20); - crc2.update(buf, 0, 20); - } - headerCnt += 20; - } else if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - - final long dataOffset = src.offset + headerCnt; - final long dataLength = src.length; - final long expectedCRC; - final ByteArrayWindow quickCopy; - - // Verify the object isn't corrupt before sending. If it is, - // we report it missing instead. - // - try { - quickCopy = curs.quickCopy(this, dataOffset, dataLength); - - if (validate && idx().hasCRC32Support()) { - assert(crc1 != null); - // Index has the CRC32 code cached, validate the object. - // - expectedCRC = idx().findCRC32(src); - if (quickCopy != null) { - quickCopy.crc32(crc1, dataOffset, (int) dataLength); - } else { - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - pos += n; - cnt -= n; - } - } - if (crc1.getValue() != expectedCRC) { - setCorrupt(src.offset); - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } - } else if (validate) { - // We don't have a CRC32 code in the index, so compute it - // now while inflating the raw data to get zlib to tell us - // whether or not the data is safe. - // - Inflater inf = curs.inflater(); - byte[] tmp = new byte[1024]; - if (quickCopy != null) { - quickCopy.check(inf, tmp, dataOffset, (int) dataLength); - } else { - assert(crc1 != null); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - inf.setInput(buf, 0, n); - while (inf.inflate(tmp, 0, tmp.length) > 0) - continue; - pos += n; - cnt -= n; - } - } - if (!inf.finished() || inf.getBytesRead() != dataLength) { - setCorrupt(src.offset); - throw new EOFException(MessageFormat.format( - JGitText.get().shortCompressedStreamAt, - Long.valueOf(src.offset))); - } - assert(crc1 != null); - expectedCRC = crc1.getValue(); - } else { - expectedCRC = -1; - } - } catch (DataFormatException dataFormat) { - setCorrupt(src.offset); - - CorruptObjectException corruptObject = new CorruptObjectException( - MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile()), - dataFormat); - - throw new StoredObjectRepresentationNotAvailableException(src, - corruptObject); - - } catch (IOException ioError) { - throw new StoredObjectRepresentationNotAvailableException(src, - ioError); - } - - if (quickCopy != null) { - // The entire object fits into a single byte array window slice, - // and we have it pinned. Write this out without copying. - // - out.writeHeader(src, inflatedLength); - quickCopy.write(out, dataOffset, (int) dataLength); - - } else if (dataLength <= buf.length) { - // Tiny optimization: Lots of objects are very small deltas or - // deflated commits that are likely to fit in the copy buffer. - // - if (!validate) { - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - pos += n; - cnt -= n; - } - } - out.writeHeader(src, inflatedLength); - out.write(buf, 0, (int) dataLength); - } else { - // Now we are committed to sending the object. As we spool it out, - // check its CRC32 code to make sure there wasn't corruption between - // the verification we did above, and us actually outputting it. - // - out.writeHeader(src, inflatedLength); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - if (validate) { - assert(crc2 != null); - crc2.update(buf, 0, n); - } - out.write(buf, 0, n); - pos += n; - cnt -= n; - } - if (validate) { - assert(crc2 != null); - if (crc2.getValue() != expectedCRC) { - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } - } - } - } - - boolean invalid() { - return invalid; - } - - void setInvalid() { - invalid = true; - } - - int incrementTransientErrorCount() { - return transientErrorCount.incrementAndGet(); - } - - void resetTransientErrorCount() { - transientErrorCount.set(0); - } - - private void readFully(final long position, final byte[] dstbuf, - int dstoff, final int cnt, final WindowCursor curs) - throws IOException { - if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt) - throw new EOFException(); - } - - private synchronized void beginCopyAsIs(ObjectToPack otp) - throws StoredObjectRepresentationNotAvailableException { - if (++activeCopyRawData == 1 && activeWindows == 0) { - try { - doOpen(); - } catch (IOException thisPackNotValid) { - throw new StoredObjectRepresentationNotAvailableException(otp, - thisPackNotValid); - } - } - } - - private synchronized void endCopyAsIs() { - if (--activeCopyRawData == 0 && activeWindows == 0) - doClose(); - } - - synchronized boolean beginWindowCache() throws IOException { - if (++activeWindows == 1) { - if (activeCopyRawData == 0) - doOpen(); - return true; - } - return false; - } - - synchronized boolean endWindowCache() { - final boolean r = --activeWindows == 0; - if (r && activeCopyRawData == 0) - doClose(); - return r; - } - - private void doOpen() throws IOException { - if (invalid) { - openFail(true, invalidatingCause); - throw new PackInvalidException(packFile, invalidatingCause); - } - try { - synchronized (readLock) { - fd = new RandomAccessFile(packFile, "r"); //$NON-NLS-1$ - length = fd.length(); - onOpenPack(); - } - } catch (InterruptedIOException e) { - // don't invalidate the pack, we are interrupted from another thread - openFail(false, e); - throw e; - } catch (FileNotFoundException fn) { - // don't invalidate the pack if opening an existing file failed - // since it may be related to a temporary lack of resources (e.g. - // max open files) - openFail(!packFile.exists(), fn); - throw fn; - } catch (EOFException | AccessDeniedException | NoSuchFileException - | CorruptObjectException | NoPackSignatureException - | PackMismatchException | UnpackException - | UnsupportedPackIndexVersionException - | UnsupportedPackVersionException pe) { - // exceptions signaling permanent problems with a pack - openFail(true, pe); - throw pe; - } catch (IOException | RuntimeException ge) { - // generic exceptions could be transient so we should not mark the - // pack invalid to avoid false MissingObjectExceptions - openFail(false, ge); - throw ge; - } - } - - private void openFail(boolean invalidate, Exception cause) { - activeWindows = 0; - activeCopyRawData = 0; - invalid = invalidate; - invalidatingCause = cause; - doClose(); - } - - private void doClose() { - synchronized (readLock) { - if (fd != null) { - try { - fd.close(); - } catch (IOException err) { - // Ignore a close event. We had it open only for reading. - // There should not be errors related to network buffers - // not flushed, etc. - } - fd = null; - } - } - } - - ByteArrayWindow read(long pos, int size) throws IOException { - synchronized (readLock) { - if (invalid || fd == null) { - // Due to concurrency between a read and another packfile invalidation thread - // one thread could come up to this point and then fail with NPE. - // Detect the situation and throw a proper exception so that can be properly - // managed by the main packfile search loop and the Git client won't receive - // any failures. - throw new PackInvalidException(packFile, invalidatingCause); - } - if (length < pos + size) - size = (int) (length - pos); - final byte[] buf = new byte[size]; - fd.seek(pos); - fd.readFully(buf, 0, size); - return new ByteArrayWindow(this, pos, buf); - } - } - - ByteWindow mmap(long pos, int size) throws IOException { - synchronized (readLock) { - if (length < pos + size) - size = (int) (length - pos); - - MappedByteBuffer map; - try { - map = fd.getChannel().map(MapMode.READ_ONLY, pos, size); - } catch (IOException ioe1) { - // The most likely reason this failed is the JVM has run out - // of virtual memory. We need to discard quickly, and try to - // force the GC to finalize and release any existing mappings. - // - System.gc(); - System.runFinalization(); - map = fd.getChannel().map(MapMode.READ_ONLY, pos, size); - } - - if (map.hasArray()) - return new ByteArrayWindow(this, pos, map.array()); - return new ByteBufferWindow(this, pos, map); - } - } - - private void onOpenPack() throws IOException { - final PackIndex idx = idx(); - final byte[] buf = new byte[20]; - - fd.seek(0); - fd.readFully(buf, 0, 12); - if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) { - throw new NoPackSignatureException(JGitText.get().notAPACKFile); - } - final long vers = NB.decodeUInt32(buf, 4); - final long packCnt = NB.decodeUInt32(buf, 8); - if (vers != 2 && vers != 3) { - throw new UnsupportedPackVersionException(vers); - } - - if (packCnt != idx.getObjectCount()) { - throw new PackMismatchException(MessageFormat.format( - JGitText.get().packObjectCountMismatch, - Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()), - getPackFile())); - } - - fd.seek(length - 20); - fd.readFully(buf, 0, 20); - if (!Arrays.equals(buf, packChecksum)) { - throw new PackMismatchException(MessageFormat.format( - JGitText.get().packChecksumMismatch, - getPackFile(), - ObjectId.fromRaw(buf).name(), - ObjectId.fromRaw(idx.packChecksum).name())); - } - } - - ObjectLoader load(WindowCursor curs, long pos) - throws IOException, LargeObjectException { - try { - final byte[] ib = curs.tempId; - Delta delta = null; - byte[] data = null; - int type = Constants.OBJ_BAD; - boolean cached = false; - - SEARCH: for (;;) { - readFully(pos, ib, 0, 20, curs); - int c = ib[0] & 0xff; - final int typeCode = (c >> 4) & 7; - long sz = c & 15; - int shift = 4; - int p = 1; - while ((c & 0x80) != 0) { - c = ib[p++] & 0xff; - sz += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - switch (typeCode) { - case Constants.OBJ_COMMIT: - case Constants.OBJ_TREE: - case Constants.OBJ_BLOB: - case Constants.OBJ_TAG: { - if (delta != null || sz < curs.getStreamFileThreshold()) { - data = decompress(pos + p, (int) sz, curs); - } - - if (delta != null) { - type = typeCode; - break SEARCH; - } - - if (data != null) { - return new ObjectLoader.SmallObject(typeCode, data); - } - return new LargePackedWholeObject(typeCode, sz, pos, p, - this, curs.db); - } - - case Constants.OBJ_OFS_DELTA: { - c = ib[p++] & 0xff; - long base = c & 127; - while ((c & 128) != 0) { - base += 1; - c = ib[p++] & 0xff; - base <<= 7; - base += (c & 127); - } - base = pos - base; - delta = new Delta(delta, pos, (int) sz, p, base); - if (sz != delta.deltaSize) - break SEARCH; - - DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); - if (e != null) { - type = e.type; - data = e.data; - cached = true; - break SEARCH; - } - pos = base; - continue SEARCH; - } - - case Constants.OBJ_REF_DELTA: { - readFully(pos + p, ib, 0, 20, curs); - long base = findDeltaBase(ObjectId.fromRaw(ib)); - delta = new Delta(delta, pos, (int) sz, p + 20, base); - if (sz != delta.deltaSize) - break SEARCH; - - DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); - if (e != null) { - type = e.type; - data = e.data; - cached = true; - break SEARCH; - } - pos = base; - continue SEARCH; - } - - default: - throw new IOException(MessageFormat.format( - JGitText.get().unknownObjectType, - Integer.valueOf(typeCode))); - } - } - - // At this point there is at least one delta to apply to data. - // (Whole objects with no deltas to apply return early above.) - - if (data == null) - throw new IOException(JGitText.get().inMemoryBufferLimitExceeded); - - assert(delta != null); - do { - // Cache only the base immediately before desired object. - if (cached) - cached = false; - else if (delta.next == null) - curs.getDeltaBaseCache().store(this, delta.basePos, data, type); - - pos = delta.deltaPos; - - final byte[] cmds = decompress(pos + delta.hdrLen, - delta.deltaSize, curs); - if (cmds == null) { - data = null; // Discard base in case of OutOfMemoryError - throw new LargeObjectException.OutOfMemory(new OutOfMemoryError()); - } - - final long sz = BinaryDelta.getResultSize(cmds); - if (Integer.MAX_VALUE <= sz) - throw new LargeObjectException.ExceedsByteArrayLimit(); - - final byte[] result; - try { - result = new byte[(int) sz]; - } catch (OutOfMemoryError tooBig) { - data = null; // Discard base in case of OutOfMemoryError - throw new LargeObjectException.OutOfMemory(tooBig); - } - - BinaryDelta.apply(data, cmds, result); - data = result; - delta = delta.next; - } while (delta != null); - - return new ObjectLoader.SmallObject(type, data); - - } catch (DataFormatException dfe) { - throw new CorruptObjectException( - MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(pos), getPackFile()), - dfe); - } - } - - private long findDeltaBase(ObjectId baseId) throws IOException, - MissingObjectException { - long ofs = idx().findOffset(baseId); - if (ofs < 0) - throw new MissingObjectException(baseId, - JGitText.get().missingDeltaBase); - return ofs; - } - - private static class Delta { - /** Child that applies onto this object. */ - final Delta next; - - /** Offset of the delta object. */ - final long deltaPos; - - /** Size of the inflated delta stream. */ - final int deltaSize; - - /** Total size of the delta's pack entry header (including base). */ - final int hdrLen; - - /** Offset of the base object this delta applies onto. */ - final long basePos; - - Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) { - this.next = next; - this.deltaPos = ofs; - this.deltaSize = sz; - this.hdrLen = hdrLen; - this.basePos = baseOffset; - } - } - - byte[] getDeltaHeader(WindowCursor wc, long pos) - throws IOException, DataFormatException { - // The delta stream starts as two variable length integers. If we - // assume they are 64 bits each, we need 16 bytes to encode them, - // plus 2 extra bytes for the variable length overhead. So 18 is - // the longest delta instruction header. - // - final byte[] hdr = new byte[18]; - wc.inflate(this, pos, hdr, true /* headerOnly */); - return hdr; - } - - int getObjectType(WindowCursor curs, long pos) throws IOException { - final byte[] ib = curs.tempId; - for (;;) { - readFully(pos, ib, 0, 20, curs); - int c = ib[0] & 0xff; - final int type = (c >> 4) & 7; - - switch (type) { - case Constants.OBJ_COMMIT: - case Constants.OBJ_TREE: - case Constants.OBJ_BLOB: - case Constants.OBJ_TAG: - return type; - - case Constants.OBJ_OFS_DELTA: { - int p = 1; - while ((c & 0x80) != 0) - c = ib[p++] & 0xff; - c = ib[p++] & 0xff; - long ofs = c & 127; - while ((c & 128) != 0) { - ofs += 1; - c = ib[p++] & 0xff; - ofs <<= 7; - ofs += (c & 127); - } - pos = pos - ofs; - continue; - } - - case Constants.OBJ_REF_DELTA: { - int p = 1; - while ((c & 0x80) != 0) - c = ib[p++] & 0xff; - readFully(pos + p, ib, 0, 20, curs); - pos = findDeltaBase(ObjectId.fromRaw(ib)); - continue; - } - - default: - throw new IOException( - MessageFormat.format(JGitText.get().unknownObjectType, - Integer.valueOf(type))); - } - } - } - - long getObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { - final long offset = idx().findOffset(id); - return 0 < offset ? getObjectSize(curs, offset) : -1; - } - - long getObjectSize(WindowCursor curs, long pos) - throws IOException { - final byte[] ib = curs.tempId; - readFully(pos, ib, 0, 20, curs); - int c = ib[0] & 0xff; - final int type = (c >> 4) & 7; - long sz = c & 15; - int shift = 4; - int p = 1; - while ((c & 0x80) != 0) { - c = ib[p++] & 0xff; - sz += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - long deltaAt; - switch (type) { - case Constants.OBJ_COMMIT: - case Constants.OBJ_TREE: - case Constants.OBJ_BLOB: - case Constants.OBJ_TAG: - return sz; - - case Constants.OBJ_OFS_DELTA: - c = ib[p++] & 0xff; - while ((c & 128) != 0) - c = ib[p++] & 0xff; - deltaAt = pos + p; - break; - - case Constants.OBJ_REF_DELTA: - deltaAt = pos + p + 20; - break; - - default: - throw new IOException(MessageFormat.format( - JGitText.get().unknownObjectType, Integer.valueOf(type))); - } - - try { - return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt)); - } catch (DataFormatException e) { - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos), - getPackFile()), e); - } - } - - LocalObjectRepresentation representation(final WindowCursor curs, - final AnyObjectId objectId) throws IOException { - final long pos = idx().findOffset(objectId); - if (pos < 0) - return null; - - final byte[] ib = curs.tempId; - readFully(pos, ib, 0, 20, curs); - int c = ib[0] & 0xff; - int p = 1; - final int typeCode = (c >> 4) & 7; - while ((c & 0x80) != 0) - c = ib[p++] & 0xff; - - long len = (findEndOffset(pos) - pos); - switch (typeCode) { - case Constants.OBJ_COMMIT: - case Constants.OBJ_TREE: - case Constants.OBJ_BLOB: - case Constants.OBJ_TAG: - return LocalObjectRepresentation.newWhole(this, pos, len - p); - - case Constants.OBJ_OFS_DELTA: { - c = ib[p++] & 0xff; - long ofs = c & 127; - while ((c & 128) != 0) { - ofs += 1; - c = ib[p++] & 0xff; - ofs <<= 7; - ofs += (c & 127); - } - ofs = pos - ofs; - return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs); - } - - case Constants.OBJ_REF_DELTA: { - len -= p; - len -= Constants.OBJECT_ID_LENGTH; - readFully(pos + p, ib, 0, 20, curs); - ObjectId id = ObjectId.fromRaw(ib); - return LocalObjectRepresentation.newDelta(this, pos, len, id); - } - - default: - throw new IOException( - MessageFormat.format(JGitText.get().unknownObjectType, - Integer.valueOf(typeCode))); - } - } - - private long findEndOffset(long startOffset) - throws IOException, CorruptObjectException { - final long maxOffset = length - 20; - return getReverseIdx().findNextOffset(startOffset, maxOffset); - } - - synchronized PackBitmapIndex getBitmapIndex() throws IOException { - if (invalid || invalidBitmap) - return null; - if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { - final PackBitmapIndex idx; - try { - idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(), - getReverseIdx()); - } catch (FileNotFoundException e) { - // Once upon a time this bitmap file existed. Now it - // has been removed. Most likely an external gc has - // removed this packfile and the bitmap - invalidBitmap = true; - return null; - } - - // At this point, idx() will have set packChecksum. - if (Arrays.equals(packChecksum, idx.packChecksum)) - bitmapIdx = idx; - else - invalidBitmap = true; - } - return bitmapIdx; - } - - private synchronized PackReverseIndex getReverseIdx() throws IOException { - if (reverseIdx == null) - reverseIdx = new PackReverseIndex(idx()); - return reverseIdx; - } - - private boolean isCorrupt(long offset) { - LongList list = corruptObjects; - if (list == null) - return false; - synchronized (list) { - return list.contains(offset); - } - } - - private void setCorrupt(long offset) { - LongList list = corruptObjects; - if (list == null) { - synchronized (readLock) { - list = corruptObjects; - if (list == null) { - list = new LongList(); - corruptObjects = list; - } - } - } - synchronized (list) { - list.add(offset); - } - } - - private File extFile(PackExt ext) { - String p = packFile.getName(); - int dot = p.lastIndexOf('.'); - String b = (dot < 0) ? p : p.substring(0, dot); - return new File(packFile.getParentFile(), b + '.' + ext.getExtension()); - } - - private boolean hasExt(PackExt ext) { - return (extensions & ext.getBit()) != 0; - } - - @SuppressWarnings("nls") - @Override - public String toString() { - return "PackFile [packFileName=" + packFile.getName() + ", length=" - + packFile.length() + ", packChecksum=" - + ObjectId.fromRaw(packChecksum).name() + "]"; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 31686befc9..942cc96745 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -34,7 +34,7 @@ import org.eclipse.jgit.util.io.SilentFileInputStream; /** * Access path to locate objects by {@link org.eclipse.jgit.lib.ObjectId} in a - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. *

* Indexes are strictly redundant information in that we can rebuild all of the * data held in the index file from the on disk representation of the pack file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java index 612b12366c..87e0b44d46 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.util.NB; /** * Creates a table of contents to support random access by - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. *

* Pack index files (the .idx suffix in a pack file pair) provides * random access to any object in the pack by associating an ObjectId to the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java index a9e0588885..0bceca72ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java @@ -16,11 +16,11 @@ import java.io.InputStream; class PackInputStream extends InputStream { private final WindowCursor wc; - private final PackFile pack; + private final Pack pack; private long pos; - PackInputStream(PackFile pack, long pos, WindowCursor wc) + PackInputStream(Pack pack, long pos, WindowCursor wc) throws IOException { this.pack = pack; this.pos = pos; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java index 2c2f7911aa..482b143e33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java @@ -18,7 +18,7 @@ import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; /** - * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.PackFile}'s + * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.Pack}'s * associated .keep file. */ public class PackLock { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index 4d80a0312a..ee458e27ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.lib.ObjectId; *

* * @see PackIndex - * @see PackFile + * @see Pack */ public class PackReverseIndex { /** Index we were created from, and that has our ObjectId data. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 3e8cb3a3f2..25653b3ce3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -33,7 +33,7 @@ import org.eclipse.jgit.storage.file.WindowCacheStats; import org.eclipse.jgit.util.Monitoring; /** - * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.PackFile} in + * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.Pack} in * memory for faster read access. *

* The WindowCache serves as a Java based "buffer cache", loading segments of a @@ -41,7 +41,7 @@ import org.eclipse.jgit.util.Monitoring; * only tiny slices of a file, the WindowCache tries to smooth out these tiny * reads into larger block-sized IO operations. *

- * Whenever a cache miss occurs, {@link #load(PackFile, long)} is invoked by + * Whenever a cache miss occurs, {@link #load(Pack, long)} is invoked by * exactly one thread for the given (PackFile,position) key tuple. * This is ensured by an array of locks, with the tuple hashed to a lock * instance. @@ -80,10 +80,10 @@ import org.eclipse.jgit.util.Monitoring; *

* This cache has an implementation rule such that: *

    - *
  • {@link #load(PackFile, long)} is invoked by at most one thread at a time + *
  • {@link #load(Pack, long)} is invoked by at most one thread at a time * for a given (PackFile,position) tuple.
    • *
  • For every load() invocation there is exactly one - * {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a + * {@link #createRef(Pack, long, ByteWindow)} invocation to wrap a * SoftReference or a StrongReference around the cached entity.
    • *
  • For every Reference created by createRef() there will be * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated @@ -91,10 +91,10 @@ import org.eclipse.jgit.util.Monitoring; *
*

* Therefore, it is safe to perform resource accounting increments during the - * {@link #load(PackFile, long)} or - * {@link #createRef(PackFile, long, ByteWindow)} methods, and matching + * {@link #load(Pack, long)} or + * {@link #createRef(Pack, long, ByteWindow)} methods, and matching * decrements during {@link #clear(PageRef)}. Implementors may need to override - * {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional + * {@link #createRef(Pack, long, ByteWindow)} in order to embed additional * accounting information into an implementation specific * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as * the cached entity may have already been evicted by the JRE's garbage @@ -170,7 +170,7 @@ public class WindowCache { * @param delta * delta of cached bytes */ - void recordOpenBytes(PackFile pack, int delta); + void recordOpenBytes(Pack pack, int delta); /** * Returns a snapshot of this recorder's stats. Note that this may be an @@ -242,7 +242,7 @@ public class WindowCache { } @Override - public void recordOpenBytes(PackFile pack, int delta) { + public void recordOpenBytes(Pack pack, int delta) { openByteCount.add(delta); String repositoryId = repositoryId(pack); LongAdder la = openByteCountPerRepository @@ -254,9 +254,8 @@ public class WindowCache { } } - private static String repositoryId(PackFile pack) { - // use repository's gitdir since packfile doesn't know its - // repository + private static String repositoryId(Pack pack) { + // use repository's gitdir since Pack doesn't know its repository return pack.getPackFile().getParentFile().getParentFile() .getParent(); } @@ -380,7 +379,7 @@ public class WindowCache { return cache.publishMBeanIfNeeded(); } - static final ByteWindow get(PackFile pack, long offset) + static final ByteWindow get(Pack pack, long offset) throws IOException { final WindowCache c = cache; final ByteWindow r = c.getOrLoad(pack, c.toStart(offset)); @@ -395,7 +394,7 @@ public class WindowCache { return r; } - static final void purge(PackFile pack) { + static final void purge(Pack pack) { cache.removeAll(pack); } @@ -506,7 +505,7 @@ public class WindowCache { return packHash + (int) (off >>> windowSizeShift); } - private ByteWindow load(PackFile pack, long offset) throws IOException { + private ByteWindow load(Pack pack, long offset) throws IOException { long startTime = System.nanoTime(); if (pack.beginWindowCache()) statsRecorder.recordOpenFiles(1); @@ -525,7 +524,7 @@ public class WindowCache { } } - private PageRef createRef(PackFile p, long o, ByteWindow v) { + private PageRef createRef(Pack p, long o, ByteWindow v) { final PageRef ref = useStrongRefs ? new StrongRef(p, o, v, queue) : new SoftRef(p, o, v, (SoftCleanupQueue) queue); @@ -539,7 +538,7 @@ public class WindowCache { close(ref.getPack()); } - private void close(PackFile pack) { + private void close(Pack pack) { if (pack.endWindowCache()) { statsRecorder.recordOpenFiles(-1); } @@ -578,9 +577,9 @@ public class WindowCache { * @return the object reference. * @throws IOException * the object reference was not in the cache and could not be - * obtained by {@link #load(PackFile, long)}. + * obtained by {@link #load(Pack, long)}. */ - private ByteWindow getOrLoad(PackFile pack, long position) + private ByteWindow getOrLoad(Pack pack, long position) throws IOException { final int slot = slot(pack, position); final Entry e1 = table.get(slot); @@ -623,7 +622,7 @@ public class WindowCache { return v; } - private ByteWindow scan(Entry n, PackFile pack, long position) { + private ByteWindow scan(Entry n, Pack pack, long position) { for (; n != null; n = n.next) { final PageRef r = n.ref; if (r.getPack() == pack && r.getPosition() == position) { @@ -704,7 +703,7 @@ public class WindowCache { /** * Clear all entries related to a single file. *

- * Typically this method is invoked during {@link PackFile#close()}, when we + * Typically this method is invoked during {@link Pack#close()}, when we * know the pack is never going to be useful to us again (for example, it no * longer exists on disk). A concurrent reader loading an entry from this * same pack may cause the pack to become stuck in the cache anyway. @@ -712,7 +711,7 @@ public class WindowCache { * @param pack * the file to purge all entries of. */ - private void removeAll(PackFile pack) { + private void removeAll(Pack pack) { for (int s = 0; s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; @@ -733,11 +732,11 @@ public class WindowCache { queue.gc(); } - private int slot(PackFile pack, long position) { + private int slot(Pack pack, long position) { return (hash(pack.hash, position) >>> 1) % tableSize; } - private Lock lock(PackFile pack, long position) { + private Lock lock(Pack pack, long position) { return locks[(hash(pack.hash, position) >>> 1) % locks.length]; } @@ -799,16 +798,20 @@ public class WindowCache { boolean kill(); /** - * Get the packfile the referenced cache page is allocated for + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for * - * @return the packfile the referenced cache page is allocated for + * @return the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for */ - PackFile getPack(); + Pack getPack(); /** - * Get the position of the referenced cache page in the packfile + * Get the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} * - * @return the position of the referenced cache page in the packfile + * @return the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} */ long getPosition(); @@ -844,7 +847,7 @@ public class WindowCache { /** A soft reference wrapped around a cached object. */ private static class SoftRef extends SoftReference implements PageRef { - private final PackFile pack; + private final Pack pack; private final long position; @@ -852,7 +855,7 @@ public class WindowCache { private long lastAccess; - protected SoftRef(final PackFile pack, final long position, + protected SoftRef(final Pack pack, final long position, final ByteWindow v, final SoftCleanupQueue queue) { super(v, queue); this.pack = pack; @@ -861,7 +864,7 @@ public class WindowCache { } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } @@ -900,7 +903,7 @@ public class WindowCache { private static class StrongRef implements PageRef { private ByteWindow referent; - private final PackFile pack; + private final Pack pack; private final long position; @@ -910,7 +913,7 @@ public class WindowCache { private CleanupQueue queue; - protected StrongRef(final PackFile pack, final long position, + protected StrongRef(final Pack pack, final long position, final ByteWindow v, final CleanupQueue queue) { this.pack = pack; this.position = position; @@ -920,7 +923,7 @@ public class WindowCache { } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 6c975708a3..e7fd7b9e76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -86,7 +86,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { /** {@inheritDoc} */ @Override public BitmapIndex getBitmapIndex() throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (index != null) return new BitmapIndexImpl(index); @@ -98,7 +98,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { @Override public Collection getCachedPacksAndUpdate( BitmapBuilder needBitmap) throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (needBitmap.removeAllOrNone(index)) return Collections. singletonList( @@ -218,7 +218,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * this cursor does not match the provider or id and the proper * window could not be acquired through the provider's cache. */ - int copy(final PackFile pack, long position, final byte[] dstbuf, + int copy(final Pack pack, long position, final byte[] dstbuf, int dstoff, final int cnt) throws IOException { final long length = pack.length; int need = cnt; @@ -239,7 +239,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { ((LocalCachedPack) pack).copyAsIs(out, this); } - void copyPackAsIs(final PackFile pack, final long length, + void copyPackAsIs(final Pack pack, final long length, final PackOutputStream out) throws IOException { long position = 12; long remaining = length - (12 + 20); @@ -275,7 +275,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * the inflater encountered an invalid chunk of data. Data * stream corruption is likely. */ - int inflate(final PackFile pack, long position, final byte[] dstbuf, + int inflate(final Pack pack, long position, final byte[] dstbuf, boolean headerOnly) throws IOException, DataFormatException { prepareInflater(); pin(pack, position); @@ -293,7 +293,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } } - ByteArrayWindow quickCopy(PackFile p, long pos, long cnt) + ByteArrayWindow quickCopy(Pack p, long pos, long cnt) throws IOException { pin(p, pos); if (window instanceof ByteArrayWindow @@ -314,7 +314,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { inf.reset(); } - void pin(PackFile pack, long position) + void pin(Pack pack, long position) throws IOException { final ByteWindow w = window; if (w == null || !w.contains(pack, position)) { -- cgit v1.2.3