Pack better represents the purpose of the object and paves the way to add a PackFile object that extends File. Change-Id: I39b4f697902d395e9b6df5e8ce53078ce72fcea3 Signed-off-by: Nasser Grainawi <quic_nasserg@quicinc.com>tags/v5.11.0.202102240950-m3
import javax.servlet.http.HttpServletResponse; | import javax.servlet.http.HttpServletResponse; | ||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory; | import org.eclipse.jgit.internal.storage.file.ObjectDirectory; | ||||
import org.eclipse.jgit.internal.storage.file.PackFile; | |||||
import org.eclipse.jgit.internal.storage.file.Pack; | |||||
import org.eclipse.jgit.lib.ObjectDatabase; | import org.eclipse.jgit.lib.ObjectDatabase; | ||||
/** Sends the current list of pack files, sorted most recent first. */ | /** Sends the current list of pack files, sorted most recent first. */ | ||||
final StringBuilder out = new StringBuilder(); | final StringBuilder out = new StringBuilder(); | ||||
final ObjectDatabase db = getRepository(req).getObjectDatabase(); | final ObjectDatabase db = getRepository(req).getObjectDatabase(); | ||||
if (db instanceof ObjectDirectory) { | if (db instanceof ObjectDirectory) { | ||||
for (PackFile pack : ((ObjectDirectory) db).getPacks()) { | |||||
for (Pack pack : ((ObjectDirectory) db).getPacks()) { | |||||
out.append("P "); | out.append("P "); | ||||
out.append(pack.getPackFile().getName()); | out.append(pack.getPackFile().getName()); | ||||
out.append('\n'); | out.append('\n'); |
import org.eclipse.jgit.internal.storage.file.FileRepository; | import org.eclipse.jgit.internal.storage.file.FileRepository; | ||||
import org.eclipse.jgit.internal.storage.file.LockFile; | import org.eclipse.jgit.internal.storage.file.LockFile; | ||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory; | import org.eclipse.jgit.internal.storage.file.ObjectDirectory; | ||||
import org.eclipse.jgit.internal.storage.file.PackFile; | |||||
import org.eclipse.jgit.internal.storage.file.Pack; | |||||
import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; | import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; | ||||
import org.eclipse.jgit.internal.storage.pack.PackWriter; | import org.eclipse.jgit.internal.storage.pack.PackWriter; | ||||
import org.eclipse.jgit.lib.AnyObjectId; | import org.eclipse.jgit.lib.AnyObjectId; | ||||
rw.writeInfoRefs(); | rw.writeInfoRefs(); | ||||
final StringBuilder w = new StringBuilder(); | final StringBuilder w = new StringBuilder(); | ||||
for (PackFile p : fr.getObjectDatabase().getPacks()) { | |||||
for (Pack p : fr.getObjectDatabase().getPacks()) { | |||||
w.append("P "); | w.append("P "); | ||||
w.append(p.getPackFile().getName()); | w.append(p.getPackFile().getName()); | ||||
w.append('\n'); | w.append('\n'); | ||||
} | } | ||||
private static void prunePacked(ObjectDirectory odb) throws IOException { | private static void prunePacked(ObjectDirectory odb) throws IOException { | ||||
for (PackFile p : odb.getPacks()) { | |||||
for (Pack p : odb.getPacks()) { | |||||
for (MutableEntry e : p) | for (MutableEntry e : p) | ||||
FileUtils.delete(odb.fileFor(e.toObjectId())); | FileUtils.delete(odb.fileFor(e.toObjectId())); | ||||
} | } |
.create(); | .create(); | ||||
tr.update("refs/tags/t1", second); | tr.update("refs/tags/t1", second); | ||||
Collection<PackFile> oldPacks = tr.getRepository().getObjectDatabase() | |||||
Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase() | |||||
.getPacks(); | .getPacks(); | ||||
assertEquals(0, oldPacks.size()); | assertEquals(0, oldPacks.size()); | ||||
stats = gc.getStatistics(); | stats = gc.getStatistics(); | ||||
stats = gc.getStatistics(); | stats = gc.getStatistics(); | ||||
assertEquals(0, stats.numberOfLooseObjects); | assertEquals(0, stats.numberOfLooseObjects); | ||||
List<PackFile> packs = new ArrayList<>( | |||||
List<Pack> packs = new ArrayList<>( | |||||
repo.getObjectDatabase().getPacks()); | repo.getObjectDatabase().getPacks()); | ||||
assertEquals(11, packs.get(0).getObjectCount()); | assertEquals(11, packs.get(0).getObjectCount()); | ||||
} | } |
} | } | ||||
} | } | ||||
PackFile getSinglePack(FileRepository r) { | |||||
Collection<PackFile> packs = r.getObjectDatabase().getPacks(); | |||||
Pack getSinglePack(FileRepository r) { | |||||
Collection<Pack> packs = r.getObjectDatabase().getPacks(); | |||||
assertEquals(1, packs.size()); | assertEquals(1, packs.size()); | ||||
return packs.iterator().next(); | return packs.iterator().next(); | ||||
} | } | ||||
SampleDataRepositoryTestCase.copyCGitTestPacks(repo); | SampleDataRepositoryTestCase.copyCGitTestPacks(repo); | ||||
ExecutorService executor = Executors.newSingleThreadExecutor(); | ExecutorService executor = Executors.newSingleThreadExecutor(); | ||||
final CountDownLatch latch = new CountDownLatch(1); | final CountDownLatch latch = new CountDownLatch(1); | ||||
Future<Collection<PackFile>> result = executor.submit(() -> { | |||||
Future<Collection<Pack>> result = executor.submit(() -> { | |||||
long start = System.currentTimeMillis(); | long start = System.currentTimeMillis(); | ||||
System.out.println("starting gc"); | System.out.println("starting gc"); | ||||
latch.countDown(); | latch.countDown(); | ||||
Collection<PackFile> r = gc.gc(); | |||||
Collection<Pack> r = gc.gc(); | |||||
System.out.println( | System.out.println( | ||||
"gc took " + (System.currentTimeMillis() - start) + " ms"); | "gc took " + (System.currentTimeMillis() - start) + " ms"); | ||||
return r; | return r; |
assertEquals(4, stats.numberOfPackedObjects); | assertEquals(4, stats.numberOfPackedObjects); | ||||
assertEquals(1, stats.numberOfPackFiles); | assertEquals(1, stats.numberOfPackFiles); | ||||
Iterator<PackFile> packIt = repo.getObjectDatabase().getPacks() | |||||
Iterator<Pack> packIt = repo.getObjectDatabase().getPacks() | |||||
.iterator(); | .iterator(); | ||||
PackFile singlePack = packIt.next(); | |||||
Pack singlePack = packIt.next(); | |||||
assertFalse(packIt.hasNext()); | assertFalse(packIt.hasNext()); | ||||
String packFileName = singlePack.getPackFile().getPath(); | String packFileName = singlePack.getPackFile().getPath(); | ||||
String keepFileName = packFileName.substring(0, | String keepFileName = packFileName.substring(0, | ||||
assertEquals(2, stats.numberOfPackFiles); | assertEquals(2, stats.numberOfPackFiles); | ||||
// check that no object is packed twice | // check that no object is packed twice | ||||
Iterator<PackFile> packs = repo.getObjectDatabase().getPacks() | |||||
Iterator<Pack> packs = repo.getObjectDatabase().getPacks() | |||||
.iterator(); | .iterator(); | ||||
PackIndex ind1 = packs.next().getIndex(); | PackIndex ind1 = packs.next().getIndex(); | ||||
assertEquals(4, ind1.getObjectCount()); | assertEquals(4, ind1.getObjectCount()); |
c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, | c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, | ||||
ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); | ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); | ||||
c.save(); | c.save(); | ||||
Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION); | |||||
Collection<Pack> packs = gc(Deflater.NO_COMPRESSION); | |||||
assertEquals("expected 1 packfile after gc", 1, packs.size()); | assertEquals("expected 1 packfile after gc", 1, packs.size()); | ||||
PackFile p1 = packs.iterator().next(); | |||||
Pack p1 = packs.iterator().next(); | |||||
PackFileSnapshot snapshot = p1.getFileSnapshot(); | PackFileSnapshot snapshot = p1.getFileSnapshot(); | ||||
packs = gc(Deflater.BEST_COMPRESSION); | packs = gc(Deflater.BEST_COMPRESSION); | ||||
assertEquals("expected 1 packfile after gc", 1, packs.size()); | assertEquals("expected 1 packfile after gc", 1, packs.size()); | ||||
PackFile p2 = packs.iterator().next(); | |||||
Pack p2 = packs.iterator().next(); | |||||
File pf = p2.getPackFile(); | File pf = p2.getPackFile(); | ||||
// changing compression level with aggressive gc may change size, | // changing compression level with aggressive gc may change size, | ||||
createTestRepo(testDataSeed, testDataLength); | createTestRepo(testDataSeed, testDataLength); | ||||
// repack to create initial packfile | // repack to create initial packfile | ||||
PackFile pf = repackAndCheck(5, null, null, null); | |||||
Path packFilePath = pf.getPackFile().toPath(); | |||||
AnyObjectId chk1 = pf.getPackChecksum(); | |||||
String name = pf.getPackName(); | |||||
Long length = Long.valueOf(pf.getPackFile().length()); | |||||
Pack p = repackAndCheck(5, null, null, null); | |||||
Path packFilePath = p.getPackFile().toPath(); | |||||
AnyObjectId chk1 = p.getPackChecksum(); | |||||
String name = p.getPackName(); | |||||
Long length = Long.valueOf(p.getPackFile().length()); | |||||
FS fs = db.getFS(); | FS fs = db.getFS(); | ||||
Instant m1 = fs.lastModifiedInstant(packFilePath); | Instant m1 = fs.lastModifiedInstant(packFilePath); | ||||
createTestRepo(testDataSeed, testDataLength); | createTestRepo(testDataSeed, testDataLength); | ||||
// Repack to create initial packfile. Make a copy of it | // Repack to create initial packfile. Make a copy of it | ||||
PackFile pf = repackAndCheck(5, null, null, null); | |||||
Path packFilePath = pf.getPackFile().toPath(); | |||||
Pack p = repackAndCheck(5, null, null, null); | |||||
Path packFilePath = p.getPackFile().toPath(); | |||||
Path fn = packFilePath.getFileName(); | Path fn = packFilePath.getFileName(); | ||||
assertNotNull(fn); | assertNotNull(fn); | ||||
String packFileName = fn.toString(); | String packFileName = fn.toString(); | ||||
Path packFileBasePath = packFilePath | Path packFileBasePath = packFilePath | ||||
.resolveSibling(packFileName.replaceAll(".pack", "")); | .resolveSibling(packFileName.replaceAll(".pack", "")); | ||||
AnyObjectId chk1 = pf.getPackChecksum(); | |||||
String name = pf.getPackName(); | |||||
Long length = Long.valueOf(pf.getPackFile().length()); | |||||
AnyObjectId chk1 = p.getPackChecksum(); | |||||
String name = p.getPackName(); | |||||
Long length = Long.valueOf(p.getPackFile().length()); | |||||
copyPack(packFileBasePath, "", ".copy1"); | copyPack(packFileBasePath, "", ".copy1"); | ||||
// Repack to create second packfile. Make a copy of it | // Repack to create second packfile. Make a copy of it | ||||
Paths.get(base + ".pack" + dstSuffix)); | Paths.get(base + ".pack" + dstSuffix)); | ||||
} | } | ||||
private PackFile repackAndCheck(int compressionLevel, String oldName, | |||||
private Pack repackAndCheck(int compressionLevel, String oldName, | |||||
Long oldLength, AnyObjectId oldChkSum) | Long oldLength, AnyObjectId oldChkSum) | ||||
throws IOException, ParseException { | throws IOException, ParseException { | ||||
PackFile p = getSinglePack(gc(compressionLevel)); | |||||
Pack p = getSinglePack(gc(compressionLevel)); | |||||
File pf = p.getPackFile(); | File pf = p.getPackFile(); | ||||
// The following two assumptions should not cause the test to fail. If | // The following two assumptions should not cause the test to fail. If | ||||
// on a certain platform we get packfiles (containing the same git | // on a certain platform we get packfiles (containing the same git | ||||
return p; | return p; | ||||
} | } | ||||
private PackFile getSinglePack(Collection<PackFile> packs) { | |||||
Iterator<PackFile> pIt = packs.iterator(); | |||||
PackFile p = pIt.next(); | |||||
private Pack getSinglePack(Collection<Pack> packs) { | |||||
Iterator<Pack> pIt = packs.iterator(); | |||||
Pack p = pIt.next(); | |||||
assertFalse(pIt.hasNext()); | assertFalse(pIt.hasNext()); | ||||
return p; | return p; | ||||
} | } | ||||
private Collection<PackFile> gc(int compressionLevel) | |||||
private Collection<Pack> gc(int compressionLevel) | |||||
throws IOException, ParseException { | throws IOException, ParseException { | ||||
GC gc = new GC(db); | GC gc = new GC(db); | ||||
PackConfig pc = new PackConfig(db.getConfig()); | PackConfig pc = new PackConfig(db.getConfig()); |
} | } | ||||
assertPacksOnly(); | assertPacksOnly(); | ||||
List<PackFile> packs = listPacks(); | |||||
List<Pack> packs = listPacks(); | |||||
assertEquals(1, packs.size()); | assertEquals(1, packs.size()); | ||||
assertEquals(3, packs.get(0).getObjectCount()); | assertEquals(3, packs.get(0).getObjectCount()); | ||||
} | } | ||||
assertPacksOnly(); | assertPacksOnly(); | ||||
List<PackFile> packs = listPacks(); | |||||
List<Pack> packs = listPacks(); | |||||
assertEquals(2, packs.size()); | assertEquals(2, packs.size()); | ||||
assertEquals(1, packs.get(0).getObjectCount()); | assertEquals(1, packs.get(0).getObjectCount()); | ||||
assertEquals(1, packs.get(1).getObjectCount()); | assertEquals(1, packs.get(1).getObjectCount()); | ||||
} | } | ||||
assertPacksOnly(); | assertPacksOnly(); | ||||
Collection<PackFile> packs = listPacks(); | |||||
Collection<Pack> packs = listPacks(); | |||||
assertEquals(1, packs.size()); | assertEquals(1, packs.size()); | ||||
PackFile p = packs.iterator().next(); | |||||
Pack p = packs.iterator().next(); | |||||
assertEquals(1, p.getObjectCount()); | assertEquals(1, p.getObjectCount()); | ||||
try (ObjectReader reader = db.newObjectReader()) { | try (ObjectReader reader = db.newObjectReader()) { | ||||
} | } | ||||
assertPacksOnly(); | assertPacksOnly(); | ||||
List<PackFile> packs = listPacks(); | |||||
List<Pack> packs = listPacks(); | |||||
assertEquals(1, packs.size()); | assertEquals(1, packs.size()); | ||||
PackFile pack = packs.get(0); | |||||
Pack pack = packs.get(0); | |||||
assertEquals(1, pack.getObjectCount()); | assertEquals(1, pack.getObjectCount()); | ||||
String inode = getInode(pack.getPackFile()); | String inode = getInode(pack.getPackFile()); | ||||
} | } | ||||
assertPacksOnly(); | assertPacksOnly(); | ||||
List<PackFile> packs = listPacks(); | |||||
List<Pack> packs = listPacks(); | |||||
assertEquals(1, packs.size()); | assertEquals(1, packs.size()); | ||||
assertEquals(2, packs.get(0).getObjectCount()); | assertEquals(2, packs.get(0).getObjectCount()); | ||||
} | } | ||||
} | } | ||||
private List<PackFile> listPacks() throws Exception { | |||||
List<PackFile> fromOpenDb = listPacks(db); | |||||
List<PackFile> reopened; | |||||
private List<Pack> listPacks() throws Exception { | |||||
List<Pack> fromOpenDb = listPacks(db); | |||||
List<Pack> reopened; | |||||
try (FileRepository db2 = new FileRepository(db.getDirectory())) { | try (FileRepository db2 = new FileRepository(db.getDirectory())) { | ||||
reopened = listPacks(db2); | reopened = listPacks(db2); | ||||
} | } | ||||
assertEquals(fromOpenDb.size(), reopened.size()); | assertEquals(fromOpenDb.size(), reopened.size()); | ||||
for (int i = 0 ; i < fromOpenDb.size(); i++) { | for (int i = 0 ; i < fromOpenDb.size(); i++) { | ||||
PackFile a = fromOpenDb.get(i); | |||||
PackFile b = reopened.get(i); | |||||
Pack a = fromOpenDb.get(i); | |||||
Pack b = reopened.get(i); | |||||
assertEquals(a.getPackName(), b.getPackName()); | assertEquals(a.getPackName(), b.getPackName()); | ||||
assertEquals( | assertEquals( | ||||
a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); | a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); | ||||
return fromOpenDb; | return fromOpenDb; | ||||
} | } | ||||
private static List<PackFile> listPacks(FileRepository db) throws Exception { | |||||
private static List<Pack> listPacks(FileRepository db) throws Exception { | |||||
return db.getObjectDatabase().getPacks().stream() | return db.getObjectDatabase().getPacks().stream() | ||||
.sorted(comparing(PackFile::getPackName)).collect(toList()); | |||||
.sorted(comparing(Pack::getPackName)).collect(toList()); | |||||
} | } | ||||
private PackInserter newInserter() { | private PackInserter newInserter() { |
import org.junit.Before; | import org.junit.Before; | ||||
import org.junit.Test; | import org.junit.Test; | ||||
public class PackFileTest extends LocalDiskRepositoryTestCase { | |||||
public class PackTest extends LocalDiskRepositoryTestCase { | |||||
private int streamThreshold = 16 * 1024; | private int streamThreshold = 16 * 1024; | ||||
private TestRng rng; | private TestRng rng; | ||||
PackedObjectInfo a = new PackedObjectInfo(idA); | PackedObjectInfo a = new PackedObjectInfo(idA); | ||||
PackedObjectInfo b = new PackedObjectInfo(idB); | PackedObjectInfo b = new PackedObjectInfo(idB); | ||||
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); | |||||
packHeader(pack, 2); | |||||
a.setOffset(pack.length()); | |||||
objectHeader(pack, Constants.OBJ_BLOB, base.length); | |||||
deflate(pack, base); | |||||
TemporaryBuffer.Heap packContents = new TemporaryBuffer.Heap(64 * 1024); | |||||
packHeader(packContents, 2); | |||||
a.setOffset(packContents.length()); | |||||
objectHeader(packContents, Constants.OBJ_BLOB, base.length); | |||||
deflate(packContents, base); | |||||
ByteArrayOutputStream tmp = new ByteArrayOutputStream(); | ByteArrayOutputStream tmp = new ByteArrayOutputStream(); | ||||
DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); | DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); | ||||
de.copy(0, 1); | de.copy(0, 1); | ||||
byte[] delta = tmp.toByteArray(); | byte[] delta = tmp.toByteArray(); | ||||
b.setOffset(pack.length()); | |||||
objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length); | |||||
idA.copyRawTo(pack); | |||||
deflate(pack, delta); | |||||
byte[] footer = digest(pack); | |||||
b.setOffset(packContents.length()); | |||||
objectHeader(packContents, Constants.OBJ_REF_DELTA, delta.length); | |||||
idA.copyRawTo(packContents); | |||||
deflate(packContents, delta); | |||||
byte[] footer = digest(packContents); | |||||
File dir = new File(repo.getObjectDatabase().getDirectory(), | File dir = new File(repo.getObjectDatabase().getDirectory(), | ||||
"pack"); | "pack"); | ||||
File idxName = new File(dir, idA.name() + ".idx"); | File idxName = new File(dir, idA.name() + ".idx"); | ||||
try (FileOutputStream f = new FileOutputStream(packName)) { | try (FileOutputStream f = new FileOutputStream(packName)) { | ||||
f.write(pack.toByteArray()); | |||||
f.write(packContents.toByteArray()); | |||||
} | } | ||||
try (FileOutputStream f = new FileOutputStream(idxName)) { | try (FileOutputStream f = new FileOutputStream(idxName)) { | ||||
new PackIndexWriterV1(f).write(list, footer); | new PackIndexWriterV1(f).write(list, footer); | ||||
} | } | ||||
PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit()); | |||||
Pack pack = new Pack(packName, PackExt.INDEX.getBit()); | |||||
try { | try { | ||||
packFile.get(wc, b); | |||||
pack.get(wc, b); | |||||
fail("expected LargeObjectException.ExceedsByteArrayLimit"); | fail("expected LargeObjectException.ExceedsByteArrayLimit"); | ||||
} catch (LargeObjectException.ExceedsByteArrayLimit bad) { | } catch (LargeObjectException.ExceedsByteArrayLimit bad) { | ||||
assertNull(bad.getObjectId()); | assertNull(bad.getObjectId()); | ||||
} finally { | } finally { | ||||
packFile.close(); | |||||
pack.close(); | |||||
} | } | ||||
} | } | ||||
} | } |
private ByteArrayOutputStream os; | private ByteArrayOutputStream os; | ||||
private PackFile pack; | |||||
private Pack pack; | |||||
private ObjectInserter inserter; | private ObjectInserter inserter; | ||||
p.setAllowThin(thin); | p.setAllowThin(thin); | ||||
p.setIndexVersion(2); | p.setIndexVersion(2); | ||||
p.parse(NullProgressMonitor.INSTANCE); | p.parse(NullProgressMonitor.INSTANCE); | ||||
pack = p.getPackFile(); | |||||
pack = p.getPack(); | |||||
assertNotNull("have PackFile after parsing", pack); | assertNotNull("have PackFile after parsing", pack); | ||||
} | } | ||||
final ObjectId id; | final ObjectId id; | ||||
final ObjectLoader or; | final ObjectLoader or; | ||||
PackFile pr = null; | |||||
for (PackFile p : db.getObjectDatabase().getPacks()) { | |||||
Pack pr = null; | |||||
for (Pack p : db.getObjectDatabase().getPacks()) { | |||||
if (PACK_NAME.equals(p.getPackName())) { | if (PACK_NAME.equals(p.getPackName())) { | ||||
pr = p; | pr = p; | ||||
break; | break; |
import org.eclipse.jgit.errors.TooLargeObjectInPackException; | import org.eclipse.jgit.errors.TooLargeObjectInPackException; | ||||
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; | import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; | ||||
import org.eclipse.jgit.internal.storage.file.PackFile; | |||||
import org.eclipse.jgit.internal.storage.file.Pack; | |||||
import org.eclipse.jgit.junit.JGitTestUtil; | import org.eclipse.jgit.junit.JGitTestUtil; | ||||
import org.eclipse.jgit.junit.RepositoryTestCase; | import org.eclipse.jgit.junit.RepositoryTestCase; | ||||
import org.eclipse.jgit.junit.TestRepository; | import org.eclipse.jgit.junit.TestRepository; | ||||
try (InputStream is = new FileInputStream(packFile)) { | try (InputStream is = new FileInputStream(packFile)) { | ||||
ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); | ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); | ||||
p.parse(NullProgressMonitor.INSTANCE); | p.parse(NullProgressMonitor.INSTANCE); | ||||
PackFile file = p.getPackFile(); | |||||
assertTrue(file.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); | |||||
Pack pack = p.getPack(); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); | |||||
} | } | ||||
} | } | ||||
try (InputStream is = new FileInputStream(packFile)) { | try (InputStream is = new FileInputStream(packFile)) { | ||||
ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); | ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); | ||||
p.parse(NullProgressMonitor.INSTANCE); | p.parse(NullProgressMonitor.INSTANCE); | ||||
PackFile file = p.getPackFile(); | |||||
assertTrue(file.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); | |||||
assertTrue(file.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); | |||||
Pack pack = p.getPack(); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); | |||||
assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); | |||||
// and lots more... | // and lots more... | ||||
} | } | ||||
} | } |
import java.io.IOException; | import java.io.IOException; | ||||
/** | /** | ||||
* Thrown when a PackFile is found not to contain the pack signature defined by | |||||
* git. | |||||
* Thrown when a Pack is found not to contain the pack signature defined by git. | |||||
* | * | ||||
* @since 4.5 | * @since 4.5 | ||||
*/ | */ |
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
/** | /** | ||||
* Thrown when a PackFile previously failed and is known to be unusable | |||||
* Thrown when a Pack previously failed and is known to be unusable | |||||
*/ | */ | ||||
public class PackInvalidException extends IOException { | public class PackInvalidException extends IOException { | ||||
private static final long serialVersionUID = 1L; | private static final long serialVersionUID = 1L; |
import java.io.IOException; | import java.io.IOException; | ||||
/** | /** | ||||
* Thrown when a PackFile no longer matches the PackIndex. | |||||
* Thrown when a Pack no longer matches the PackIndex. | |||||
*/ | */ | ||||
public class PackMismatchException extends IOException { | public class PackMismatchException extends IOException { | ||||
private static final long serialVersionUID = 1L; | private static final long serialVersionUID = 1L; |
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
/** | /** | ||||
* Thrown when a PackFile uses a pack version not supported by JGit. | |||||
* Thrown when a Pack uses a pack version not supported by JGit. | |||||
* | * | ||||
* @since 4.5 | * @since 4.5 | ||||
*/ | */ |
final class ByteArrayWindow extends ByteWindow { | final class ByteArrayWindow extends ByteWindow { | ||||
private final byte[] array; | private final byte[] array; | ||||
ByteArrayWindow(PackFile pack, long o, byte[] b) { | |||||
ByteArrayWindow(Pack pack, long o, byte[] b) { | |||||
super(pack, o, b.length); | super(pack, o, b.length); | ||||
array = b; | array = b; | ||||
} | } |
final class ByteBufferWindow extends ByteWindow { | final class ByteBufferWindow extends ByteWindow { | ||||
private final ByteBuffer buffer; | private final ByteBuffer buffer; | ||||
ByteBufferWindow(PackFile pack, long o, ByteBuffer b) { | |||||
ByteBufferWindow(Pack pack, long o, ByteBuffer b) { | |||||
super(pack, o, b.capacity()); | super(pack, o, b.capacity()); | ||||
buffer = b; | buffer = b; | ||||
} | } |
* </p> | * </p> | ||||
*/ | */ | ||||
abstract class ByteWindow { | abstract class ByteWindow { | ||||
protected final PackFile pack; | |||||
protected final Pack pack; | |||||
protected final long start; | protected final long start; | ||||
* Constructor for ByteWindow. | * Constructor for ByteWindow. | ||||
* | * | ||||
* @param p | * @param p | ||||
* a {@link org.eclipse.jgit.internal.storage.file.PackFile}. | |||||
* a {@link org.eclipse.jgit.internal.storage.file.Pack}. | |||||
* @param s | * @param s | ||||
* where the byte window starts in the pack file | * where the byte window starts in the pack file | ||||
* @param n | * @param n | ||||
* size of the byte window | * size of the byte window | ||||
*/ | */ | ||||
protected ByteWindow(PackFile p, long s, int n) { | |||||
protected ByteWindow(Pack p, long s, int n) { | |||||
pack = p; | pack = p; | ||||
start = s; | start = s; | ||||
end = start + n; | end = start + n; | ||||
return (int) (end - start); | return (int) (end - start); | ||||
} | } | ||||
final boolean contains(PackFile neededFile, long neededPos) { | |||||
return pack == neededFile && start <= neededPos && neededPos < end; | |||||
final boolean contains(Pack neededPack, long neededPos) { | |||||
return pack == neededPack && start <= neededPos && neededPos < end; | |||||
} | } | ||||
/** | /** |
} | } | ||||
@Override | @Override | ||||
PackFile openPack(File pack) throws IOException { | |||||
Pack openPack(File pack) throws IOException { | |||||
return wrapped.openPack(pack); | return wrapped.openPack(pack); | ||||
} | } | ||||
} | } | ||||
@Override | @Override | ||||
Collection<PackFile> getPacks() { | |||||
Collection<Pack> getPacks() { | |||||
return wrapped.getPacks(); | return wrapped.getPacks(); | ||||
} | } | ||||
cache = new Slot[CACHE_SZ]; | cache = new Slot[CACHE_SZ]; | ||||
} | } | ||||
Entry get(PackFile pack, long position) { | |||||
Entry get(Pack pack, long position) { | |||||
Slot e = cache[hash(position)]; | Slot e = cache[hash(position)]; | ||||
if (e == null) | if (e == null) | ||||
return null; | return null; | ||||
return null; | return null; | ||||
} | } | ||||
void store(final PackFile pack, final long position, | |||||
void store(final Pack pack, final long position, | |||||
final byte[] data, final int objectType) { | final byte[] data, final int objectType) { | ||||
if (data.length > maxByteCount) | if (data.length > maxByteCount) | ||||
return; // Too large to cache. | return; // Too large to cache. | ||||
Slot lruNext; | Slot lruNext; | ||||
PackFile provider; | |||||
Pack provider; | |||||
long position; | long position; | ||||
abstract InsertLooseObjectResult insertUnpackedObject(File tmp, | abstract InsertLooseObjectResult insertUnpackedObject(File tmp, | ||||
ObjectId id, boolean createDuplicate) throws IOException; | ObjectId id, boolean createDuplicate) throws IOException; | ||||
abstract PackFile openPack(File pack) throws IOException; | |||||
abstract Pack openPack(File pack) throws IOException; | |||||
abstract Collection<PackFile> getPacks(); | |||||
abstract Collection<Pack> getPacks(); | |||||
} | } |
* gc.log. | * gc.log. | ||||
* | * | ||||
* @return the collection of | * @return the collection of | ||||
* {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack}'s which | |||||
* are newly created | * are newly created | ||||
* @throws java.io.IOException | * @throws java.io.IOException | ||||
* @throws java.text.ParseException | * @throws java.text.ParseException | ||||
* If the configuration parameter "gc.pruneexpire" couldn't be | * If the configuration parameter "gc.pruneexpire" couldn't be | ||||
* parsed | * parsed | ||||
*/ | */ | ||||
// TODO(ms): change signature and return Future<Collection<PackFile>> | |||||
// TODO(ms): change signature and return Future<Collection<Pack>> | |||||
@SuppressWarnings("FutureReturnValueIgnored") | @SuppressWarnings("FutureReturnValueIgnored") | ||||
public Collection<PackFile> gc() throws IOException, ParseException { | |||||
public Collection<Pack> gc() throws IOException, ParseException { | |||||
if (!background) { | if (!background) { | ||||
return doGc(); | return doGc(); | ||||
} | } | ||||
return Collections.emptyList(); | return Collections.emptyList(); | ||||
} | } | ||||
Callable<Collection<PackFile>> gcTask = () -> { | |||||
Callable<Collection<Pack>> gcTask = () -> { | |||||
try { | try { | ||||
Collection<PackFile> newPacks = doGc(); | |||||
Collection<Pack> newPacks = doGc(); | |||||
if (automatic && tooManyLooseObjects()) { | if (automatic && tooManyLooseObjects()) { | ||||
String message = JGitText.get().gcTooManyUnpruned; | String message = JGitText.get().gcTooManyUnpruned; | ||||
gcLog.write(message); | gcLog.write(message); | ||||
return (executor != null) ? executor : WorkQueue.getExecutor(); | return (executor != null) ? executor : WorkQueue.getExecutor(); | ||||
} | } | ||||
private Collection<PackFile> doGc() throws IOException, ParseException { | |||||
private Collection<Pack> doGc() throws IOException, ParseException { | |||||
if (automatic && !needGc()) { | if (automatic && !needGc()) { | ||||
return Collections.emptyList(); | return Collections.emptyList(); | ||||
} | } | ||||
pm.start(6 /* tasks */); | pm.start(6 /* tasks */); | ||||
packRefs(); | packRefs(); | ||||
// TODO: implement reflog_expire(pm, repo); | // TODO: implement reflog_expire(pm, repo); | ||||
Collection<PackFile> newPacks = repack(); | |||||
Collection<Pack> newPacks = repack(); | |||||
prune(Collections.emptySet()); | prune(Collections.emptySet()); | ||||
// TODO: implement rerere_gc(pm); | // TODO: implement rerere_gc(pm); | ||||
return newPacks; | return newPacks; | ||||
* @param existing | * @param existing | ||||
* @throws IOException | * @throws IOException | ||||
*/ | */ | ||||
private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing) | |||||
private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet<ObjectId> existing) | |||||
throws IOException { | throws IOException { | ||||
for (PackIndex.MutableEntry entry : pack) { | for (PackIndex.MutableEntry entry : pack) { | ||||
ObjectId oid = entry.toObjectId(); | ObjectId oid = entry.toObjectId(); | ||||
* @throws ParseException | * @throws ParseException | ||||
* @throws IOException | * @throws IOException | ||||
*/ | */ | ||||
private void deleteOldPacks(Collection<PackFile> oldPacks, | |||||
Collection<PackFile> newPacks) throws ParseException, IOException { | |||||
private void deleteOldPacks(Collection<Pack> oldPacks, | |||||
Collection<Pack> newPacks) throws ParseException, IOException { | |||||
HashSet<ObjectId> ids = new HashSet<>(); | HashSet<ObjectId> ids = new HashSet<>(); | ||||
for (PackFile pack : newPacks) { | |||||
for (Pack pack : newPacks) { | |||||
for (PackIndex.MutableEntry entry : pack) { | for (PackIndex.MutableEntry entry : pack) { | ||||
ids.add(entry.toObjectId()); | ids.add(entry.toObjectId()); | ||||
} | } | ||||
prunePreserved(); | prunePreserved(); | ||||
long packExpireDate = getPackExpireDate(); | long packExpireDate = getPackExpireDate(); | ||||
oldPackLoop: for (PackFile oldPack : oldPacks) { | |||||
oldPackLoop: for (Pack oldPack : oldPacks) { | |||||
checkCancelled(); | checkCancelled(); | ||||
String oldName = oldPack.getPackName(); | String oldName = oldPack.getPackName(); | ||||
// check whether an old pack file is also among the list of new | // check whether an old pack file is also among the list of new | ||||
// pack files. Then we must not delete it. | // pack files. Then we must not delete it. | ||||
for (PackFile newPack : newPacks) | |||||
for (Pack newPack : newPacks) | |||||
if (oldName.equals(newPack.getPackName())) | if (oldName.equals(newPack.getPackName())) | ||||
continue oldPackLoop; | continue oldPackLoop; | ||||
*/ | */ | ||||
public void prunePacked() throws IOException { | public void prunePacked() throws IOException { | ||||
ObjectDirectory objdb = repo.getObjectDatabase(); | ObjectDirectory objdb = repo.getObjectDatabase(); | ||||
Collection<PackFile> packs = objdb.getPacks(); | |||||
Collection<Pack> packs = objdb.getPacks(); | |||||
File objects = repo.getObjectsDirectory(); | File objects = repo.getObjectsDirectory(); | ||||
String[] fanout = objects.list(); | String[] fanout = objects.list(); | ||||
continue; | continue; | ||||
} | } | ||||
boolean found = false; | boolean found = false; | ||||
for (PackFile p : packs) { | |||||
for (Pack p : packs) { | |||||
checkCancelled(); | checkCancelled(); | ||||
if (p.hasObject(id)) { | if (p.hasObject(id)) { | ||||
found = true; | found = true; | ||||
* reflog-entries or during writing to the packfiles | * reflog-entries or during writing to the packfiles | ||||
* {@link java.io.IOException} occurs | * {@link java.io.IOException} occurs | ||||
*/ | */ | ||||
public Collection<PackFile> repack() throws IOException { | |||||
Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks(); | |||||
public Collection<Pack> repack() throws IOException { | |||||
Collection<Pack> toBeDeleted = repo.getObjectDatabase().getPacks(); | |||||
long time = System.currentTimeMillis(); | long time = System.currentTimeMillis(); | ||||
Collection<Ref> refsBefore = getAllRefs(); | Collection<Ref> refsBefore = getAllRefs(); | ||||
} | } | ||||
List<ObjectIdSet> excluded = new LinkedList<>(); | List<ObjectIdSet> excluded = new LinkedList<>(); | ||||
for (PackFile f : repo.getObjectDatabase().getPacks()) { | |||||
for (Pack p : repo.getObjectDatabase().getPacks()) { | |||||
checkCancelled(); | checkCancelled(); | ||||
if (f.shouldBeKept()) | |||||
excluded.add(f.getIndex()); | |||||
if (p.shouldBeKept()) | |||||
excluded.add(p.getIndex()); | |||||
} | } | ||||
// Don't exclude tags that are also branch tips | // Don't exclude tags that are also branch tips | ||||
nonHeads.clear(); | nonHeads.clear(); | ||||
} | } | ||||
List<PackFile> ret = new ArrayList<>(2); | |||||
PackFile heads = null; | |||||
List<Pack> ret = new ArrayList<>(2); | |||||
Pack heads = null; | |||||
if (!allHeadsAndTags.isEmpty()) { | if (!allHeadsAndTags.isEmpty()) { | ||||
heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, | heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, | ||||
tagTargets, excluded); | tagTargets, excluded); | ||||
} | } | ||||
} | } | ||||
if (!nonHeads.isEmpty()) { | if (!nonHeads.isEmpty()) { | ||||
PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, | |||||
Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, | |||||
tagTargets, excluded); | tagTargets, excluded); | ||||
if (rest != null) | if (rest != null) | ||||
ret.add(rest); | ret.add(rest); | ||||
} | } | ||||
if (!txnHeads.isEmpty()) { | if (!txnHeads.isEmpty()) { | ||||
PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, | |||||
Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, | |||||
null, excluded); | null, excluded); | ||||
if (txn != null) | if (txn != null) | ||||
ret.add(txn); | ret.add(txn); | ||||
} | } | ||||
} | } | ||||
private PackFile writePack(@NonNull Set<? extends ObjectId> want, | |||||
private Pack writePack(@NonNull Set<? extends ObjectId> want, | |||||
@NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags, | @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags, | ||||
Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects) | Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects) | ||||
throws IOException { | throws IOException { | ||||
*/ | */ | ||||
public RepoStatistics getStatistics() throws IOException { | public RepoStatistics getStatistics() throws IOException { | ||||
RepoStatistics ret = new RepoStatistics(); | RepoStatistics ret = new RepoStatistics(); | ||||
Collection<PackFile> packs = repo.getObjectDatabase().getPacks(); | |||||
for (PackFile f : packs) { | |||||
ret.numberOfPackedObjects += f.getIndex().getObjectCount(); | |||||
Collection<Pack> packs = repo.getObjectDatabase().getPacks(); | |||||
for (Pack p : packs) { | |||||
ret.numberOfPackedObjects += p.getIndex().getObjectCount(); | |||||
ret.numberOfPackFiles++; | ret.numberOfPackFiles++; | ||||
ret.sizeOfPackedObjects += f.getPackFile().length(); | |||||
if (f.getBitmapIndex() != null) | |||||
ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount(); | |||||
ret.sizeOfPackedObjects += p.getPackFile().length(); | |||||
if (p.getBitmapIndex() != null) | |||||
ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); | |||||
} | } | ||||
File objDir = repo.getObjectsDirectory(); | File objDir = repo.getObjectsDirectory(); | ||||
String[] fanout = objDir.list(); | String[] fanout = objDir.list(); |
private final int headerLength; | private final int headerLength; | ||||
private final PackFile pack; | |||||
private final Pack pack; | |||||
private final FileObjectDatabase db; | private final FileObjectDatabase db; | ||||
LargePackedWholeObject(int type, long size, long objectOffset, | LargePackedWholeObject(int type, long size, long objectOffset, | ||||
int headerLength, PackFile pack, FileObjectDatabase db) { | |||||
int headerLength, Pack pack, FileObjectDatabase db) { | |||||
this.type = type; | this.type = type; | ||||
this.size = size; | this.size = size; | ||||
this.objectOffset = objectOffset; | this.objectOffset = objectOffset; |
private final String[] packNames; | private final String[] packNames; | ||||
private PackFile[] packs; | |||||
private Pack[] packs; | |||||
LocalCachedPack(ObjectDirectory odb, List<String> packNames) { | LocalCachedPack(ObjectDirectory odb, List<String> packNames) { | ||||
this.odb = odb; | this.odb = odb; | ||||
this.packNames = packNames.toArray(new String[0]); | this.packNames = packNames.toArray(new String[0]); | ||||
} | } | ||||
LocalCachedPack(List<PackFile> packs) { | |||||
LocalCachedPack(List<Pack> packs) { | |||||
odb = null; | odb = null; | ||||
packNames = null; | packNames = null; | ||||
this.packs = packs.toArray(new PackFile[0]); | |||||
this.packs = packs.toArray(new Pack[0]); | |||||
} | } | ||||
/** {@inheritDoc} */ | /** {@inheritDoc} */ | ||||
@Override | @Override | ||||
public long getObjectCount() throws IOException { | public long getObjectCount() throws IOException { | ||||
long cnt = 0; | long cnt = 0; | ||||
for (PackFile pack : getPacks()) | |||||
for (Pack pack : getPacks()) | |||||
cnt += pack.getObjectCount(); | cnt += pack.getObjectCount(); | ||||
return cnt; | return cnt; | ||||
} | } | ||||
void copyAsIs(PackOutputStream out, WindowCursor wc) | void copyAsIs(PackOutputStream out, WindowCursor wc) | ||||
throws IOException { | throws IOException { | ||||
for (PackFile pack : getPacks()) | |||||
for (Pack pack : getPacks()) | |||||
pack.copyPackAsIs(out, wc); | pack.copyPackAsIs(out, wc); | ||||
} | } | ||||
public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { | public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { | ||||
try { | try { | ||||
LocalObjectRepresentation local = (LocalObjectRepresentation) rep; | LocalObjectRepresentation local = (LocalObjectRepresentation) rep; | ||||
for (PackFile pack : getPacks()) { | |||||
for (Pack pack : getPacks()) { | |||||
if (local.pack == pack) | if (local.pack == pack) | ||||
return true; | return true; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
private PackFile[] getPacks() throws FileNotFoundException { | |||||
private Pack[] getPacks() throws FileNotFoundException { | |||||
if (packs == null) { | if (packs == null) { | ||||
PackFile[] p = new PackFile[packNames.length]; | |||||
Pack[] p = new Pack[packNames.length]; | |||||
for (int i = 0; i < packNames.length; i++) | for (int i = 0; i < packNames.length; i++) | ||||
p[i] = getPackFile(packNames[i]); | p[i] = getPackFile(packNames[i]); | ||||
packs = p; | packs = p; | ||||
return packs; | return packs; | ||||
} | } | ||||
private PackFile getPackFile(String packName) throws FileNotFoundException { | |||||
for (PackFile pack : odb.getPacks()) { | |||||
private Pack getPackFile(String packName) throws FileNotFoundException { | |||||
for (Pack pack : odb.getPacks()) { | |||||
if (packName.equals(pack.getPackName())) | if (packName.equals(pack.getPackName())) | ||||
return pack; | return pack; | ||||
} | } |
import org.eclipse.jgit.lib.ObjectId; | import org.eclipse.jgit.lib.ObjectId; | ||||
class LocalObjectRepresentation extends StoredObjectRepresentation { | class LocalObjectRepresentation extends StoredObjectRepresentation { | ||||
static LocalObjectRepresentation newWhole(PackFile f, long p, long length) { | |||||
static LocalObjectRepresentation newWhole(Pack pack, long offset, long length) { | |||||
LocalObjectRepresentation r = new LocalObjectRepresentation() { | LocalObjectRepresentation r = new LocalObjectRepresentation() { | ||||
@Override | @Override | ||||
public int getFormat() { | public int getFormat() { | ||||
return PACK_WHOLE; | return PACK_WHOLE; | ||||
} | } | ||||
}; | }; | ||||
r.pack = f; | |||||
r.offset = p; | |||||
r.pack = pack; | |||||
r.offset = offset; | |||||
r.length = length; | r.length = length; | ||||
return r; | return r; | ||||
} | } | ||||
static LocalObjectRepresentation newDelta(PackFile f, long p, long n, | |||||
static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, | |||||
ObjectId base) { | ObjectId base) { | ||||
LocalObjectRepresentation r = new Delta(); | LocalObjectRepresentation r = new Delta(); | ||||
r.pack = f; | |||||
r.offset = p; | |||||
r.length = n; | |||||
r.pack = pack; | |||||
r.offset = offset; | |||||
r.length = length; | |||||
r.baseId = base; | r.baseId = base; | ||||
return r; | return r; | ||||
} | } | ||||
static LocalObjectRepresentation newDelta(PackFile f, long p, long n, | |||||
static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, | |||||
long base) { | long base) { | ||||
LocalObjectRepresentation r = new Delta(); | LocalObjectRepresentation r = new Delta(); | ||||
r.pack = f; | |||||
r.offset = p; | |||||
r.length = n; | |||||
r.pack = pack; | |||||
r.offset = offset; | |||||
r.length = length; | |||||
r.baseOffset = base; | r.baseOffset = base; | ||||
return r; | return r; | ||||
} | } | ||||
PackFile pack; | |||||
Pack pack; | |||||
long offset; | long offset; | ||||
/** {@link ObjectToPack} for {@link ObjectDirectory}. */ | /** {@link ObjectToPack} for {@link ObjectDirectory}. */ | ||||
class LocalObjectToPack extends ObjectToPack { | class LocalObjectToPack extends ObjectToPack { | ||||
/** Pack to reuse compressed data from, otherwise null. */ | /** Pack to reuse compressed data from, otherwise null. */ | ||||
PackFile pack; | |||||
Pack pack; | |||||
/** Offset of the object's header in {@link #pack}. */ | /** Offset of the object's header in {@link #pack}. */ | ||||
long offset; | long offset; |
* This is the classical object database representation for a Git repository, | * This is the classical object database representation for a Git repository, | ||||
* where objects are stored loose by hashing them into directories by their | * where objects are stored loose by hashing them into directories by their | ||||
* {@link org.eclipse.jgit.lib.ObjectId}, or are stored in compressed containers | * {@link org.eclipse.jgit.lib.ObjectId}, or are stored in compressed containers | ||||
* known as {@link org.eclipse.jgit.internal.storage.file.PackFile}s. | |||||
* known as {@link org.eclipse.jgit.internal.storage.file.Pack}s. | |||||
* <p> | * <p> | ||||
* Optionally an object database can reference one or more alternates; other | * Optionally an object database can reference one or more alternates; other | ||||
* ObjectDatabase instances that are searched in addition to the current | * ObjectDatabase instances that are searched in addition to the current | ||||
/** {@inheritDoc} */ | /** {@inheritDoc} */ | ||||
@Override | @Override | ||||
public Collection<PackFile> getPacks() { | |||||
public Collection<Pack> getPacks() { | |||||
return packed.getPacks(); | return packed.getPacks(); | ||||
} | } | ||||
* Add a single existing pack to the list of available pack files. | * Add a single existing pack to the list of available pack files. | ||||
*/ | */ | ||||
@Override | @Override | ||||
public PackFile openPack(File pack) | |||||
public Pack openPack(File pack) | |||||
throws IOException { | throws IOException { | ||||
final String p = pack.getName(); | final String p = pack.getName(); | ||||
if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ | if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ | ||||
} | } | ||||
} | } | ||||
PackFile res = new PackFile(pack, extensions); | |||||
Pack res = new Pack(pack, extensions); | |||||
packed.insert(res); | packed.insert(res); | ||||
return res; | return res; | ||||
} | } | ||||
// PackConfig) then make sure we get rid of all handles on the file. | // PackConfig) then make sure we get rid of all handles on the file. | ||||
// Windows will not allow for rename otherwise. | // Windows will not allow for rename otherwise. | ||||
if (packFile.exists()) { | if (packFile.exists()) { | ||||
for (PackFile p : packed.getPacks()) { | |||||
for (Pack p : packed.getPacks()) { | |||||
if (packFile.getPath().equals(p.getPackFile().getPath())) { | if (packFile.getPath().equals(p.getPackFile().getPath())) { | ||||
p.close(); | p.close(); | ||||
break; | break; |
private Deflater def; | private Deflater def; | ||||
/** The pack that was created, if parsing was successful. */ | /** The pack that was created, if parsing was successful. */ | ||||
private PackFile newPack; | |||||
private Pack newPack; | |||||
private PackConfig pconfig; | private PackConfig pconfig; | ||||
} | } | ||||
/** | /** | ||||
* Get the imported {@link org.eclipse.jgit.internal.storage.file.PackFile}. | |||||
* Get the imported {@link org.eclipse.jgit.internal.storage.file.Pack}. | |||||
* <p> | * <p> | ||||
* This method is supplied only to support testing; applications shouldn't | * This method is supplied only to support testing; applications shouldn't | ||||
* be using it directly to access the imported data. | * be using it directly to access the imported data. | ||||
* | * | ||||
* @return the imported PackFile, if parsing was successful. | * @return the imported PackFile, if parsing was successful. | ||||
*/ | */ | ||||
public PackFile getPackFile() { | |||||
public Pack getPack() { | |||||
return newPack; | return newPack; | ||||
} | } | ||||
* delta packed format yielding high compression of lots of object where some | * delta packed format yielding high compression of lots of object where some | ||||
* objects are similar. | * objects are similar. | ||||
*/ | */ | ||||
public class PackFile implements Iterable<PackIndex.MutableEntry> { | |||||
private static final Logger LOG = LoggerFactory.getLogger(PackFile.class); | |||||
public class Pack implements Iterable<PackIndex.MutableEntry> { | |||||
private static final Logger LOG = LoggerFactory.getLogger(Pack.class); | |||||
/** | /** | ||||
* Sorts PackFiles to be most recently created to least recently created. | * Sorts PackFiles to be most recently created to least recently created. | ||||
*/ | */ | ||||
public static final Comparator<PackFile> SORT = (a, b) -> b.packLastModified | |||||
public static final Comparator<Pack> SORT = (a, b) -> b.packLastModified | |||||
.compareTo(a.packLastModified); | .compareTo(a.packLastModified); | ||||
private final File packFile; | private final File packFile; | ||||
* @param extensions | * @param extensions | ||||
* additional pack file extensions with the same base as the pack | * additional pack file extensions with the same base as the pack | ||||
*/ | */ | ||||
public PackFile(File packFile, int extensions) { | |||||
public Pack(File packFile, int extensions) { | |||||
this.packFile = packFile; | this.packFile = packFile; | ||||
this.fileSnapshot = PackFileSnapshot.save(packFile); | this.fileSnapshot = PackFileSnapshot.save(packFile); | ||||
this.packLastModified = fileSnapshot.lastModifiedInstant(); | this.packLastModified = fileSnapshot.lastModifiedInstant(); | ||||
@SuppressWarnings("nls") | @SuppressWarnings("nls") | ||||
@Override | @Override | ||||
public String toString() { | public String toString() { | ||||
return "PackFile [packFileName=" + packFile.getName() + ", length=" | |||||
return "Pack [packFileName=" + packFile.getName() + ", length=" | |||||
+ packFile.length() + ", packChecksum=" | + packFile.length() + ", packChecksum=" | ||||
+ ObjectId.fromRaw(packChecksum).name() + "]"; | + ObjectId.fromRaw(packChecksum).name() + "]"; | ||||
} | } |
/** | /** | ||||
* Traditional file system packed objects directory handler. | * Traditional file system packed objects directory handler. | ||||
* <p> | * <p> | ||||
* This is the {@code PackFile}s object representation for a Git object | |||||
* database, where objects are stored in compressed containers known as | |||||
* {@link org.eclipse.jgit.internal.storage.file.PackFile}s. | |||||
* This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object | |||||
* representation for a Git object database, where objects are stored in | |||||
* compressed containers known as | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack}s. | |||||
*/ | */ | ||||
class PackDirectory { | class PackDirectory { | ||||
private final static Logger LOG = LoggerFactory | private final static Logger LOG = LoggerFactory | ||||
.getLogger(PackDirectory.class); | .getLogger(PackDirectory.class); | ||||
private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, | private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, | ||||
new PackFile[0]); | |||||
new Pack[0]); | |||||
private final Config config; | private final Config config; | ||||
void close() { | void close() { | ||||
PackList packs = packList.get(); | PackList packs = packList.get(); | ||||
if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { | if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { | ||||
for (PackFile p : packs.packs) { | |||||
for (Pack p : packs.packs) { | |||||
p.close(); | p.close(); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
Collection<PackFile> getPacks() { | |||||
Collection<Pack> getPacks() { | |||||
PackList list = packList.get(); | PackList list = packList.get(); | ||||
if (list == NO_PACKS) { | if (list == NO_PACKS) { | ||||
list = scanPacks(list); | list = scanPacks(list); | ||||
} | } | ||||
PackFile[] packs = list.packs; | |||||
Pack[] packs = list.packs; | |||||
return Collections.unmodifiableCollection(Arrays.asList(packs)); | return Collections.unmodifiableCollection(Arrays.asList(packs)); | ||||
} | } | ||||
PackList pList; | PackList pList; | ||||
do { | do { | ||||
pList = packList.get(); | pList = packList.get(); | ||||
for (PackFile p : pList.packs) { | |||||
for (Pack p : pList.packs) { | |||||
try { | try { | ||||
if (p.hasObject(objectId)) { | if (p.hasObject(objectId)) { | ||||
return true; | return true; | ||||
PackList pList; | PackList pList; | ||||
do { | do { | ||||
pList = packList.get(); | pList = packList.get(); | ||||
for (PackFile p : pList.packs) { | |||||
for (Pack p : pList.packs) { | |||||
try { | try { | ||||
p.resolve(matches, id, matchLimit); | p.resolve(matches, id, matchLimit); | ||||
p.resetTransientErrorCount(); | p.resetTransientErrorCount(); | ||||
do { | do { | ||||
SEARCH: for (;;) { | SEARCH: for (;;) { | ||||
pList = packList.get(); | pList = packList.get(); | ||||
for (PackFile p : pList.packs) { | |||||
for (Pack p : pList.packs) { | |||||
try { | try { | ||||
ObjectLoader ldr = p.get(curs, objectId); | ObjectLoader ldr = p.get(curs, objectId); | ||||
p.resetTransientErrorCount(); | p.resetTransientErrorCount(); | ||||
do { | do { | ||||
SEARCH: for (;;) { | SEARCH: for (;;) { | ||||
pList = packList.get(); | pList = packList.get(); | ||||
for (PackFile p : pList.packs) { | |||||
for (Pack p : pList.packs) { | |||||
try { | try { | ||||
long len = p.getObjectSize(curs, id); | long len = p.getObjectSize(curs, id); | ||||
p.resetTransientErrorCount(); | p.resetTransientErrorCount(); | ||||
WindowCursor curs) { | WindowCursor curs) { | ||||
PackList pList = packList.get(); | PackList pList = packList.get(); | ||||
SEARCH: for (;;) { | SEARCH: for (;;) { | ||||
for (PackFile p : pList.packs) { | |||||
for (Pack p : pList.packs) { | |||||
try { | try { | ||||
LocalObjectRepresentation rep = p.representation(curs, otp); | LocalObjectRepresentation rep = p.representation(curs, otp); | ||||
p.resetTransientErrorCount(); | p.resetTransientErrorCount(); | ||||
} | } | ||||
} | } | ||||
private void handlePackError(IOException e, PackFile p) { | |||||
private void handlePackError(IOException e, Pack p) { | |||||
String warnTmpl = null; | String warnTmpl = null; | ||||
int transientErrorCount = 0; | int transientErrorCount = 0; | ||||
String errTmpl = JGitText.get().exceptionWhileReadingPack; | String errTmpl = JGitText.get().exceptionWhileReadingPack; | ||||
&& old != scanPacks(old); | && old != scanPacks(old); | ||||
} | } | ||||
void insert(PackFile pf) { | |||||
void insert(Pack pack) { | |||||
PackList o, n; | PackList o, n; | ||||
do { | do { | ||||
o = packList.get(); | o = packList.get(); | ||||
// (picked up by a concurrent thread that did a scan?) we | // (picked up by a concurrent thread that did a scan?) we | ||||
// do not want to insert it a second time. | // do not want to insert it a second time. | ||||
// | // | ||||
final PackFile[] oldList = o.packs; | |||||
final String name = pf.getPackFile().getName(); | |||||
for (PackFile p : oldList) { | |||||
final Pack[] oldList = o.packs; | |||||
final String name = pack.getPackFile().getName(); | |||||
for (Pack p : oldList) { | |||||
if (name.equals(p.getPackFile().getName())) { | if (name.equals(p.getPackFile().getName())) { | ||||
return; | return; | ||||
} | } | ||||
} | } | ||||
final PackFile[] newList = new PackFile[1 + oldList.length]; | |||||
newList[0] = pf; | |||||
final Pack[] newList = new Pack[1 + oldList.length]; | |||||
newList[0] = pack; | |||||
System.arraycopy(oldList, 0, newList, 1, oldList.length); | System.arraycopy(oldList, 0, newList, 1, oldList.length); | ||||
n = new PackList(o.snapshot, newList); | n = new PackList(o.snapshot, newList); | ||||
} while (!packList.compareAndSet(o, n)); | } while (!packList.compareAndSet(o, n)); | ||||
} | } | ||||
private void remove(PackFile deadPack) { | |||||
private void remove(Pack deadPack) { | |||||
PackList o, n; | PackList o, n; | ||||
do { | do { | ||||
o = packList.get(); | o = packList.get(); | ||||
final PackFile[] oldList = o.packs; | |||||
final Pack[] oldList = o.packs; | |||||
final int j = indexOf(oldList, deadPack); | final int j = indexOf(oldList, deadPack); | ||||
if (j < 0) { | if (j < 0) { | ||||
break; | break; | ||||
} | } | ||||
final PackFile[] newList = new PackFile[oldList.length - 1]; | |||||
final Pack[] newList = new Pack[oldList.length - 1]; | |||||
System.arraycopy(oldList, 0, newList, 0, j); | System.arraycopy(oldList, 0, newList, 0, j); | ||||
System.arraycopy(oldList, j + 1, newList, j, newList.length - j); | System.arraycopy(oldList, j + 1, newList, j, newList.length - j); | ||||
n = new PackList(o.snapshot, newList); | n = new PackList(o.snapshot, newList); | ||||
deadPack.close(); | deadPack.close(); | ||||
} | } | ||||
private static int indexOf(PackFile[] list, PackFile pack) { | |||||
private static int indexOf(Pack[] list, Pack pack) { | |||||
for (int i = 0; i < list.length; i++) { | for (int i = 0; i < list.length; i++) { | ||||
if (list[i] == pack) { | if (list[i] == pack) { | ||||
return i; | return i; | ||||
} | } | ||||
private PackList scanPacksImpl(PackList old) { | private PackList scanPacksImpl(PackList old) { | ||||
final Map<String, PackFile> forReuse = reuseMap(old); | |||||
final Map<String, Pack> forReuse = reuseMap(old); | |||||
final FileSnapshot snapshot = FileSnapshot.save(directory); | final FileSnapshot snapshot = FileSnapshot.save(directory); | ||||
final Set<String> names = listPackDirectory(); | final Set<String> names = listPackDirectory(); | ||||
final List<PackFile> list = new ArrayList<>(names.size() >> 2); | |||||
final List<Pack> list = new ArrayList<>(names.size() >> 2); | |||||
boolean foundNew = false; | boolean foundNew = false; | ||||
for (String indexName : names) { | for (String indexName : names) { | ||||
// Must match "pack-[0-9a-f]{40}.idx" to be an index. | // Must match "pack-[0-9a-f]{40}.idx" to be an index. | ||||
final String packName = base + PACK.getExtension(); | final String packName = base + PACK.getExtension(); | ||||
final File packFile = new File(directory, packName); | final File packFile = new File(directory, packName); | ||||
final PackFile oldPack = forReuse.get(packName); | |||||
final Pack oldPack = forReuse.get(packName); | |||||
if (oldPack != null | if (oldPack != null | ||||
&& !oldPack.getFileSnapshot().isModified(packFile)) { | && !oldPack.getFileSnapshot().isModified(packFile)) { | ||||
forReuse.remove(packName); | forReuse.remove(packName); | ||||
continue; | continue; | ||||
} | } | ||||
list.add(new PackFile(packFile, extensions)); | |||||
list.add(new Pack(packFile, extensions)); | |||||
foundNew = true; | foundNew = true; | ||||
} | } | ||||
return old; | return old; | ||||
} | } | ||||
for (PackFile p : forReuse.values()) { | |||||
for (Pack p : forReuse.values()) { | |||||
p.close(); | p.close(); | ||||
} | } | ||||
return new PackList(snapshot, NO_PACKS.packs); | return new PackList(snapshot, NO_PACKS.packs); | ||||
} | } | ||||
final PackFile[] r = list.toArray(new PackFile[0]); | |||||
Arrays.sort(r, PackFile.SORT); | |||||
final Pack[] r = list.toArray(new Pack[0]); | |||||
Arrays.sort(r, Pack.SORT); | |||||
return new PackList(snapshot, r); | return new PackList(snapshot, r); | ||||
} | } | ||||
private static Map<String, PackFile> reuseMap(PackList old) { | |||||
final Map<String, PackFile> forReuse = new HashMap<>(); | |||||
for (PackFile p : old.packs) { | |||||
private static Map<String, Pack> reuseMap(PackList old) { | |||||
final Map<String, Pack> forReuse = new HashMap<>(); | |||||
for (Pack p : old.packs) { | |||||
if (p.invalid()) { | if (p.invalid()) { | ||||
// The pack instance is corrupted, and cannot be safely used | // The pack instance is corrupted, and cannot be safely used | ||||
// again. Do not include it in our reuse map. | // again. Do not include it in our reuse map. | ||||
continue; | continue; | ||||
} | } | ||||
final PackFile prior = forReuse.put(p.getPackFile().getName(), p); | |||||
final Pack prior = forReuse.put(p.getPackFile().getName(), p); | |||||
if (prior != null) { | if (prior != null) { | ||||
// This should never occur. It should be impossible for us | // This should never occur. It should be impossible for us | ||||
// to have two pack files with the same name, as all of them | // to have two pack files with the same name, as all of them | ||||
/** State just before reading the pack directory. */ | /** State just before reading the pack directory. */ | ||||
final FileSnapshot snapshot; | final FileSnapshot snapshot; | ||||
/** All known packs, sorted by {@link PackFile#SORT}. */ | |||||
final PackFile[] packs; | |||||
/** All known packs, sorted by {@link Pack#SORT}. */ | |||||
final Pack[] packs; | |||||
PackList(FileSnapshot monitor, PackFile[] packs) { | |||||
PackList(FileSnapshot monitor, Pack[] packs) { | |||||
this.snapshot = monitor; | this.snapshot = monitor; | ||||
this.packs = packs; | this.packs = packs; | ||||
} | } |
/** | /** | ||||
* Access path to locate objects by {@link org.eclipse.jgit.lib.ObjectId} in a | * Access path to locate objects by {@link org.eclipse.jgit.lib.ObjectId} in a | ||||
* {@link org.eclipse.jgit.internal.storage.file.PackFile}. | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack}. | |||||
* <p> | * <p> | ||||
* Indexes are strictly redundant information in that we can rebuild all of the | * Indexes are strictly redundant information in that we can rebuild all of the | ||||
* data held in the index file from the on disk representation of the pack file | * data held in the index file from the on disk representation of the pack file |
/** | /** | ||||
* Creates a table of contents to support random access by | * Creates a table of contents to support random access by | ||||
* {@link org.eclipse.jgit.internal.storage.file.PackFile}. | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack}. | |||||
* <p> | * <p> | ||||
* Pack index files (the <code>.idx</code> suffix in a pack file pair) provides | * Pack index files (the <code>.idx</code> suffix in a pack file pair) provides | ||||
* random access to any object in the pack by associating an ObjectId to the | * random access to any object in the pack by associating an ObjectId to the |
class PackInputStream extends InputStream { | class PackInputStream extends InputStream { | ||||
private final WindowCursor wc; | private final WindowCursor wc; | ||||
private final PackFile pack; | |||||
private final Pack pack; | |||||
private long pos; | private long pos; | ||||
PackInputStream(PackFile pack, long pos, WindowCursor wc) | |||||
PackInputStream(Pack pack, long pos, WindowCursor wc) | |||||
throws IOException { | throws IOException { | ||||
this.pack = pack; | this.pack = pack; | ||||
this.pos = pos; | this.pos = pos; |
import org.eclipse.jgit.util.FileUtils; | import org.eclipse.jgit.util.FileUtils; | ||||
/** | /** | ||||
* Keeps track of a {@link org.eclipse.jgit.internal.storage.file.PackFile}'s | |||||
* Keeps track of a {@link org.eclipse.jgit.internal.storage.file.Pack}'s | |||||
* associated <code>.keep</code> file. | * associated <code>.keep</code> file. | ||||
*/ | */ | ||||
public class PackLock { | public class PackLock { |
* </p> | * </p> | ||||
* | * | ||||
* @see PackIndex | * @see PackIndex | ||||
* @see PackFile | |||||
* @see Pack | |||||
*/ | */ | ||||
public class PackReverseIndex { | public class PackReverseIndex { | ||||
/** Index we were created from, and that has our ObjectId data. */ | /** Index we were created from, and that has our ObjectId data. */ |
import org.eclipse.jgit.util.Monitoring; | import org.eclipse.jgit.util.Monitoring; | ||||
/** | /** | ||||
* Caches slices of a {@link org.eclipse.jgit.internal.storage.file.PackFile} in | |||||
* Caches slices of a {@link org.eclipse.jgit.internal.storage.file.Pack} in | |||||
* memory for faster read access. | * memory for faster read access. | ||||
* <p> | * <p> | ||||
* The WindowCache serves as a Java based "buffer cache", loading segments of a | * The WindowCache serves as a Java based "buffer cache", loading segments of a | ||||
* only tiny slices of a file, the WindowCache tries to smooth out these tiny | * only tiny slices of a file, the WindowCache tries to smooth out these tiny | ||||
* reads into larger block-sized IO operations. | * reads into larger block-sized IO operations. | ||||
* <p> | * <p> | ||||
* Whenever a cache miss occurs, {@link #load(PackFile, long)} is invoked by | |||||
* Whenever a cache miss occurs, {@link #load(Pack, long)} is invoked by | |||||
* exactly one thread for the given <code>(PackFile,position)</code> key tuple. | * exactly one thread for the given <code>(PackFile,position)</code> key tuple. | ||||
* This is ensured by an array of locks, with the tuple hashed to a lock | * This is ensured by an array of locks, with the tuple hashed to a lock | ||||
* instance. | * instance. | ||||
* <p> | * <p> | ||||
* This cache has an implementation rule such that: | * This cache has an implementation rule such that: | ||||
* <ul> | * <ul> | ||||
* <li>{@link #load(PackFile, long)} is invoked by at most one thread at a time | |||||
* <li>{@link #load(Pack, long)} is invoked by at most one thread at a time | |||||
* for a given <code>(PackFile,position)</code> tuple.</li> | * for a given <code>(PackFile,position)</code> tuple.</li> | ||||
* <li>For every <code>load()</code> invocation there is exactly one | * <li>For every <code>load()</code> invocation there is exactly one | ||||
* {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a | |||||
* {@link #createRef(Pack, long, ByteWindow)} invocation to wrap a | |||||
* SoftReference or a StrongReference around the cached entity.</li> | * SoftReference or a StrongReference around the cached entity.</li> | ||||
* <li>For every Reference created by <code>createRef()</code> there will be | * <li>For every Reference created by <code>createRef()</code> there will be | ||||
* exactly one call to {@link #clear(PageRef)} to cleanup any resources associated | * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated | ||||
* </ul> | * </ul> | ||||
* <p> | * <p> | ||||
* Therefore, it is safe to perform resource accounting increments during the | * Therefore, it is safe to perform resource accounting increments during the | ||||
* {@link #load(PackFile, long)} or | |||||
* {@link #createRef(PackFile, long, ByteWindow)} methods, and matching | |||||
* {@link #load(Pack, long)} or | |||||
* {@link #createRef(Pack, long, ByteWindow)} methods, and matching | |||||
* decrements during {@link #clear(PageRef)}. Implementors may need to override | * decrements during {@link #clear(PageRef)}. Implementors may need to override | ||||
* {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional | |||||
* {@link #createRef(Pack, long, ByteWindow)} in order to embed additional | |||||
* accounting information into an implementation specific | * accounting information into an implementation specific | ||||
* {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as | * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as | ||||
* the cached entity may have already been evicted by the JRE's garbage | * the cached entity may have already been evicted by the JRE's garbage | ||||
* @param delta | * @param delta | ||||
* delta of cached bytes | * delta of cached bytes | ||||
*/ | */ | ||||
void recordOpenBytes(PackFile pack, int delta); | |||||
void recordOpenBytes(Pack pack, int delta); | |||||
/** | /** | ||||
* Returns a snapshot of this recorder's stats. Note that this may be an | * Returns a snapshot of this recorder's stats. Note that this may be an | ||||
} | } | ||||
@Override | @Override | ||||
public void recordOpenBytes(PackFile pack, int delta) { | |||||
public void recordOpenBytes(Pack pack, int delta) { | |||||
openByteCount.add(delta); | openByteCount.add(delta); | ||||
String repositoryId = repositoryId(pack); | String repositoryId = repositoryId(pack); | ||||
LongAdder la = openByteCountPerRepository | LongAdder la = openByteCountPerRepository | ||||
} | } | ||||
} | } | ||||
private static String repositoryId(PackFile pack) { | |||||
// use repository's gitdir since packfile doesn't know its | |||||
// repository | |||||
private static String repositoryId(Pack pack) { | |||||
// use repository's gitdir since Pack doesn't know its repository | |||||
return pack.getPackFile().getParentFile().getParentFile() | return pack.getPackFile().getParentFile().getParentFile() | ||||
.getParent(); | .getParent(); | ||||
} | } | ||||
return cache.publishMBeanIfNeeded(); | return cache.publishMBeanIfNeeded(); | ||||
} | } | ||||
static final ByteWindow get(PackFile pack, long offset) | |||||
static final ByteWindow get(Pack pack, long offset) | |||||
throws IOException { | throws IOException { | ||||
final WindowCache c = cache; | final WindowCache c = cache; | ||||
final ByteWindow r = c.getOrLoad(pack, c.toStart(offset)); | final ByteWindow r = c.getOrLoad(pack, c.toStart(offset)); | ||||
return r; | return r; | ||||
} | } | ||||
static final void purge(PackFile pack) { | |||||
static final void purge(Pack pack) { | |||||
cache.removeAll(pack); | cache.removeAll(pack); | ||||
} | } | ||||
return packHash + (int) (off >>> windowSizeShift); | return packHash + (int) (off >>> windowSizeShift); | ||||
} | } | ||||
private ByteWindow load(PackFile pack, long offset) throws IOException { | |||||
private ByteWindow load(Pack pack, long offset) throws IOException { | |||||
long startTime = System.nanoTime(); | long startTime = System.nanoTime(); | ||||
if (pack.beginWindowCache()) | if (pack.beginWindowCache()) | ||||
statsRecorder.recordOpenFiles(1); | statsRecorder.recordOpenFiles(1); | ||||
} | } | ||||
} | } | ||||
private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) { | |||||
private PageRef<ByteWindow> createRef(Pack p, long o, ByteWindow v) { | |||||
final PageRef<ByteWindow> ref = useStrongRefs | final PageRef<ByteWindow> ref = useStrongRefs | ||||
? new StrongRef(p, o, v, queue) | ? new StrongRef(p, o, v, queue) | ||||
: new SoftRef(p, o, v, (SoftCleanupQueue) queue); | : new SoftRef(p, o, v, (SoftCleanupQueue) queue); | ||||
close(ref.getPack()); | close(ref.getPack()); | ||||
} | } | ||||
private void close(PackFile pack) { | |||||
private void close(Pack pack) { | |||||
if (pack.endWindowCache()) { | if (pack.endWindowCache()) { | ||||
statsRecorder.recordOpenFiles(-1); | statsRecorder.recordOpenFiles(-1); | ||||
} | } | ||||
* @return the object reference. | * @return the object reference. | ||||
* @throws IOException | * @throws IOException | ||||
* the object reference was not in the cache and could not be | * the object reference was not in the cache and could not be | ||||
* obtained by {@link #load(PackFile, long)}. | |||||
* obtained by {@link #load(Pack, long)}. | |||||
*/ | */ | ||||
private ByteWindow getOrLoad(PackFile pack, long position) | |||||
private ByteWindow getOrLoad(Pack pack, long position) | |||||
throws IOException { | throws IOException { | ||||
final int slot = slot(pack, position); | final int slot = slot(pack, position); | ||||
final Entry e1 = table.get(slot); | final Entry e1 = table.get(slot); | ||||
return v; | return v; | ||||
} | } | ||||
private ByteWindow scan(Entry n, PackFile pack, long position) { | |||||
private ByteWindow scan(Entry n, Pack pack, long position) { | |||||
for (; n != null; n = n.next) { | for (; n != null; n = n.next) { | ||||
final PageRef<ByteWindow> r = n.ref; | final PageRef<ByteWindow> r = n.ref; | ||||
if (r.getPack() == pack && r.getPosition() == position) { | if (r.getPack() == pack && r.getPosition() == position) { | ||||
/** | /** | ||||
* Clear all entries related to a single file. | * Clear all entries related to a single file. | ||||
* <p> | * <p> | ||||
* Typically this method is invoked during {@link PackFile#close()}, when we | |||||
* Typically this method is invoked during {@link Pack#close()}, when we | |||||
* know the pack is never going to be useful to us again (for example, it no | * know the pack is never going to be useful to us again (for example, it no | ||||
* longer exists on disk). A concurrent reader loading an entry from this | * longer exists on disk). A concurrent reader loading an entry from this | ||||
* same pack may cause the pack to become stuck in the cache anyway. | * same pack may cause the pack to become stuck in the cache anyway. | ||||
* @param pack | * @param pack | ||||
* the file to purge all entries of. | * the file to purge all entries of. | ||||
*/ | */ | ||||
private void removeAll(PackFile pack) { | |||||
private void removeAll(Pack pack) { | |||||
for (int s = 0; s < tableSize; s++) { | for (int s = 0; s < tableSize; s++) { | ||||
final Entry e1 = table.get(s); | final Entry e1 = table.get(s); | ||||
boolean hasDead = false; | boolean hasDead = false; | ||||
queue.gc(); | queue.gc(); | ||||
} | } | ||||
private int slot(PackFile pack, long position) { | |||||
private int slot(Pack pack, long position) { | |||||
return (hash(pack.hash, position) >>> 1) % tableSize; | return (hash(pack.hash, position) >>> 1) % tableSize; | ||||
} | } | ||||
private Lock lock(PackFile pack, long position) { | |||||
private Lock lock(Pack pack, long position) { | |||||
return locks[(hash(pack.hash, position) >>> 1) % locks.length]; | return locks[(hash(pack.hash, position) >>> 1) % locks.length]; | ||||
} | } | ||||
boolean kill(); | boolean kill(); | ||||
/** | /** | ||||
* Get the packfile the referenced cache page is allocated for | |||||
* Get the {@link org.eclipse.jgit.internal.storage.file.Pack} the | |||||
* referenced cache page is allocated for | |||||
* | * | ||||
* @return the packfile the referenced cache page is allocated for | |||||
* @return the {@link org.eclipse.jgit.internal.storage.file.Pack} the | |||||
* referenced cache page is allocated for | |||||
*/ | */ | ||||
PackFile getPack(); | |||||
Pack getPack(); | |||||
/** | /** | ||||
* Get the position of the referenced cache page in the packfile | |||||
* Get the position of the referenced cache page in the | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack} | |||||
* | * | ||||
* @return the position of the referenced cache page in the packfile | |||||
* @return the position of the referenced cache page in the | |||||
* {@link org.eclipse.jgit.internal.storage.file.Pack} | |||||
*/ | */ | ||||
long getPosition(); | long getPosition(); | ||||
/** A soft reference wrapped around a cached object. */ | /** A soft reference wrapped around a cached object. */ | ||||
private static class SoftRef extends SoftReference<ByteWindow> | private static class SoftRef extends SoftReference<ByteWindow> | ||||
implements PageRef<ByteWindow> { | implements PageRef<ByteWindow> { | ||||
private final PackFile pack; | |||||
private final Pack pack; | |||||
private final long position; | private final long position; | ||||
private long lastAccess; | private long lastAccess; | ||||
protected SoftRef(final PackFile pack, final long position, | |||||
protected SoftRef(final Pack pack, final long position, | |||||
final ByteWindow v, final SoftCleanupQueue queue) { | final ByteWindow v, final SoftCleanupQueue queue) { | ||||
super(v, queue); | super(v, queue); | ||||
this.pack = pack; | this.pack = pack; | ||||
} | } | ||||
@Override | @Override | ||||
public PackFile getPack() { | |||||
public Pack getPack() { | |||||
return pack; | return pack; | ||||
} | } | ||||
private static class StrongRef implements PageRef<ByteWindow> { | private static class StrongRef implements PageRef<ByteWindow> { | ||||
private ByteWindow referent; | private ByteWindow referent; | ||||
private final PackFile pack; | |||||
private final Pack pack; | |||||
private final long position; | private final long position; | ||||
private CleanupQueue queue; | private CleanupQueue queue; | ||||
protected StrongRef(final PackFile pack, final long position, | |||||
protected StrongRef(final Pack pack, final long position, | |||||
final ByteWindow v, final CleanupQueue queue) { | final ByteWindow v, final CleanupQueue queue) { | ||||
this.pack = pack; | this.pack = pack; | ||||
this.position = position; | this.position = position; | ||||
} | } | ||||
@Override | @Override | ||||
public PackFile getPack() { | |||||
public Pack getPack() { | |||||
return pack; | return pack; | ||||
} | } | ||||
/** {@inheritDoc} */ | /** {@inheritDoc} */ | ||||
@Override | @Override | ||||
public BitmapIndex getBitmapIndex() throws IOException { | public BitmapIndex getBitmapIndex() throws IOException { | ||||
for (PackFile pack : db.getPacks()) { | |||||
for (Pack pack : db.getPacks()) { | |||||
PackBitmapIndex index = pack.getBitmapIndex(); | PackBitmapIndex index = pack.getBitmapIndex(); | ||||
if (index != null) | if (index != null) | ||||
return new BitmapIndexImpl(index); | return new BitmapIndexImpl(index); | ||||
@Override | @Override | ||||
public Collection<CachedPack> getCachedPacksAndUpdate( | public Collection<CachedPack> getCachedPacksAndUpdate( | ||||
BitmapBuilder needBitmap) throws IOException { | BitmapBuilder needBitmap) throws IOException { | ||||
for (PackFile pack : db.getPacks()) { | |||||
for (Pack pack : db.getPacks()) { | |||||
PackBitmapIndex index = pack.getBitmapIndex(); | PackBitmapIndex index = pack.getBitmapIndex(); | ||||
if (needBitmap.removeAllOrNone(index)) | if (needBitmap.removeAllOrNone(index)) | ||||
return Collections.<CachedPack> singletonList( | return Collections.<CachedPack> singletonList( | ||||
* this cursor does not match the provider or id and the proper | * this cursor does not match the provider or id and the proper | ||||
* window could not be acquired through the provider's cache. | * window could not be acquired through the provider's cache. | ||||
*/ | */ | ||||
int copy(final PackFile pack, long position, final byte[] dstbuf, | |||||
int copy(final Pack pack, long position, final byte[] dstbuf, | |||||
int dstoff, final int cnt) throws IOException { | int dstoff, final int cnt) throws IOException { | ||||
final long length = pack.length; | final long length = pack.length; | ||||
int need = cnt; | int need = cnt; | ||||
((LocalCachedPack) pack).copyAsIs(out, this); | ((LocalCachedPack) pack).copyAsIs(out, this); | ||||
} | } | ||||
void copyPackAsIs(final PackFile pack, final long length, | |||||
void copyPackAsIs(final Pack pack, final long length, | |||||
final PackOutputStream out) throws IOException { | final PackOutputStream out) throws IOException { | ||||
long position = 12; | long position = 12; | ||||
long remaining = length - (12 + 20); | long remaining = length - (12 + 20); | ||||
* the inflater encountered an invalid chunk of data. Data | * the inflater encountered an invalid chunk of data. Data | ||||
* stream corruption is likely. | * stream corruption is likely. | ||||
*/ | */ | ||||
int inflate(final PackFile pack, long position, final byte[] dstbuf, | |||||
int inflate(final Pack pack, long position, final byte[] dstbuf, | |||||
boolean headerOnly) throws IOException, DataFormatException { | boolean headerOnly) throws IOException, DataFormatException { | ||||
prepareInflater(); | prepareInflater(); | ||||
pin(pack, position); | pin(pack, position); | ||||
} | } | ||||
} | } | ||||
ByteArrayWindow quickCopy(PackFile p, long pos, long cnt) | |||||
ByteArrayWindow quickCopy(Pack p, long pos, long cnt) | |||||
throws IOException { | throws IOException { | ||||
pin(p, pos); | pin(p, pos); | ||||
if (window instanceof ByteArrayWindow | if (window instanceof ByteArrayWindow | ||||
inf.reset(); | inf.reset(); | ||||
} | } | ||||
void pin(PackFile pack, long position) | |||||
void pin(Pack pack, long position) | |||||
throws IOException { | throws IOException { | ||||
final ByteWindow w = window; | final ByteWindow w = window; | ||||
if (w == null || !w.contains(pack, position)) { | if (w == null || !w.contains(pack, position)) { |