An unreferenced object might appear in a pack. This could only happen because it was previously referenced, and then later that reference was removed. When we gc, we copy the referenced objects into a new pack, and delete the old pack. This would remove the unreferenced object. Now we first create a loose object from any unreferenced object in the doomed pack. This kicks off the two-week grace period for that object, after which it will be collected if it's not referenced. This matches the behavior of regular git. Change-Id: I59539aca1d0d83622c41aa9bfbdd72fa868ee9fb Signed-off-by: David Turner <dturner@twosigma.com> Signed-off-by: Jonathan Nieder <jrn@google.com>tags/v4.7.0.201704051617-r
@@ -55,6 +55,7 @@ import java.util.Date; | |||
import java.util.List; | |||
import org.eclipse.jgit.junit.TestRepository.BranchBuilder; | |||
import org.eclipse.jgit.lib.RefUpdate; | |||
import org.eclipse.jgit.revwalk.RevCommit; | |||
import org.eclipse.jgit.storage.pack.PackConfig; | |||
import org.junit.Test; | |||
@@ -188,16 +189,26 @@ public class GcBasicPackingTest extends GcTestCase { | |||
BranchBuilder bb = tr.branch("refs/heads/master"); | |||
bb.commit().message("M").add("M", "M").create(); | |||
String tempRef = "refs/heads/soon-to-be-unreferenced"; | |||
BranchBuilder bb2 = tr.branch(tempRef); | |||
bb2.commit().message("M").add("M", "M").create(); | |||
gc.setExpireAgeMillis(0); | |||
gc.gc(); | |||
stats = gc.getStatistics(); | |||
assertEquals(0, stats.numberOfLooseObjects); | |||
assertEquals(3, stats.numberOfPackedObjects); | |||
assertEquals(4, stats.numberOfPackedObjects); | |||
assertEquals(1, stats.numberOfPackFiles); | |||
File oldPackfile = tr.getRepository().getObjectDatabase().getPacks() | |||
.iterator().next().getPackFile(); | |||
fsTick(); | |||
// delete the temp ref, orphaning its commit | |||
RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); | |||
update.setForceUpdate(true); | |||
update.delete(); | |||
bb.commit().message("B").add("B", "Q").create(); | |||
// The old packfile is too young to be deleted. We should end up with | |||
@@ -208,7 +219,7 @@ public class GcBasicPackingTest extends GcTestCase { | |||
assertEquals(0, stats.numberOfLooseObjects); | |||
// if objects exist in multiple packFiles then they are counted multiple | |||
// times | |||
assertEquals(9, stats.numberOfPackedObjects); | |||
assertEquals(10, stats.numberOfPackedObjects); | |||
assertEquals(2, stats.numberOfPackFiles); | |||
// repack again but now without a grace period for loose objects. Since | |||
@@ -219,15 +230,19 @@ public class GcBasicPackingTest extends GcTestCase { | |||
assertEquals(0, stats.numberOfLooseObjects); | |||
// if objects exist in multiple packFiles then they are counted multiple | |||
// times | |||
assertEquals(9, stats.numberOfPackedObjects); | |||
assertEquals(10, stats.numberOfPackedObjects); | |||
assertEquals(2, stats.numberOfPackFiles); | |||
// repack again but now without a grace period for packfiles. We should | |||
// end up with one packfile | |||
gc.setPackExpireAgeMillis(0); | |||
// we want to keep newly-loosened objects though | |||
gc.setExpireAgeMillis(-1); | |||
gc.gc(); | |||
stats = gc.getStatistics(); | |||
assertEquals(0, stats.numberOfLooseObjects); | |||
assertEquals(1, stats.numberOfLooseObjects); | |||
// if objects exist in multiple packFiles then they are counted multiple | |||
// times | |||
assertEquals(6, stats.numberOfPackedObjects); |
@@ -81,6 +81,8 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; | |||
import org.eclipse.jgit.errors.MissingObjectException; | |||
import org.eclipse.jgit.errors.NoWorkTreeException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory; | |||
import org.eclipse.jgit.internal.storage.file.ObjectDirectoryInserter; | |||
import org.eclipse.jgit.internal.storage.pack.PackExt; | |||
import org.eclipse.jgit.internal.storage.pack.PackWriter; | |||
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; | |||
@@ -90,6 +92,8 @@ import org.eclipse.jgit.lib.FileMode; | |||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.lib.ObjectIdSet; | |||
import org.eclipse.jgit.lib.ObjectLoader; | |||
import org.eclipse.jgit.lib.ObjectReader; | |||
import org.eclipse.jgit.lib.ProgressMonitor; | |||
import org.eclipse.jgit.lib.Ref; | |||
import org.eclipse.jgit.lib.Ref.Storage; | |||
@@ -208,6 +212,26 @@ public class GC { | |||
return newPacks; | |||
} | |||
/** | |||
* Loosen objects in a pack file which are not also in the newly-created | |||
* pack files. | |||
*/ | |||
private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing) | |||
throws IOException { | |||
for (PackIndex.MutableEntry entry : pack) { | |||
ObjectId oid = entry.toObjectId(); | |||
if (existing.contains(oid)) { | |||
continue; | |||
} | |||
existing.add(oid); | |||
ObjectLoader loader = reader.open(oid); | |||
inserter.insert(loader.getType(), | |||
loader.getSize(), | |||
loader.openStream(), | |||
true /* create this object even though it's a duplicate */); | |||
} | |||
} | |||
/** | |||
* Delete old pack files. What is 'old' is defined by specifying a set of | |||
* old pack files and a set of new pack files. Each pack file contained in | |||
@@ -215,6 +239,9 @@ public class GC { | |||
* preserveOldPacks is set, keep a copy of the pack file in the preserve | |||
* directory. If an expirationDate is set then pack files which are younger | |||
* than the expirationDate will not be deleted nor preserved. | |||
* <p> | |||
* If we're not immediately expiring loose objects, loosen any objects | |||
* in the old pack files which aren't in the new pack files. | |||
* | |||
* @param oldPacks | |||
* @param newPacks | |||
@@ -223,6 +250,17 @@ public class GC { | |||
*/ | |||
private void deleteOldPacks(Collection<PackFile> oldPacks, | |||
Collection<PackFile> newPacks) throws ParseException, IOException { | |||
HashSet<ObjectId> ids = new HashSet<>(); | |||
for (PackFile pack : newPacks) { | |||
for (PackIndex.MutableEntry entry : pack) { | |||
ids.add(entry.toObjectId()); | |||
} | |||
} | |||
ObjectReader reader = repo.newObjectReader(); | |||
ObjectDirectory dir = repo.getObjectDatabase(); | |||
ObjectDirectoryInserter inserter = dir.newInserter(); | |||
boolean shouldLoosen = getExpireDate() < Long.MAX_VALUE; | |||
prunePreserved(); | |||
long packExpireDate = getPackExpireDate(); | |||
oldPackLoop: for (PackFile oldPack : oldPacks) { | |||
@@ -237,6 +275,9 @@ public class GC { | |||
&& repo.getFS().lastModified( | |||
oldPack.getPackFile()) < packExpireDate) { | |||
oldPack.close(); | |||
if (shouldLoosen) { | |||
loosen(inserter, reader, oldPack, ids); | |||
} | |||
prunePack(oldName); | |||
} | |||
} |
@@ -86,34 +86,56 @@ class ObjectDirectoryInserter extends ObjectInserter { | |||
@Override | |||
public ObjectId insert(int type, byte[] data, int off, int len) | |||
throws IOException { | |||
return insert(type, data, off, len, false); | |||
} | |||
/** | |||
* Insert a loose object into the database. If createDuplicate is | |||
* true, write the loose object even if we already have it in the | |||
* loose or packed ODB. | |||
*/ | |||
private ObjectId insert( | |||
int type, byte[] data, int off, int len, boolean createDuplicate) | |||
throws IOException { | |||
ObjectId id = idFor(type, data, off, len); | |||
if (db.has(id)) { | |||
if (!createDuplicate && db.has(id)) { | |||
return id; | |||
} else { | |||
File tmp = toTemp(type, data, off, len); | |||
return insertOneObject(tmp, id); | |||
return insertOneObject(tmp, id, createDuplicate); | |||
} | |||
} | |||
@Override | |||
public ObjectId insert(final int type, long len, final InputStream is) | |||
throws IOException { | |||
return insert(type, len, is, false); | |||
} | |||
/** | |||
* Insert a loose object into the database. If createDuplicate is | |||
* true, write the loose object even if we already have it in the | |||
* loose or packed ODB. | |||
*/ | |||
ObjectId insert(int type, long len, InputStream is, boolean createDuplicate) | |||
throws IOException { | |||
if (len <= buffer().length) { | |||
byte[] buf = buffer(); | |||
int actLen = IO.readFully(is, buf, 0); | |||
return insert(type, buf, 0, actLen); | |||
return insert(type, buf, 0, actLen, createDuplicate); | |||
} else { | |||
MessageDigest md = digest(); | |||
File tmp = toTemp(md, type, len, is); | |||
ObjectId id = ObjectId.fromRaw(md.digest()); | |||
return insertOneObject(tmp, id); | |||
return insertOneObject(tmp, id, createDuplicate); | |||
} | |||
} | |||
private ObjectId insertOneObject(final File tmp, final ObjectId id) | |||
private ObjectId insertOneObject( | |||
File tmp, ObjectId id, boolean createDuplicate) | |||
throws IOException, ObjectWritingException { | |||
switch (db.insertUnpackedObject(tmp, id, false /* no duplicate */)) { | |||
switch (db.insertUnpackedObject(tmp, id, createDuplicate)) { | |||
case INSERTED: | |||
case EXISTS_PACKED: | |||
case EXISTS_LOOSE: |