import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.junit.MockSystemReader;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BatchRefUpdate;
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevBlob;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.pack.PackConfig;
+import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.util.SystemReader;
import org.junit.After;
import org.junit.Before;
assertEquals(2, odb.getPacks().length);
}
+ @SuppressWarnings("boxing")
+ @Test
+ public void producesNewReftable() throws Exception {
+ String master = "refs/heads/master";
+ RevCommit commit0 = commit().message("0").create();
+ RevCommit commit1 = commit().message("1").parent(commit0).create();
+
+ BatchRefUpdate bru = git.getRepository().getRefDatabase()
+ .newBatchUpdate();
+ bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
+ for (int i = 1; i <= 5100; i++) {
+ bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
+ String.format("refs/pulls/%04d", i)));
+ }
+ try (RevWalk rw = new RevWalk(git.getRepository())) {
+ bru.execute(rw, NullProgressMonitor.INSTANCE);
+ }
+
+ DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+ gc.setReftableConfig(new ReftableConfig());
+ run(gc);
+
+ // Single GC pack present with all objects.
+ assertEquals(1, odb.getPacks().length);
+ DfsPackFile pack = odb.getPacks()[0];
+ DfsPackDescription desc = pack.getPackDescription();
+ assertEquals(GC, desc.getPackSource());
+ assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+ assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+ // Sibling REFTABLE is also present.
+ assertTrue(desc.hasFileExt(REFTABLE));
+ ReftableWriter.Stats stats = desc.getReftableStats();
+ assertNotNull(stats);
+ assertTrue(stats.totalBytes() > 0);
+ assertEquals(5101, stats.refCount());
+ assertEquals(1, stats.minUpdateIndex());
+ assertEquals(1, stats.maxUpdateIndex());
+
+ DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
+ try (DfsReader ctx = odb.newReader();
+ ReftableReader rr = table.open(ctx);
+ RefCursor rc = rr.seekRef("refs/pulls/5100")) {
+ assertTrue(rc.next());
+ assertEquals(commit0, rc.getRef().getObjectId());
+ assertFalse(rc.next());
+ }
+ }
+
+ @Test
+ public void leavesNonGcReftablesIfNotConfigured() throws Exception {
+ String master = "refs/heads/master";
+ RevCommit commit0 = commit().message("0").create();
+ RevCommit commit1 = commit().message("1").parent(commit0).create();
+ git.update(master, commit1);
+
+ DfsPackDescription t1 = odb.newPack(INSERT);
+ try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+ out.write("ignored".getBytes(StandardCharsets.UTF_8));
+ t1.addFileExt(REFTABLE);
+ }
+ odb.commitPack(Collections.singleton(t1), null);
+
+ DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+ gc.setReftableConfig(null);
+ run(gc);
+
+ // Single GC pack present with all objects.
+ assertEquals(1, odb.getPacks().length);
+ DfsPackFile pack = odb.getPacks()[0];
+ DfsPackDescription desc = pack.getPackDescription();
+ assertEquals(GC, desc.getPackSource());
+ assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+ assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+ // Only INSERT REFTABLE above is present.
+ DfsReftable[] tables = odb.getReftables();
+ assertEquals(1, tables.length);
+ assertEquals(t1, tables[0].getPackDescription());
+ }
+
+ @Test
+ public void prunesNonGcReftables() throws Exception {
+ String master = "refs/heads/master";
+ RevCommit commit0 = commit().message("0").create();
+ RevCommit commit1 = commit().message("1").parent(commit0).create();
+ git.update(master, commit1);
+
+ DfsPackDescription t1 = odb.newPack(INSERT);
+ try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+ out.write("ignored".getBytes(StandardCharsets.UTF_8));
+ t1.addFileExt(REFTABLE);
+ }
+ odb.commitPack(Collections.singleton(t1), null);
+ odb.clearCache();
+
+ DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+ gc.setReftableConfig(new ReftableConfig());
+ run(gc);
+
+ // Single GC pack present with all objects.
+ assertEquals(1, odb.getPacks().length);
+ DfsPackFile pack = odb.getPacks()[0];
+ DfsPackDescription desc = pack.getPackDescription();
+ assertEquals(GC, desc.getPackSource());
+ assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+ assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+ // Only sibling GC REFTABLE is present.
+ DfsReftable[] tables = odb.getReftables();
+ assertEquals(1, tables.length);
+ assertEquals(desc, tables[0].getPackDescription());
+ assertTrue(desc.hasFileExt(REFTABLE));
+ }
+
+ @Test
+ public void compactsReftables() throws Exception {
+ String master = "refs/heads/master";
+ RevCommit commit0 = commit().message("0").create();
+ RevCommit commit1 = commit().message("1").parent(commit0).create();
+ git.update(master, commit1);
+
+ DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+ gc.setReftableConfig(new ReftableConfig());
+ run(gc);
+
+ DfsPackDescription t1 = odb.newPack(INSERT);
+ Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
+ "refs/heads/next", commit0.copy());
+ try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+ ReftableWriter w = new ReftableWriter();
+ w.setMinUpdateIndex(42);
+ w.setMaxUpdateIndex(42);
+ w.begin(out);
+ w.sortAndWriteRefs(Collections.singleton(next));
+ w.finish();
+ t1.addFileExt(REFTABLE);
+ t1.setReftableStats(w.getStats());
+ }
+ odb.commitPack(Collections.singleton(t1), null);
+
+ gc = new DfsGarbageCollector(repo);
+ gc.setReftableConfig(new ReftableConfig());
+ run(gc);
+
+ // Single GC pack present with all objects.
+ assertEquals(1, odb.getPacks().length);
+ DfsPackFile pack = odb.getPacks()[0];
+ DfsPackDescription desc = pack.getPackDescription();
+ assertEquals(GC, desc.getPackSource());
+ assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+ assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+ // Only sibling GC REFTABLE is present.
+ DfsReftable[] tables = odb.getReftables();
+ assertEquals(1, tables.length);
+ assertEquals(desc, tables[0].getPackDescription());
+ assertTrue(desc.hasFileExt(REFTABLE));
+
+ // GC reftable contains the compaction.
+ DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
+ try (DfsReader ctx = odb.newReader();
+ ReftableReader rr = table.open(ctx);
+ RefCursor rc = rr.allRefs()) {
+ assertEquals(1, rr.minUpdateIndex());
+ assertEquals(42, rr.maxUpdateIndex());
+
+ assertTrue(rc.next());
+ assertEquals(master, rc.getRef().getName());
+ assertEquals(commit1, rc.getRef().getObjectId());
+
+ assertTrue(rc.next());
+ assertEquals(next.getName(), rc.getRef().getName());
+ assertEquals(commit0, rc.getRef().getObjectId());
+
+ assertFalse(rc.next());
+ }
+ }
+
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
return git.commit();
}
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
+import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.EnumSet;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.internal.storage.pack.PackWriter;
+import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
private final DfsObjDatabase objdb;
private final List<DfsPackDescription> newPackDesc;
-
private final List<PackStatistics> newPackStats;
-
private final List<ObjectIdSet> newPackObj;
private DfsReader ctx;
private PackConfig packConfig;
+ private ReftableConfig reftableConfig;
+ private long reftableInitialMinUpdateIndex = 1;
+ private long reftableInitialMaxUpdateIndex = 1;
// See packIsCoalesceableGarbage(), below, for how these two variables
// interact.
private long startTimeMillis;
private List<DfsPackFile> packsBefore;
+ private List<DfsReftable> reftablesBefore;
private List<DfsPackFile> expiredGarbagePacks;
+ private Collection<Ref> refsBefore;
private Set<ObjectId> allHeadsAndTags;
private Set<ObjectId> allTags;
private Set<ObjectId> nonHeads;
return this;
}
+ /**
+ * @param cfg
+ * configuration to write a reftable. Reftable writing is
+ * disabled (default) when {@code cfg} is {@code null}.
+ * @return {@code this}
+ */
+ public DfsGarbageCollector setReftableConfig(ReftableConfig cfg) {
+ reftableConfig = cfg;
+ return this;
+ }
+
+ /**
+ * Set minUpdateIndex for the initial reftable created during conversion.
+ * <p>
+ * <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
+ * normally performed at the start of {@link #pack(ProgressMonitor)}.
+ * Callers must ensure the reference cache is current and will have been
+ * read before the pack list.
+ *
+ * @param u
+ * minUpdateIndex for the initial reftable created by scanning
+ * {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
+ * has also set {@link #setReftableConfig(ReftableConfig)}.
+ * Defaults to {@code 1}. Must be {@code u >= 0}.
+ * @return {@code this}
+ */
+ public DfsGarbageCollector setReftableInitialMinUpdateIndex(long u) {
+ reftableInitialMinUpdateIndex = Math.max(u, 0);
+ return this;
+ }
+
+ /**
+ * Set maxUpdateIndex for the initial reftable created during conversion.
+ * <p>
+ * <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
+ * normally performed at the start of {@link #pack(ProgressMonitor)}.
+ * Callers must ensure the reference cache is current and will have been
+ * read before the pack list.
+ *
+ * @param u
+ * maxUpdateIndex for the initial reftable created by scanning
+ * {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
+ * has also set {@link #setReftableConfig(ReftableConfig)}.
+ * Defaults to {@code 1}. Must be {@code u >= 0}.
+ * @return {@code this}
+ */
+ public DfsGarbageCollector setReftableInitialMaxUpdateIndex(long u) {
+ reftableInitialMaxUpdateIndex = Math.max(0, u);
+ return this;
+ }
+
/** @return garbage packs smaller than this size will be repacked. */
public long getCoalesceGarbageLimit() {
return coalesceGarbageLimit;
refdb.refresh();
objdb.clearCache();
- Collection<Ref> refsBefore = getAllRefs();
+ refsBefore = getAllRefs();
readPacksBefore();
+ readReftablesBefore();
Set<ObjectId> allHeads = new HashSet<>();
allHeadsAndTags = new HashSet<>();
}
}
+ private void readReftablesBefore() throws IOException {
+ DfsReftable[] tables = objdb.getReftables();
+ reftablesBefore = new ArrayList<>(Arrays.asList(tables));
+ }
+
private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
// Consider the garbage pack as expired when it's older than
// garbagePackTtl. This check gives concurrent inserter threads
}
/** @return all of the source packs that fed into this compaction. */
- public List<DfsPackDescription> getSourcePacks() {
+ public Set<DfsPackDescription> getSourcePacks() {
return toPrune();
}
return newPackStats;
}
- private List<DfsPackDescription> toPrune() {
- int cnt = packsBefore.size();
- List<DfsPackDescription> all = new ArrayList<>(cnt);
+ private Set<DfsPackDescription> toPrune() {
+ Set<DfsPackDescription> toPrune = new HashSet<>();
for (DfsPackFile pack : packsBefore) {
- all.add(pack.getPackDescription());
+ toPrune.add(pack.getPackDescription());
+ }
+ if (reftableConfig != null) {
+ for (DfsReftable table : reftablesBefore) {
+ toPrune.add(table.getPackDescription());
+ }
}
for (DfsPackFile pack : expiredGarbagePacks) {
- all.add(pack.getPackDescription());
+ toPrune.add(pack.getPackDescription());
}
- return all;
+ return toPrune;
}
private void packHeads(ProgressMonitor pm) throws IOException {
- if (allHeadsAndTags.isEmpty())
+ if (allHeadsAndTags.isEmpty()) {
+ writeReftable();
return;
+ }
try (PackWriter pw = newPackWriter()) {
pw.setTagTargets(tagTargets);
pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
- if (0 < pw.getObjectCount())
- writePack(GC, pw, pm,
- estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC));
+ if (0 < pw.getObjectCount()) {
+ long estSize = estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC);
+ writePack(GC, pw, pm, estSize);
+ } else {
+ writeReftable();
+ }
}
}
estimatedPackSize);
newPackDesc.add(pack);
+ if (source == GC && reftableConfig != null) {
+ writeReftable(pack);
+ }
+
try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
pw.writePack(pm, pm, out);
pack.addFileExt(PACK);
newPackObj.add(pw.getObjectSet());
return pack;
}
+
+ private void writeReftable() throws IOException {
+ if (reftableConfig != null) {
+ DfsPackDescription pack = objdb.newPack(GC);
+ newPackDesc.add(pack);
+ writeReftable(pack);
+ }
+ }
+
+ private void writeReftable(DfsPackDescription pack) throws IOException {
+ if (!hasGcReftable()) {
+ writeReftable(pack, refsBefore);
+ return;
+ }
+
+ try (ReftableStack stack = ReftableStack.open(ctx, reftablesBefore)) {
+ ReftableCompactor compact = new ReftableCompactor();
+ compact.addAll(stack.readers());
+ compact.setIncludeDeletes(false);
+ compactReftable(pack, compact);
+ }
+ }
+
+ private boolean hasGcReftable() {
+ for (DfsReftable table : reftablesBefore) {
+ if (table.getPackDescription().getPackSource() == GC) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
+ throws IOException {
+ try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
+ ReftableConfig cfg = configureReftable(reftableConfig, out);
+ ReftableWriter writer = new ReftableWriter(cfg)
+ .setMinUpdateIndex(reftableInitialMinUpdateIndex)
+ .setMaxUpdateIndex(reftableInitialMaxUpdateIndex)
+ .begin(out)
+ .sortAndWriteRefs(refs)
+ .finish();
+ pack.addFileExt(REFTABLE);
+ pack.setReftableStats(writer.getStats());
+ }
+ }
+
+ private void compactReftable(DfsPackDescription pack,
+ ReftableCompactor compact) throws IOException {
+ try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
+ compact.setConfig(configureReftable(reftableConfig, out));
+ compact.compact(out);
+ pack.addFileExt(REFTABLE);
+ pack.setReftableStats(compact.getStats());
+ }
+ }
}
projects / jgit.git / commitdiff