dfs: write reftable from DfsGarbageCollector

If a ReftableConfig has been supplied by the caller, write out a
reftable as a sibling of the the GC pack, alongside the heads.

To bootstrap from a non-reftable system, the refs are read from the
DfsRefDatabase if no GC reftables are present.  Its assumed the
references are fully current, and do not need to be merged with any
other reftables.  Any non-GC reftables will be pruned at the end of
the GC cycle, just like any packs that were replaced.

If a GC reftable is present, all existing reftables are compacted, and
references from DfsRefDatabase are only used to seed the packer.  Its
assumed these are consistent with each other.

Change-Id: Ie397eb58aaaefb6865c816d9b39de3ac12998019

1 files changed, 190 insertions, 0 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
index e4dcc2e873..55a5f726de 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
@@ -5,6 +5,7 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -13,19 +14,29 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
 import org.eclipse.jgit.junit.MockSystemReader;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BatchRefUpdate;
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevBlob;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.storage.pack.PackConfig;
+import org.eclipse.jgit.transport.ReceiveCommand;
 import org.eclipse.jgit.util.SystemReader;
 import org.junit.After;
 import org.junit.Before;
@@ -653,6 +664,185 @@ public class DfsGarbageCollectorTest {
 		assertEquals(2, odb.getPacks().length);
 	}
 
+	@SuppressWarnings("boxing")
+	@Test
+	public void producesNewReftable() throws Exception {
+		String master = "refs/heads/master";
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+
+		BatchRefUpdate bru = git.getRepository().getRefDatabase()
+				.newBatchUpdate();
+		bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
+		for (int i = 1; i <= 5100; i++) {
+			bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
+					String.format("refs/pulls/%04d", i)));
+		}
+		try (RevWalk rw = new RevWalk(git.getRepository())) {
+			bru.execute(rw, NullProgressMonitor.INSTANCE);
+		}
+
+		DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+		gc.setReftableConfig(new ReftableConfig());
+		run(gc);
+
+		// Single GC pack present with all objects.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		DfsPackDescription desc = pack.getPackDescription();
+		assertEquals(GC, desc.getPackSource());
+		assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+		assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+		// Sibling REFTABLE is also present.
+		assertTrue(desc.hasFileExt(REFTABLE));
+		ReftableWriter.Stats stats = desc.getReftableStats();
+		assertNotNull(stats);
+		assertTrue(stats.totalBytes() > 0);
+		assertEquals(5101, stats.refCount());
+		assertEquals(1, stats.minUpdateIndex());
+		assertEquals(1, stats.maxUpdateIndex());
+
+		DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
+		try (DfsReader ctx = odb.newReader();
+				ReftableReader rr = table.open(ctx);
+				RefCursor rc = rr.seekRef("refs/pulls/5100")) {
+			assertTrue(rc.next());
+			assertEquals(commit0, rc.getRef().getObjectId());
+			assertFalse(rc.next());
+		}
+	}
+
+	@Test
+	public void leavesNonGcReftablesIfNotConfigured() throws Exception {
+		String master = "refs/heads/master";
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update(master, commit1);
+
+		DfsPackDescription t1 = odb.newPack(INSERT);
+		try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+			out.write("ignored".getBytes(StandardCharsets.UTF_8));
+			t1.addFileExt(REFTABLE);
+		}
+		odb.commitPack(Collections.singleton(t1), null);
+
+		DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+		gc.setReftableConfig(null);
+		run(gc);
+
+		// Single GC pack present with all objects.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		DfsPackDescription desc = pack.getPackDescription();
+		assertEquals(GC, desc.getPackSource());
+		assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+		assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+		// Only INSERT REFTABLE above is present.
+		DfsReftable[] tables = odb.getReftables();
+		assertEquals(1, tables.length);
+		assertEquals(t1, tables[0].getPackDescription());
+	}
+
+	@Test
+	public void prunesNonGcReftables() throws Exception {
+		String master = "refs/heads/master";
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update(master, commit1);
+
+		DfsPackDescription t1 = odb.newPack(INSERT);
+		try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+			out.write("ignored".getBytes(StandardCharsets.UTF_8));
+			t1.addFileExt(REFTABLE);
+		}
+		odb.commitPack(Collections.singleton(t1), null);
+		odb.clearCache();
+
+		DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+		gc.setReftableConfig(new ReftableConfig());
+		run(gc);
+
+		// Single GC pack present with all objects.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		DfsPackDescription desc = pack.getPackDescription();
+		assertEquals(GC, desc.getPackSource());
+		assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+		assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+		// Only sibling GC REFTABLE is present.
+		DfsReftable[] tables = odb.getReftables();
+		assertEquals(1, tables.length);
+		assertEquals(desc, tables[0].getPackDescription());
+		assertTrue(desc.hasFileExt(REFTABLE));
+	}
+
+	@Test
+	public void compactsReftables() throws Exception {
+		String master = "refs/heads/master";
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update(master, commit1);
+
+		DfsGarbageCollector gc = new DfsGarbageCollector(repo);
+		gc.setReftableConfig(new ReftableConfig());
+		run(gc);
+
+		DfsPackDescription t1 = odb.newPack(INSERT);
+		Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
+				"refs/heads/next", commit0.copy());
+		try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
+			ReftableWriter w = new ReftableWriter();
+			w.setMinUpdateIndex(42);
+			w.setMaxUpdateIndex(42);
+			w.begin(out);
+			w.sortAndWriteRefs(Collections.singleton(next));
+			w.finish();
+			t1.addFileExt(REFTABLE);
+			t1.setReftableStats(w.getStats());
+		}
+		odb.commitPack(Collections.singleton(t1), null);
+
+		gc = new DfsGarbageCollector(repo);
+		gc.setReftableConfig(new ReftableConfig());
+		run(gc);
+
+		// Single GC pack present with all objects.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		DfsPackDescription desc = pack.getPackDescription();
+		assertEquals(GC, desc.getPackSource());
+		assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
+		assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
+
+		// Only sibling GC REFTABLE is present.
+		DfsReftable[] tables = odb.getReftables();
+		assertEquals(1, tables.length);
+		assertEquals(desc, tables[0].getPackDescription());
+		assertTrue(desc.hasFileExt(REFTABLE));
+
+		// GC reftable contains the compaction.
+		DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
+		try (DfsReader ctx = odb.newReader();
+				ReftableReader rr = table.open(ctx);
+				RefCursor rc = rr.allRefs()) {
+			assertEquals(1, rr.minUpdateIndex());
+			assertEquals(42, rr.maxUpdateIndex());
+
+			assertTrue(rc.next());
+			assertEquals(master, rc.getRef().getName());
+			assertEquals(commit1, rc.getRef().getObjectId());
+
+			assertTrue(rc.next());
+			assertEquals(next.getName(), rc.getRef().getName());
+			assertEquals(commit0, rc.getRef().getObjectId());
+
+			assertFalse(rc.next());
+		}
+	}
+
 	private TestRepository<InMemoryRepository>.CommitBuilder commit() {
 		return git.commit();
 	}