reftable: debug tools

Simple debug programs to experiment with the reftable file format:

  debug-read-reftable
  debug-write-reftable
  debug-verify-reftable
  debug-benchmark-reftable

Change-Id: I79db351d86900f1e58b17e922e195dff06ee71f1

6 files changed, 932 insertions, 2 deletions

diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
index e73b352729..c59f636a51 100644
--- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
@@ -37,8 +37,11 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)",
  org.eclipse.jgit.errors;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.gitrepo;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.internal.ketch;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.dfs;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.internal.storage.file;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.io;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.internal.storage.pack;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.reftable;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.internal.storage.reftree;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.lfs;version="[4.9.0,4.10.0)",
  org.eclipse.jgit.lfs.lib;version="[4.9.0,4.10.0)",
diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
index 5495be6f88..902547305d 100644
--- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
+++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
@@ -38,10 +38,12 @@ org.eclipse.jgit.pgm.Tag
 org.eclipse.jgit.pgm.UploadPack
 org.eclipse.jgit.pgm.Version
 
+org.eclipse.jgit.pgm.debug.BenchmarkReftable
 org.eclipse.jgit.pgm.debug.DiffAlgorithms
 org.eclipse.jgit.pgm.debug.LfsStore
 org.eclipse.jgit.pgm.debug.MakeCacheTree
 org.eclipse.jgit.pgm.debug.ReadDirCache
+org.eclipse.jgit.pgm.debug.ReadReftable
 org.eclipse.jgit.pgm.debug.RebuildCommitGraph
 org.eclipse.jgit.pgm.debug.RebuildRefTree
 org.eclipse.jgit.pgm.debug.ShowCacheTree
@@ -49,5 +51,6 @@ org.eclipse.jgit.pgm.debug.ShowCommands
 org.eclipse.jgit.pgm.debug.ShowDirCache
 org.eclipse.jgit.pgm.debug.ShowPackDelta
 org.eclipse.jgit.pgm.debug.TextHashFunctions
-org.eclipse.jgit.pgm.debug.WriteDirCache
-
+org.eclipse.jgit.pgm.debug.VerifyReftable
+org.eclipse.jgit.pgm.debug.WriteReftable
+org.eclipse.jgit.pgm.debug.WriteReftable
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java
new file mode 100644
index 0000000000..71c8db8e45
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java
@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.HEAD;
+import static org.eclipse.jgit.lib.Constants.MASTER;
+import static org.eclipse.jgit.lib.Constants.R_HEADS;
+import static org.eclipse.jgit.lib.Ref.Storage.NEW;
+import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.SymbolicRef;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.eclipse.jgit.util.RefList;
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.Option;
+
+@Command
+class BenchmarkReftable extends TextBuiltin {
+	enum Test {
+		SCAN,
+		SEEK_COLD, SEEK_HOT,
+		BY_ID_COLD, BY_ID_HOT;
+	}
+
+	@Option(name = "--tries")
+	private int tries = 10;
+
+	@Option(name = "--test")
+	private Test test = Test.SCAN;
+
+	@Option(name = "--ref")
+	private String ref;
+
+	@Option(name = "--object-id")
+	private String objectId;
+
+	@Argument(index = 0)
+	private String lsRemotePath;
+
+	@Argument(index = 1)
+	private String reftablePath;
+
+	@Override
+	protected void run() throws Exception {
+		switch (test) {
+		case SCAN:
+			scan();
+			break;
+
+		case SEEK_COLD:
+			seekCold(ref);
+			break;
+		case SEEK_HOT:
+			seekHot(ref);
+			break;
+
+		case BY_ID_COLD:
+			byIdCold(ObjectId.fromString(objectId));
+			break;
+		case BY_ID_HOT:
+			byIdHot(ObjectId.fromString(objectId));
+			break;
+		}
+	}
+
+	private void printf(String fmt, Object... args) throws IOException {
+		errw.println(String.format(fmt, args));
+	}
+
+	@SuppressWarnings({ "nls", "boxing" })
+	private void scan() throws Exception {
+		long start, tot;
+
+		start = System.currentTimeMillis();
+		for (int i = 0; i < tries; i++) {
+			readLsRemote();
+		}
+		tot = System.currentTimeMillis() - start;
+		printf("%12s %10d ms  %6d ms/run", "packed-refs", tot, tot / tries);
+
+		start = System.currentTimeMillis();
+		for (int i = 0; i < tries; i++) {
+			try (FileInputStream in = new FileInputStream(reftablePath);
+					BlockSource src = BlockSource.from(in);
+					ReftableReader reader = new ReftableReader(src)) {
+				try (RefCursor rc = reader.allRefs()) {
+					while (rc.next()) {
+						rc.getRef();
+					}
+				}
+			}
+		}
+		tot = System.currentTimeMillis() - start;
+		printf("%12s %10d ms  %6d ms/run", "reftable", tot, tot / tries);
+	}
+
+	private RefList<Ref> readLsRemote()
+			throws IOException, FileNotFoundException {
+		RefList.Builder<Ref> list = new RefList.Builder<>();
+		try (BufferedReader br = new BufferedReader(new InputStreamReader(
+				new FileInputStream(lsRemotePath), UTF_8))) {
+			Ref last = null;
+			String line;
+			while ((line = br.readLine()) != null) {
+				ObjectId id = ObjectId.fromString(line.substring(0, 40));
+				String name = line.substring(41, line.length());
+				if (last != null && name.endsWith("^{}")) { //$NON-NLS-1$
+					last = new ObjectIdRef.PeeledTag(PACKED, last.getName(),
+							last.getObjectId(), id);
+					list.set(list.size() - 1, last);
+					continue;
+				}
+
+				if (name.equals(HEAD)) {
+					last = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
+							R_HEADS + MASTER, null));
+				} else {
+					last = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
+				}
+				list.add(last);
+			}
+		}
+		list.sort();
+		return list.toRefList();
+	}
+
+	@SuppressWarnings({ "nls", "boxing" })
+	private void seekCold(String refName) throws Exception {
+		long start, tot;
+
+		int lsTries = Math.min(tries, 64);
+		start = System.nanoTime();
+		for (int i = 0; i < lsTries; i++) {
+			readLsRemote().get(refName);
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "packed-refs",
+				tot / 1000,
+				(((double) tot) / lsTries) / 1000,
+				lsTries);
+
+		start = System.nanoTime();
+		for (int i = 0; i < tries; i++) {
+			try (FileInputStream in = new FileInputStream(reftablePath);
+					BlockSource src = BlockSource.from(in);
+					ReftableReader reader = new ReftableReader(src)) {
+				try (RefCursor rc = reader.seekRef(refName)) {
+					while (rc.next()) {
+						rc.getRef();
+					}
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "reftable",
+				tot / 1000,
+				(((double) tot) / tries) / 1000,
+				tries);
+	}
+
+	@SuppressWarnings({ "nls", "boxing" })
+	private void seekHot(String refName) throws Exception {
+		long start, tot;
+
+		int lsTries = Math.min(tries, 64);
+		start = System.nanoTime();
+		RefList<Ref> lsRemote = readLsRemote();
+		for (int i = 0; i < lsTries; i++) {
+			lsRemote.get(refName);
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "packed-refs",
+				tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+		start = System.nanoTime();
+		try (FileInputStream in = new FileInputStream(reftablePath);
+				BlockSource src = BlockSource.from(in);
+				ReftableReader reader = new ReftableReader(src)) {
+			for (int i = 0; i < tries; i++) {
+				try (RefCursor rc = reader.seekRef(refName)) {
+					while (rc.next()) {
+						rc.getRef();
+					}
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "reftable",
+				tot / 1000, (((double) tot) / tries) / 1000, tries);
+	}
+
+	@SuppressWarnings({ "nls", "boxing" })
+	private void byIdCold(ObjectId id) throws Exception {
+		long start, tot;
+
+		int lsTries = Math.min(tries, 64);
+		start = System.nanoTime();
+		for (int i = 0; i < lsTries; i++) {
+			for (Ref r : readLsRemote()) {
+				if (id.equals(r.getObjectId())) {
+					continue;
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "packed-refs",
+				tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+		start = System.nanoTime();
+		for (int i = 0; i < tries; i++) {
+			try (FileInputStream in = new FileInputStream(reftablePath);
+					BlockSource src = BlockSource.from(in);
+					ReftableReader reader = new ReftableReader(src)) {
+				try (RefCursor rc = reader.byObjectId(id)) {
+					while (rc.next()) {
+						rc.getRef();
+					}
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "reftable",
+				tot / 1000, (((double) tot) / tries) / 1000, tries);
+	}
+
+	@SuppressWarnings({ "nls", "boxing" })
+	private void byIdHot(ObjectId id) throws Exception {
+		long start, tot;
+
+		int lsTries = Math.min(tries, 64);
+		start = System.nanoTime();
+		RefList<Ref> lsRemote = readLsRemote();
+		for (int i = 0; i < lsTries; i++) {
+			for (Ref r : lsRemote) {
+				if (id.equals(r.getObjectId())) {
+					continue;
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "packed-refs",
+				tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+		start = System.nanoTime();
+		try (FileInputStream in = new FileInputStream(reftablePath);
+				BlockSource src = BlockSource.from(in);
+				ReftableReader reader = new ReftableReader(src)) {
+			for (int i = 0; i < tries; i++) {
+				try (RefCursor rc = reader.byObjectId(id)) {
+					while (rc.next()) {
+						rc.getRef();
+					}
+				}
+			}
+		}
+		tot = System.nanoTime() - start;
+		printf("%12s %10d usec  %9.1f usec/run  %5d runs", "reftable",
+				tot / 1000, (((double) tot) / tries) / 1000, tries);
+	}
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java
new file mode 100644
index 0000000000..9b8db3e0f8
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+
+@Command
+class ReadReftable extends TextBuiltin {
+	@Argument(index = 0)
+	private String input;
+
+	@Argument(index = 1, required = false)
+	private String ref;
+
+	@Override
+	protected void run() throws Exception {
+		try (FileInputStream in = new FileInputStream(input);
+				BlockSource src = BlockSource.from(in);
+				ReftableReader reader = new ReftableReader(src)) {
+			try (RefCursor rc = ref != null
+					? reader.seekRef(ref)
+					: reader.allRefs()) {
+				while (rc.next()) {
+					write(rc.getRef());
+				}
+			}
+		}
+	}
+
+	private void write(Ref r) throws IOException {
+		if (r.isSymbolic()) {
+			outw.println(r.getTarget().getName() + '\t' + r.getName());
+			return;
+		}
+
+		ObjectId id1 = r.getObjectId();
+		if (id1 != null) {
+			outw.println(id1.name() + '\t' + r.getName());
+		}
+
+		ObjectId id2 = r.getPeeledObjectId();
+		if (id2 != null) {
+			outw.println('^' + id2.name());
+		}
+	}
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
new file mode 100644
index 0000000000..dffb579895
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefComparator;
+import org.eclipse.jgit.lib.TextProgressMonitor;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+
+@Command
+class VerifyReftable extends TextBuiltin {
+	private static final long SEED1 = 0xaba8bb4de4caf86cL;
+	private static final long SEED2 = 0x28bb5c25ad43ecb5L;
+
+	@Argument(index = 0)
+	private String lsRemotePath;
+
+	@Argument(index = 1)
+	private String reftablePath;
+
+	@Override
+	protected void run() throws Exception {
+		List<Ref> refs = WriteReftable.readRefs(lsRemotePath);
+
+		try (FileInputStream in = new FileInputStream(reftablePath);
+				BlockSource src = BlockSource.from(in);
+				ReftableReader reader = new ReftableReader(src)) {
+			scan(refs, reader);
+			seek(refs, reader);
+			byId(refs, reader);
+		}
+	}
+
+	@SuppressWarnings("nls")
+	private void scan(List<Ref> refs, ReftableReader reader)
+			throws IOException {
+		errw.print(String.format("%-20s", "sequential scan..."));
+		errw.flush();
+		try (RefCursor rc = reader.allRefs()) {
+			for (Ref exp : refs) {
+				verify(exp, rc);
+			}
+			if (rc.next()) {
+				throw die("expected end of table");
+			}
+		}
+		errw.println(" OK");
+	}
+
+	@SuppressWarnings("nls")
+	private void seek(List<Ref> refs, ReftableReader reader)
+			throws IOException {
+		List<Ref> rnd = new ArrayList<>(refs);
+		Collections.shuffle(rnd, new Random(SEED1));
+
+		TextProgressMonitor pm = new TextProgressMonitor(errw);
+		pm.beginTask("random seek", rnd.size());
+		for (Ref exp : rnd) {
+			try (RefCursor rc = reader.seekRef(exp.getName())) {
+				verify(exp, rc);
+				if (rc.next()) {
+					throw die("should not have ref after " + exp.getName());
+				}
+			}
+			pm.update(1);
+		}
+		pm.endTask();
+	}
+
+	@SuppressWarnings("nls")
+	private void byId(List<Ref> refs, ReftableReader reader)
+			throws IOException {
+		Map<ObjectId, List<Ref>> want = groupById(refs);
+		List<List<Ref>> rnd = new ArrayList<>(want.values());
+		Collections.shuffle(rnd, new Random(SEED2));
+
+		TextProgressMonitor pm = new TextProgressMonitor(errw);
+		pm.beginTask("byObjectId", rnd.size());
+		for (List<Ref> exp : rnd) {
+			Collections.sort(exp, RefComparator.INSTANCE);
+			ObjectId id = exp.get(0).getObjectId();
+			try (RefCursor rc = reader.byObjectId(id)) {
+				for (Ref r : exp) {
+					verify(r, rc);
+				}
+			}
+			pm.update(1);
+		}
+		pm.endTask();
+	}
+
+	private static Map<ObjectId, List<Ref>> groupById(List<Ref> refs) {
+		Map<ObjectId, List<Ref>> m = new HashMap<>();
+		for (Ref r : refs) {
+			ObjectId id = r.getObjectId();
+			if (id != null) {
+				List<Ref> c = m.get(id);
+				if (c == null) {
+					c = new ArrayList<>(2);
+					m.put(id, c);
+				}
+				c.add(r);
+			}
+		}
+		return m;
+	}
+
+	@SuppressWarnings("nls")
+	private void verify(Ref exp, RefCursor rc) throws IOException {
+		if (!rc.next()) {
+			throw die("ended before " + exp.getName());
+		}
+
+		Ref act = rc.getRef();
+		if (!exp.getName().equals(act.getName())) {
+			throw die(String.format("expected %s, found %s",
+					exp.getName(),
+					act.getName()));
+		}
+
+		if (exp.isSymbolic()) {
+			if (!act.isSymbolic()) {
+				throw die("expected " + act.getName() + " to be symbolic");
+			}
+			if (!exp.getTarget().getName().equals(act.getTarget().getName())) {
+				throw die(String.format("expected %s to be %s, found %s",
+						exp.getName(),
+						exp.getLeaf().getName(),
+						act.getLeaf().getName()));
+			}
+			return;
+		}
+
+		if (!AnyObjectId.equals(exp.getObjectId(), act.getObjectId())) {
+			throw die(String.format("expected %s to be %s, found %s",
+					exp.getName(),
+					id(exp.getObjectId()),
+					id(act.getObjectId())));
+		}
+
+		if (exp.getPeeledObjectId() != null
+				&& !AnyObjectId.equals(exp.getPeeledObjectId(), act.getPeeledObjectId())) {
+			throw die(String.format("expected %s to be %s, found %s",
+					exp.getName(),
+					id(exp.getPeeledObjectId()),
+					id(act.getPeeledObjectId())));
+		}
+	}
+
+	@SuppressWarnings("nls")
+	private static String id(ObjectId id) {
+		return id != null ? id.name() : "<null>";
+	}
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java
new file mode 100644
index 0000000000..76ffa194a3
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java
@@ -0,0 +1,301 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.HEAD;
+import static org.eclipse.jgit.lib.Constants.MASTER;
+import static org.eclipse.jgit.lib.Constants.R_HEADS;
+import static org.eclipse.jgit.lib.Ref.Storage.NEW;
+import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.SymbolicRef;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.Option;
+
+@Command
+class WriteReftable extends TextBuiltin {
+	private static final int KIB = 1 << 10;
+	private static final int MIB = 1 << 20;
+
+	@Option(name = "--block-size")
+	private int refBlockSize;
+
+	@Option(name = "--log-block-size")
+	private int logBlockSize;
+
+	@Option(name = "--restart-interval")
+	private int restartInterval;
+
+	@Option(name = "--index-levels")
+	private int indexLevels;
+
+	@Option(name = "--reflog-in")
+	private String reflogIn;
+
+	@Option(name = "--no-index-objects")
+	private boolean noIndexObjects;
+
+	@Argument(index = 0)
+	private String in;
+
+	@Argument(index = 1)
+	private String out;
+
+	@SuppressWarnings({ "nls", "boxing" })
+	@Override
+	protected void run() throws Exception {
+		List<Ref> refs = readRefs(in);
+		List<LogEntry> logs = readLog(reflogIn);
+
+		ReftableWriter.Stats stats;
+		try (OutputStream os = new FileOutputStream(out)) {
+			ReftableConfig cfg = new ReftableConfig();
+			cfg.setIndexObjects(!noIndexObjects);
+			if (refBlockSize > 0) {
+				cfg.setRefBlockSize(refBlockSize);
+			}
+			if (logBlockSize > 0) {
+				cfg.setLogBlockSize(logBlockSize);
+			}
+			if (restartInterval > 0) {
+				cfg.setRestartInterval(restartInterval);
+			}
+			if (indexLevels > 0) {
+				cfg.setMaxIndexLevels(indexLevels);
+			}
+
+			ReftableWriter w = new ReftableWriter(cfg);
+			w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
+			w.begin(os);
+			w.sortAndWriteRefs(refs);
+			for (LogEntry e : logs) {
+				w.writeLog(e.ref, e.updateIndex, e.who,
+						e.oldId, e.newId, e.message);
+			}
+			stats = w.finish().getStats();
+		}
+
+		double fileMiB = ((double) stats.totalBytes()) / MIB;
+		printf("Summary:");
+		printf("  file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
+		printf("  padding : %d KiB", stats.paddingBytes() / KIB);
+		errw.println();
+
+		printf("Refs:");
+		printf("  ref blk : %d", stats.refBlockSize());
+		printf("  restarts: %d", stats.restartInterval());
+		printf("  refs    : %d", stats.refCount());
+		if (stats.refIndexLevels() > 0) {
+			int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
+			printf("  idx sz  : %d KiB", idxSize);
+			printf("  idx lvl : %d", stats.refIndexLevels());
+		}
+		printf("  avg ref : %d bytes", stats.refBytes() / refs.size());
+		errw.println();
+
+		if (stats.objCount() > 0) {
+			int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
+			int idLen = stats.objIdLength();
+			printf("Objects:");
+			printf("  obj blk : %d", stats.refBlockSize());
+			printf("  restarts: %d", stats.restartInterval());
+			printf("  objects : %d", stats.objCount());
+			printf("  obj sz  : %d MiB (%d bytes)", objMiB, stats.objBytes());
+			if (stats.objIndexSize() > 0) {
+				int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
+				printf("  idx sz  : %d KiB", s);
+				printf("  idx lvl : %d", stats.objIndexLevels());
+			}
+			printf("  id len  : %d bytes (%d hex digits)", idLen, 2 * idLen);
+			printf("  avg obj : %d bytes", stats.objBytes() / stats.objCount());
+			errw.println();
+		}
+		if (stats.logCount() > 0) {
+			int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
+			printf("Log:");
+			printf("  log blk : %d", stats.logBlockSize());
+			printf("  logs    : %d", stats.logCount());
+			printf("  log sz  : %d MiB (%d bytes)", logMiB, stats.logBytes());
+			printf("  avg log : %d bytes", stats.logBytes() / logs.size());
+			errw.println();
+		}
+	}
+
+	private void printf(String fmt, Object... args) throws IOException {
+		errw.println(String.format(fmt, args));
+	}
+
+	static List<Ref> readRefs(String inputFile) throws IOException {
+		List<Ref> refs = new ArrayList<>();
+		try (BufferedReader br = new BufferedReader(
+				new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
+			String line;
+			while ((line = br.readLine()) != null) {
+				ObjectId id = ObjectId.fromString(line.substring(0, 40));
+				String name = line.substring(41, line.length());
+				if (name.endsWith("^{}")) { //$NON-NLS-1$
+					int lastIdx = refs.size() - 1;
+					Ref last = refs.get(lastIdx);
+					refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
+							last.getName(), last.getObjectId(), id));
+					continue;
+				}
+
+				Ref ref;
+				if (name.equals(HEAD)) {
+					ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
+							R_HEADS + MASTER, null));
+				} else {
+					ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
+				}
+				refs.add(ref);
+			}
+		}
+		Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
+		return refs;
+	}
+
+	private static List<LogEntry> readLog(String logPath)
+			throws FileNotFoundException, IOException {
+		if (logPath == null) {
+			return Collections.emptyList();
+		}
+
+		List<LogEntry> log = new ArrayList<>();
+		try (BufferedReader br = new BufferedReader(
+				new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
+			@SuppressWarnings("nls")
+			Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
+					+ ",([0-9]+(?:[.][0-9]+)?)" // 2: time
+					+ ",([^,]+)" // 3: who
+					+ ",([^,]+)" // 4: old
+					+ ",([^,]+)" // 5: new
+					+ ",(.*)"); // 6: msg
+			String line;
+			while ((line = br.readLine()) != null) {
+				Matcher m = pattern.matcher(line);
+				if (!m.matches()) {
+					throw new IOException("unparsed line: " + line); //$NON-NLS-1$
+				}
+				String ref = m.group(1);
+				double t = Double.parseDouble(m.group(2));
+				long time = ((long) t) * 1000L;
+				long index = (long) (t * 1e6);
+				String user = m.group(3);
+				ObjectId oldId = parseId(m.group(4));
+				ObjectId newId = parseId(m.group(5));
+				String msg = m.group(6);
+				String email = user + "@gerrit"; //$NON-NLS-1$
+				PersonIdent who = new PersonIdent(user, email, time, -480);
+				log.add(new LogEntry(ref, index, who, oldId, newId, msg));
+			}
+		}
+		Collections.sort(log, LogEntry::compare);
+		return log;
+	}
+
+	private static long min(List<LogEntry> log) {
+		return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
+	}
+
+	private static long max(List<LogEntry> log) {
+		return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
+	}
+
+	private static ObjectId parseId(String s) {
+		if ("NULL".equals(s)) { //$NON-NLS-1$
+			return ObjectId.zeroId();
+		}
+		return ObjectId.fromString(s);
+	}
+
+	private static class LogEntry {
+		static int compare(LogEntry a, LogEntry b) {
+			int cmp = a.ref.compareTo(b.ref);
+			if (cmp == 0) {
+				cmp = Long.signum(b.updateIndex - a.updateIndex);
+			}
+			return cmp;
+		}
+
+		final String ref;
+		final long updateIndex;
+		final PersonIdent who;
+		final ObjectId oldId;
+		final ObjectId newId;
+		final String message;
+
+		LogEntry(String ref, long updateIndex, PersonIdent who,
+				ObjectId oldId, ObjectId newId, String message) {
+			this.ref = ref;
+			this.updateIndex = updateIndex;
+			this.who = who;
+			this.oldId = oldId;
+			this.newId = newId;
+			this.message = message;
+		}
+	}
+}