Merge changes from topic 'dfs-reftable'

* changes:
  dfs: helper to open multiple reftables
  dfs: expose DfsReftable from DfsObjDatabase
  dfs: support reading reftables through DfsBlockCache

8 files changed, 456 insertions, 64 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
index 813e7f4cd3..b9758bd64e 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
@@ -53,7 +53,7 @@ import org.eclipse.jgit.errors.PackInvalidException;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 
 /** Block based file stored in {@link DfsBlockCache}. */
-public abstract class BlockBasedFile {
+abstract class BlockBasedFile {
 	/** Cache that owns this file and its data. */
 	final DfsBlockCache cache;
 
@@ -129,6 +129,10 @@ public abstract class BlockBasedFile {
 		return size;
 	}
 
+	DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
+		return cache.getOrLoad(this, pos, ctx, null);
+	}
+
 	DfsBlock readOneBlock(long pos, DfsReader ctx,
 			@Nullable ReadableChannel fileChannel) throws IOException {
 		if (invalid)
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java
index dae922eb4c..62a9be3e5c 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java
@@ -46,6 +46,7 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.zip.CRC32;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
@@ -55,11 +56,8 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
 /** A cached slice of a {@link BlockBasedFile}. */
 final class DfsBlock {
 	final DfsStreamKey stream;
-
 	final long start;
-
 	final long end;
-
 	private final byte[] block;
 
 	DfsBlock(DfsStreamKey p, long pos, byte[] buf) {
@@ -73,6 +71,12 @@ final class DfsBlock {
 		return block.length;
 	}
 
+	ByteBuffer zeroCopyByteBuffer(int n) {
+		ByteBuffer b = ByteBuffer.wrap(block);
+		b.position(n);
+		return b;
+	}
+
 	boolean contains(DfsStreamKey want, long pos) {
 		return stream.equals(want) && start <= pos && pos < end;
 	}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
index 76189c1615..9439822016 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
@@ -48,6 +48,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -61,7 +62,9 @@ import org.eclipse.jgit.lib.ObjectReader;
 
 /** Manages objects stored in {@link DfsPackFile} on a storage system. */
 public abstract class DfsObjDatabase extends ObjectDatabase {
-	private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]) {
+	private static final PackList NO_PACKS = new PackList(
+			new DfsPackFile[0],
+			new DfsReftable[0]) {
 		@Override
 		boolean dirty() {
 			return true;
@@ -192,6 +195,18 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	}
 
 	/**
+	 * Scan and list all available reftable files in the repository.
+	 *
+	 * @return list of available reftables. The returned array is shared with
+	 *         the implementation and must not be modified by the caller.
+	 * @throws IOException
+	 *             the pack list cannot be initialized.
+	 */
+	public DfsReftable[] getReftables() throws IOException {
+		return getPackList().reftables;
+	}
+
+	/**
 	 * Scan and list all available pack files in the repository.
 	 *
 	 * @return list of available packs, with some additional metadata. The
@@ -220,6 +235,16 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	}
 
 	/**
+	 * List currently known reftable files in the repository, without scanning.
+	 *
+	 * @return list of available reftables. The returned array is shared with
+	 *         the implementation and must not be modified by the caller.
+	 */
+	public DfsReftable[] getCurrentReftables() {
+		return getCurrentPackList().reftables;
+	}
+
+	/**
 	 * List currently known pack files in the repository, without scanning.
 	 *
 	 * @return list of available packs, with some additional metadata. The
@@ -428,7 +453,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 			DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
 			packs[0] = newPack;
 			System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
-			n = new PackListImpl(packs);
+			n = new PackListImpl(packs, o.reftables);
 		} while (!packList.compareAndSet(o, n));
 	}
 
@@ -454,59 +479,93 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 
 	private PackList scanPacksImpl(PackList old) throws IOException {
 		DfsBlockCache cache = DfsBlockCache.getInstance();
-		Map<DfsPackDescription, DfsPackFile> forReuse = reuseMap(old);
+		Map<DfsPackDescription, DfsPackFile> packs = packMap(old);
+		Map<DfsPackDescription, DfsReftable> reftables = reftableMap(old);
+
 		List<DfsPackDescription> scanned = listPacks();
 		Collections.sort(scanned);
 
-		List<DfsPackFile> list = new ArrayList<>(scanned.size());
+		List<DfsPackFile> newPacks = new ArrayList<>(scanned.size());
+		List<DfsReftable> newReftables = new ArrayList<>(scanned.size());
 		boolean foundNew = false;
 		for (DfsPackDescription dsc : scanned) {
-			DfsPackFile oldPack = forReuse.remove(dsc);
+			DfsPackFile oldPack = packs.remove(dsc);
 			if (oldPack != null) {
-				list.add(oldPack);
+				newPacks.add(oldPack);
 			} else if (dsc.hasFileExt(PackExt.PACK)) {
-				list.add(new DfsPackFile(cache, dsc));
+				newPacks.add(new DfsPackFile(cache, dsc));
+				foundNew = true;
+			}
+
+			DfsReftable oldReftable = reftables.remove(dsc);
+			if (oldReftable != null) {
+				newReftables.add(oldReftable);
+			} else if (dsc.hasFileExt(PackExt.REFTABLE)) {
+				newReftables.add(new DfsReftable(cache, dsc));
 				foundNew = true;
 			}
 		}
 
-		for (DfsPackFile p : forReuse.values())
-			p.close();
-		if (list.isEmpty())
-			return new PackListImpl(NO_PACKS.packs);
+		if (newPacks.isEmpty())
+			return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables);
 		if (!foundNew) {
 			old.clearDirty();
 			return old;
 		}
-		return new PackListImpl(list.toArray(new DfsPackFile[list.size()]));
+		Collections.sort(newReftables, reftableComparator());
+		return new PackListImpl(
+				newPacks.toArray(new DfsPackFile[0]),
+				newReftables.toArray(new DfsReftable[0]));
 	}
 
-	private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
+	private static Map<DfsPackDescription, DfsPackFile> packMap(PackList old) {
 		Map<DfsPackDescription, DfsPackFile> forReuse = new HashMap<>();
 		for (DfsPackFile p : old.packs) {
-			if (p.invalid()) {
-				// The pack instance is corrupted, and cannot be safely used
-				// again. Do not include it in our reuse map.
-				//
-				p.close();
-				continue;
+			if (!p.invalid()) {
+				forReuse.put(p.desc, p);
 			}
+		}
+		return forReuse;
+	}
 
-			DfsPackFile prior = forReuse.put(p.getPackDescription(), p);
-			if (prior != null) {
-				// This should never occur. It should be impossible for us
-				// to have two pack files with the same name, as all of them
-				// came out of the same directory. If it does, we promised to
-				// close any PackFiles we did not reuse, so close the second,
-				// readers are likely to be actively using the first.
-				//
-				forReuse.put(prior.getPackDescription(), prior);
-				p.close();
+	private static Map<DfsPackDescription, DfsReftable> reftableMap(PackList old) {
+		Map<DfsPackDescription, DfsReftable> forReuse = new HashMap<>();
+		for (DfsReftable p : old.reftables) {
+			if (!p.invalid()) {
+				forReuse.put(p.desc, p);
 			}
 		}
 		return forReuse;
 	}
 
+	/** @return comparator to sort {@link DfsReftable} by priority. */
+	protected Comparator<DfsReftable> reftableComparator() {
+		return (fa, fb) -> {
+			DfsPackDescription a = fa.getPackDescription();
+			DfsPackDescription b = fb.getPackDescription();
+
+			// GC, COMPACT reftables first by higher category.
+			int c = category(b) - category(a);
+			if (c != 0) {
+				return c;
+			}
+
+			// Lower maxUpdateIndex first.
+			c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex());
+			if (c != 0) {
+				return c;
+			}
+
+			// Older reftable first.
+			return Long.signum(a.getLastModified() - b.getLastModified());
+		};
+	}
+
+	static int category(DfsPackDescription d) {
+		PackSource s = d.getPackSource();
+		return s != null ? s.category : 0;
+	}
+
 	/** Clears the cached list of packs, forcing them to be scanned again. */
 	protected void clearCache() {
 		packList.set(NO_PACKS);
@@ -514,12 +573,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 
 	@Override
 	public void close() {
-		// PackList packs = packList.get();
 		packList.set(NO_PACKS);
-
-		// TODO Close packs if they aren't cached.
-		// for (DfsPackFile p : packs.packs)
-		// p.close();
 	}
 
 	/** Snapshot of packs scanned in a single pass. */
@@ -527,10 +581,14 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		/** All known packs, sorted. */
 		public final DfsPackFile[] packs;
 
+		/** All known reftables, sorted. */
+		public final DfsReftable[] reftables;
+
 		private long lastModified = -1;
 
-		PackList(DfsPackFile[] packs) {
+		PackList(DfsPackFile[] packs, DfsReftable[] reftables) {
 			this.packs = packs;
+			this.reftables = reftables;
 		}
 
 		/** @return last modified time of all packs, in milliseconds. */
@@ -561,8 +619,8 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	private static final class PackListImpl extends PackList {
 		private volatile boolean dirty;
 
-		PackListImpl(DfsPackFile[] packs) {
-			super(packs);
+		PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) {
+			super(packs, reftables);
 		}
 
 		@Override
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java
index 58a006e45b..e865e6b542 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java
@@ -44,11 +44,13 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
 
 import java.util.Arrays;
 
 import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 
 /**
@@ -68,7 +70,11 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 	private int[] blockSizeMap;
 	private long objectCount;
 	private long deltaCount;
-	private PackStatistics stats;
+	private long minUpdateIndex;
+	private long maxUpdateIndex;
+
+	private PackStatistics packStats;
+	private ReftableWriter.Stats refStats;
 	private int extensions;
 	private int indexVersion;
 	private long estimatedPackSize;
@@ -170,6 +176,36 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 		return this;
 	}
 
+	/** @return minUpdateIndex for the reftable, if present. */
+	public long getMinUpdateIndex() {
+		return minUpdateIndex;
+	}
+
+	/**
+	 * @param min
+	 *            minUpdateIndex for the reftable, or 0.
+	 * @return {@code this}
+	 */
+	public DfsPackDescription setMinUpdateIndex(long min) {
+		minUpdateIndex = min;
+		return this;
+	}
+
+	/** @return maxUpdateIndex for the reftable, if present. */
+	public long getMaxUpdateIndex() {
+		return maxUpdateIndex;
+	}
+
+	/**
+	 * @param max
+	 *            maxUpdateIndex for the reftable, or 0.
+	 * @return {@code this}
+	 */
+	public DfsPackDescription setMaxUpdateIndex(long max) {
+		maxUpdateIndex = max;
+		return this;
+	}
+
 	/**
 	 * @param ext
 	 *            the file extension.
@@ -281,24 +317,38 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 	 *         is being committed to the repository.
 	 */
 	public PackStatistics getPackStats() {
-		return stats;
+		return packStats;
 	}
 
 	DfsPackDescription setPackStats(PackStatistics stats) {
-		this.stats = stats;
+		this.packStats = stats;
 		setFileSize(PACK, stats.getTotalBytes());
 		setObjectCount(stats.getTotalObjects());
 		setDeltaCount(stats.getTotalDeltas());
 		return this;
 	}
 
+	/** @return stats from the sibling reftable, if created. */
+	public ReftableWriter.Stats getReftableStats() {
+		return refStats;
+	}
+
+	void setReftableStats(ReftableWriter.Stats stats) {
+		this.refStats = stats;
+		setMinUpdateIndex(stats.minUpdateIndex());
+		setMaxUpdateIndex(stats.maxUpdateIndex());
+		setFileSize(REFTABLE, stats.totalBytes());
+		setBlockSize(REFTABLE, stats.refBlockSize());
+	}
+
 	/**
 	 * Discard the pack statistics, if it was populated.
 	 *
 	 * @return {@code this}
 	 */
 	public DfsPackDescription clearPackStats() {
-		stats = null;
+		packStats = null;
+		refStats = null;
 		return this;
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
index 2326219fd8..dfb41e204f 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
@@ -385,12 +385,6 @@ public final class DfsPackFile extends BlockBasedFile {
 		idx(ctx).resolve(matches, id, matchLimit);
 	}
 
-	/** Release all memory used by this DfsPackFile instance. */
-	public void close() {
-		index = null;
-		reverseIndex = null;
-	}
-
 	/**
 	 * Obtain the total number of objects available in this pack. This method
 	 * relies on pack index, giving number of effectively available objects.
@@ -739,10 +733,6 @@ public final class DfsPackFile extends BlockBasedFile {
 			throw new EOFException();
 	}
 
-	DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
-		return cache.getOrLoad(this, pos, ctx, null);
-	}
-
 	ObjectLoader load(DfsReader ctx, long pos)
 			throws IOException {
 		try {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java
index 4b0d583435..3c8422077b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java
@@ -655,7 +655,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 	/**
 	 * Copy bytes from the window to a caller supplied buffer.
 	 *
-	 * @param pack
+	 * @param file
 	 *            the file the desired window is stored within.
 	 * @param position
 	 *            position within the file to read from.
@@ -674,24 +674,24 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 	 *             this cursor does not match the provider or id and the proper
 	 *             window could not be acquired through the provider's cache.
 	 */
-	int copy(DfsPackFile pack, long position, byte[] dstbuf, int dstoff, int cnt)
-			throws IOException {
+	int copy(BlockBasedFile file, long position, byte[] dstbuf, int dstoff,
+			int cnt) throws IOException {
 		if (cnt == 0)
 			return 0;
 
-		long length = pack.length;
+		long length = file.length;
 		if (0 <= length && length <= position)
 			return 0;
 
 		int need = cnt;
 		do {
-			pin(pack, position);
+			pin(file, position);
 			int r = block.copy(position, dstbuf, dstoff, need);
 			position += r;
 			dstoff += r;
 			need -= r;
 			if (length < 0)
-				length = pack.length;
+				length = file.length;
 		} while (0 < need && position < length);
 		return cnt - need;
 	}
@@ -756,14 +756,14 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 			inf.reset();
 	}
 
-	void pin(DfsPackFile pack, long position) throws IOException {
-		if (block == null || !block.contains(pack.key, position)) {
+	void pin(BlockBasedFile file, long position) throws IOException {
+		if (block == null || !block.contains(file.key, position)) {
 			// If memory is low, we may need what is in our window field to
 			// be cleaned up by the GC during the get for the next window.
 			// So we always clear it, even though we are just going to set
 			// it again.
 			block = null;
-			block = pack.getOrLoadBlock(position, this);
+			block = file.getOrLoadBlock(position, this);
 		}
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java
new file mode 100644
index 0000000000..5a8ea92a84
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.dfs;
+
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+
+/** A reftable stored in {@link DfsBlockCache}. */
+public class DfsReftable extends BlockBasedFile {
+	/**
+	 * Construct a reader for an existing reftable.
+	 *
+	 * @param desc
+	 *            description of the reftable within the DFS.
+	 */
+	public DfsReftable(DfsPackDescription desc) {
+		this(DfsBlockCache.getInstance(), desc);
+	}
+
+	/**
+	 * Construct a reader for an existing reftable.
+	 *
+	 * @param cache
+	 *            cache that will store the reftable data.
+	 * @param desc
+	 *            description of the reftable within the DFS.
+	 */
+	public DfsReftable(DfsBlockCache cache, DfsPackDescription desc) {
+		super(cache, desc, REFTABLE);
+
+		int bs = desc.getBlockSize(REFTABLE);
+		if (bs > 0) {
+			setBlockSize(bs);
+		}
+
+		long sz = desc.getFileSize(REFTABLE);
+		length = sz > 0 ? sz : -1;
+	}
+
+	/** @return description that was originally used to configure this file. */
+	public DfsPackDescription getPackDescription() {
+		return desc;
+	}
+
+	/**
+	 * Open reader on the reftable.
+	 * <p>
+	 * The returned reader is not thread safe.
+	 *
+	 * @param ctx
+	 *            reader to access the DFS storage.
+	 * @return cursor to read the table; caller must close.
+	 * @throws IOException
+	 *             table cannot be opened.
+	 */
+	public ReftableReader open(DfsReader ctx) throws IOException {
+		return new ReftableReader(new CacheSource(this, cache, ctx));
+	}
+
+	private static final class CacheSource extends BlockSource {
+		private final DfsReftable file;
+		private final DfsBlockCache cache;
+		private final DfsReader ctx;
+		private ReadableChannel ch;
+		private int readAhead;
+
+		CacheSource(DfsReftable file, DfsBlockCache cache, DfsReader ctx) {
+			this.file = file;
+			this.cache = cache;
+			this.ctx = ctx;
+		}
+
+		@Override
+		public ByteBuffer read(long pos, int cnt) throws IOException {
+			if (ch == null && readAhead > 0 && notInCache(pos)) {
+				open().setReadAheadBytes(readAhead);
+			}
+
+			DfsBlock block = cache.getOrLoad(file, pos, ctx, ch);
+			if (block.start == pos && block.size() >= cnt) {
+				return block.zeroCopyByteBuffer(cnt);
+			}
+
+			byte[] dst = new byte[cnt];
+			ByteBuffer buf = ByteBuffer.wrap(dst);
+			buf.position(ctx.copy(file, pos, dst, 0, cnt));
+			return buf;
+		}
+
+		private boolean notInCache(long pos) {
+			return cache.get(file.key, file.alignToBlock(pos)) == null;
+		}
+
+		@Override
+		public long size() throws IOException {
+			long n = file.length;
+			if (n < 0) {
+				n = open().size();
+				file.length = n;
+			}
+			return n;
+		}
+
+		@Override
+		public void adviseSequentialRead(long start, long end) {
+			int sz = ctx.getOptions().getStreamPackBufferSize();
+			if (sz > 0) {
+				readAhead = (int) Math.min(sz, end - start);
+			}
+		}
+
+		private ReadableChannel open() throws IOException {
+			if (ch == null) {
+				ch = ctx.db.openFile(file.desc, file.ext);
+			}
+			return ch;
+		}
+
+		@Override
+		public void close() {
+			if (ch != null) {
+				try {
+					ch.close();
+				} catch (IOException e) {
+					// Ignore read close failures.
+				} finally {
+					ch = null;
+				}
+			}
+		}
+	}
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java
new file mode 100644
index 0000000000..8d1cc989da
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableStack.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.dfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.eclipse.jgit.internal.storage.reftable.Reftable;
+
+/** Tracks multiple open {@link Reftable} instances. */
+public class ReftableStack implements AutoCloseable {
+	/**
+	 * Opens a stack of tables for reading.
+	 *
+	 * @param ctx
+	 *            context to read the tables with. This {@code ctx} will be
+	 *            retained by the stack and each of the table readers.
+	 * @param tables
+	 *            the tables to open.
+	 * @return stack reference to close the tables.
+	 * @throws IOException
+	 *             a table could not be opened
+	 */
+	public static ReftableStack open(DfsReader ctx, List<DfsReftable> tables)
+			throws IOException {
+		ReftableStack stack = new ReftableStack(tables.size());
+		boolean close = true;
+		try {
+			for (DfsReftable t : tables) {
+				stack.tables.add(t.open(ctx));
+			}
+			close = false;
+			return stack;
+		} finally {
+			if (close) {
+				stack.close();
+			}
+		}
+	}
+
+	private final List<Reftable> tables;
+
+	private ReftableStack(int tableCnt) {
+		this.tables = new ArrayList<>(tableCnt);
+	}
+
+	/**
+	 * @return unmodifiable list of tables, in the same order the files were
+	 *         passed to {@link #open(DfsReader, List)}.
+	 */
+	public List<Reftable> readers() {
+		return Collections.unmodifiableList(tables);
+	}
+
+	@Override
+	public void close() {
+		for (Reftable t : tables) {
+			try {
+				t.close();
+			} catch (IOException e) {
+				// Ignore close failures.
+			}
+		}
+	}
+}