Use read ahead during copyPackThroughCache

If a block is missing from the block cache, open the pack stream,
retain the ReadableChannel, and turn on read-ahead.  This should help
to load a medium sized pack into a cold cache more quickly from a
slower IO stream, as the pack is scanned sequentially and missing
blocks are more likely to be available through the read-ahead.

Change-Id: I3300d936b9299be6d9eb642992df7c04bb439cde

2 files changed, 47 insertions, 18 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java
index 6fff656e7b..96a2db9748 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java
@@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.concurrent.locks.ReentrantLock;
 
+import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.internal.JGitText;
 
 /**
@@ -308,12 +309,14 @@ public final class DfsBlockCache {
 	 *            offset within <code>pack</code> of the object.
 	 * @param ctx
 	 *            current thread's reader.
+	 * @param packChannel
+	 *            optional channel to read {@code pack}.
 	 * @return the object reference.
 	 * @throws IOException
 	 *             the reference was not in the cache and could not be loaded.
 	 */
-	DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx)
-			throws IOException {
+	DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx,
+			@Nullable ReadableChannel packChannel) throws IOException {
 		final long requestedPosition = position;
 		position = pack.alignToBlock(position);
 
@@ -345,7 +348,7 @@ public final class DfsBlockCache {
 			statMiss.incrementAndGet();
 			boolean credit = true;
 			try {
-				v = pack.readOneBlock(position, ctx);
+				v = pack.readOneBlock(position, ctx, packChannel);
 				credit = false;
 			} finally {
 				if (credit)
@@ -376,7 +379,7 @@ public final class DfsBlockCache {
 		// that was loaded is the wrong block for the requested position.
 		if (v.contains(pack.key, requestedPosition))
 			return v;
-		return getOrLoad(pack, requestedPosition, ctx);
+		return getOrLoad(pack, requestedPosition, ctx, packChannel);
 	}
 
 	@SuppressWarnings("unchecked")
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
index ae2e7e4127..81972cde91 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
@@ -62,6 +62,7 @@ import java.util.zip.CRC32;
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 
+import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.LargeObjectException;
 import org.eclipse.jgit.errors.MissingObjectException;
@@ -489,15 +490,36 @@ public final class DfsPackFile {
 
 	private void copyPackThroughCache(PackOutputStream out, DfsReader ctx)
 			throws IOException {
-		long position = 12;
-		long remaining = length - (12 + 20);
-		while (0 < remaining) {
-			DfsBlock b = cache.getOrLoad(this, position, ctx);
-			int ptr = (int) (position - b.start);
-			int n = (int) Math.min(b.size() - ptr, remaining);
-			b.write(out, position, n);
-			position += n;
-			remaining -= n;
+		ReadableChannel rc = null;
+		try {
+			long position = 12;
+			long remaining = length - (12 + 20);
+			while (0 < remaining) {
+				DfsBlock b;
+				if (rc != null) {
+					b = cache.getOrLoad(this, position, ctx, rc);
+				} else {
+					b = cache.get(key, alignToBlock(position));
+					if (b == null) {
+						rc = ctx.db.openFile(packDesc, PACK);
+						int sz = ctx.getOptions().getStreamPackBufferSize();
+						if (sz > 0) {
+							rc.setReadAheadBytes(sz);
+						}
+						b = cache.getOrLoad(this, position, ctx, rc);
+					}
+				}
+
+				int ptr = (int) (position - b.start);
+				int n = (int) Math.min(b.size() - ptr, remaining);
+				b.write(out, position, n);
+				position += n;
+				remaining -= n;
+			}
+		} finally {
+			if (rc != null) {
+				rc.close();
+			}
 		}
 	}
 
@@ -780,17 +802,19 @@ public final class DfsPackFile {
 	}
 
 	DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
-		return cache.getOrLoad(this, pos, ctx);
+		return cache.getOrLoad(this, pos, ctx, null);
 	}
 
-	DfsBlock readOneBlock(long pos, DfsReader ctx)
-			throws IOException {
+	DfsBlock readOneBlock(long pos, DfsReader ctx,
+			@Nullable ReadableChannel packChannel) throws IOException {
 		if (invalid)
 			throw new PackInvalidException(getPackName());
 
 		ctx.stats.readBlock++;
 		long start = System.nanoTime();
-		ReadableChannel rc = ctx.db.openFile(packDesc, PACK);
+		ReadableChannel rc = packChannel != null
+				? packChannel
+				: ctx.db.openFile(packDesc, PACK);
 		try {
 			int size = blockSize(rc);
 			pos = (pos / size) * size;
@@ -840,7 +864,9 @@ public final class DfsPackFile {
 
 			return new DfsBlock(key, pos, buf);
 		} finally {
-			rc.close();
+			if (rc != packChannel) {
+				rc.close();
+			}
 			ctx.stats.readBlockMicros += elapsedMicros(start);
 		}
 	}