DfsReader: Expose when indices are loaded

We want to measure the data used to serve a request. As a first step,
we want to know how many indices are accessed during the request and
their sizes.

Expose an interface in DfsReader to announce when an index is loaded
into the reader, i.e. when its reference is set.

The interface is more flexible to implementors (what/how to collect)
than the existing DfsReaderIOStats object.

Change-Id: I56f7658fde1758efaf869fa779d11b533a81a0a7

2 files changed, 210 insertions, 23 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java
index 298812d6ed..82fc56347e 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java
@@ -18,8 +18,12 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.zip.Deflater;
 
+import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
+import org.eclipse.jgit.internal.storage.dfs.DfsReader.PackLoadListener;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
 import org.eclipse.jgit.internal.storage.pack.PackWriter;
@@ -130,6 +134,93 @@ public class DfsPackFileTest {
 		assertFalse(pack.hasObjectSizeIndex(reader));
 	}
 
+	private static class TestPackLoadListener implements PackLoadListener {
+		final Map<PackExt, Integer> indexLoadCount = new HashMap<>();
+
+		int blockLoadCount;
+
+		@Override
+		public void onIndexLoad(String packName, PackSource src, PackExt ext,
+				long size, Object loadedIdx) {
+			indexLoadCount.merge(ext, 1, Integer::sum);
+		}
+
+		@Override
+		public void onBlockLoad(String packName, PackSource src, PackExt ext, long position,
+				DfsBlockData dfsBlockData) {
+			blockLoadCount += 1;
+		}
+	}
+
+	@Test
+	public void testIndexLoadCallback_indexNotInCache() throws IOException {
+		bypassCache = false;
+		clearCache = true;
+		setObjectSizeIndexMinBytes(-1);
+		setupPack(512, 800);
+
+		TestPackLoadListener tal = new TestPackLoadListener();
+		DfsReader reader = db.getObjectDatabase().newReader();
+		reader.addPackLoadListener(tal);
+		DfsPackFile pack = db.getObjectDatabase().getPacks()[0];
+		pack.getPackIndex(reader);
+
+		assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue());
+	}
+
+	@Test
+	public void testIndexLoadCallback_indexInCache() throws IOException {
+		bypassCache = false;
+		clearCache = false;
+		setObjectSizeIndexMinBytes(-1);
+		setupPack(512, 800);
+
+		TestPackLoadListener tal = new TestPackLoadListener();
+		DfsReader reader = db.getObjectDatabase().newReader();
+		reader.addPackLoadListener(tal);
+		DfsPackFile pack = db.getObjectDatabase().getPacks()[0];
+		pack.getPackIndex(reader);
+		pack.getPackIndex(reader);
+		pack.getPackIndex(reader);
+
+		assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue());
+	}
+
+	@Test
+	public void testIndexLoadCallback_multipleReads() throws IOException {
+		bypassCache = false;
+		clearCache = true;
+		setObjectSizeIndexMinBytes(-1);
+		setupPack(512, 800);
+
+		TestPackLoadListener tal = new TestPackLoadListener();
+		DfsReader reader = db.getObjectDatabase().newReader();
+		reader.addPackLoadListener(tal);
+		DfsPackFile pack = db.getObjectDatabase().getPacks()[0];
+		pack.getPackIndex(reader);
+		pack.getPackIndex(reader);
+		pack.getPackIndex(reader);
+
+		assertEquals(1, tal.indexLoadCount.get(PackExt.INDEX).intValue());
+	}
+
+
+	@Test
+	public void testBlockLoadCallback_loadInCache() throws IOException {
+		bypassCache = false;
+		clearCache = true;
+		setObjectSizeIndexMinBytes(-1);
+		setupPack(512, 800);
+
+		TestPackLoadListener tal = new TestPackLoadListener();
+		DfsReader reader = db.getObjectDatabase().newReader();
+		reader.addPackLoadListener(tal);
+		DfsPackFile pack = db.getObjectDatabase().getPacks()[0];
+		ObjectId anObject = pack.getPackIndex(reader).getObjectId(0);
+		pack.get(reader, anObject).getBytes();
+		assertEquals(2, tal.blockLoadCount);
+	}
+
 	private ObjectId setupPack(int bs, int ps) throws IOException {
 		DfsBlockCacheConfig cfg = new DfsBlockCacheConfig().setBlockSize(bs)
 				.setBlockLimit(bs * 100).setStreamRatio(bypassCache ? 0F : 1F);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java
index 4dd7ba494f..8fc9a0adf9 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java
@@ -11,15 +11,20 @@ package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION;
+import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
+import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
+import org.eclipse.jgit.internal.storage.dfs.DfsReader.PackLoadListener;
+import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.TestRng;
-import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectInserter;
 import org.junit.Before;
@@ -40,31 +45,31 @@ public class DfsReaderTest {
 		ObjectId obj = insertBlobWithSize(200);
 		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
 			assertFalse("limit < threshold < obj",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 50));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 50));
 			assertEquals(1, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(1, ctx.stats.objectSizeIndexHit);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
 
 			assertFalse("limit = threshold < obj",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 100));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 100));
 			assertEquals(2, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(2, ctx.stats.objectSizeIndexHit);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
 
 			assertFalse("threshold < limit < obj",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 150));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 150));
 			assertEquals(3, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(3, ctx.stats.objectSizeIndexHit);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("threshold < limit = obj",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 200));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 200));
 			assertEquals(4, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(4, ctx.stats.objectSizeIndexHit);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("threshold < obj < limit",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 250));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 250));
 			assertEquals(5, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(5, ctx.stats.objectSizeIndexHit);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
@@ -80,31 +85,31 @@ public class DfsReaderTest {
 		ObjectId obj = insertBlobWithSize(50);
 		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
 			assertFalse("limit < obj < threshold",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 10));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 10));
 			assertEquals(1, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexHit);
 			assertEquals(1, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("limit = obj < threshold",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 50));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 50));
 			assertEquals(2, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexHit);
 			assertEquals(2, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("obj < limit < threshold",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 80));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 80));
 			assertEquals(3, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexHit);
 			assertEquals(3, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("obj < limit = threshold",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 100));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 100));
 			assertEquals(4, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexHit);
 			assertEquals(4, ctx.stats.objectSizeIndexMiss);
 
 			assertTrue("obj < threshold < limit",
-					ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 120));
+					ctx.isNotLargerThan(obj, OBJ_BLOB, 120));
 			assertEquals(5, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexHit);
 			assertEquals(5, ctx.stats.objectSizeIndexMiss);
@@ -116,11 +121,11 @@ public class DfsReaderTest {
 		setObjectSizeIndexMinBytes(100);
 		ObjectId obj = insertBlobWithSize(10);
 		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
-			assertFalse(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 0));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 10));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 40));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 50));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 100));
+			assertFalse(ctx.isNotLargerThan(obj, OBJ_BLOB, 0));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 10));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 40));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 50));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 100));
 
 			assertEquals(5, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(5, ctx.stats.objectSizeIndexMiss);
@@ -133,11 +138,11 @@ public class DfsReaderTest {
 		setObjectSizeIndexMinBytes(-1);
 		ObjectId obj = insertBlobWithSize(10);
 		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
-			assertFalse(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 0));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 10));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 40));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 50));
-			assertTrue(ctx.isNotLargerThan(obj, Constants.OBJ_BLOB, 100));
+			assertFalse(ctx.isNotLargerThan(obj, OBJ_BLOB, 0));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 10));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 40));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 50));
+			assertTrue(ctx.isNotLargerThan(obj, OBJ_BLOB, 100));
 
 			assertEquals(5, ctx.stats.isNotLargerThanCallCount);
 			assertEquals(0, ctx.stats.objectSizeIndexMiss);
@@ -145,12 +150,103 @@ public class DfsReaderTest {
 		}
 	}
 
+	@Test
+	public void packLoadListener_noInvocations() throws IOException {
+		insertBlobWithSize(100);
+		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
+			CounterPackLoadListener listener = new CounterPackLoadListener();
+			ctx.addPackLoadListener(listener);
+			assertEquals(null, listener.callsPerExt.get(PackExt.INDEX));
+		}
+	}
+
+	@Test
+	public void packLoadListener_has_openIdx() throws IOException {
+		ObjectId obj = insertBlobWithSize(100);
+		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
+			CounterPackLoadListener listener = new CounterPackLoadListener();
+			ctx.addPackLoadListener(listener);
+			boolean has = ctx.has(obj);
+			assertTrue(has);
+			assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.INDEX));
+		}
+	}
+
+	@Test
+	public void packLoadListener_notLargerThan_openMultipleIndices() throws IOException {
+			setObjectSizeIndexMinBytes(100);
+			ObjectId obj = insertBlobWithSize(200);
+			try (DfsReader ctx = db.getObjectDatabase().newReader()) {
+				CounterPackLoadListener listener = new CounterPackLoadListener();
+				ctx.addPackLoadListener(listener);
+				boolean notLargerThan = ctx.isNotLargerThan(obj, OBJ_BLOB, 1000);
+				assertTrue(notLargerThan);
+				assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.INDEX));
+				assertEquals(Integer.valueOf(1), listener.callsPerExt.get(PackExt.OBJECT_SIZE_INDEX));
+			}
+	}
+
+	@Test
+	public void packLoadListener_has_openMultipleIndices() throws IOException {
+		setObjectSizeIndexMinBytes(100);
+		insertBlobWithSize(200);
+		insertBlobWithSize(230);
+		insertBlobWithSize(100);
+		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
+			CounterPackLoadListener listener = new CounterPackLoadListener();
+			ctx.addPackLoadListener(listener);
+			ObjectId oid = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f129");
+			boolean has = ctx.has(oid);
+			assertFalse(has);
+			// Open 3 indices trying to find the pack
+			assertEquals(Integer.valueOf(3), listener.callsPerExt.get(PackExt.INDEX));
+		}
+	}
+
+
+	@Test
+	public void packLoadListener_has_repeatedCalls_openMultipleIndices() throws IOException {
+		// Two objects NOT in the repo
+		ObjectId oid = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f129");
+		ObjectId oid2 = ObjectId.fromString("aa48de2aa61d9dffa8a05439dc115fe82f10f130");
+
+		setObjectSizeIndexMinBytes(100);
+		insertBlobWithSize(200);
+		insertBlobWithSize(230);
+		insertBlobWithSize(100);
+		CounterPackLoadListener listener = new CounterPackLoadListener();
+		try (DfsReader ctx = db.getObjectDatabase().newReader()) {
+			ctx.addPackLoadListener(listener);
+			boolean has = ctx.has(oid);
+			ctx.has(oid);
+			ctx.has(oid2);
+			assertFalse(has);
+			// The 3 indices were loaded only once each
+			assertEquals(Integer.valueOf(3), listener.callsPerExt.get(PackExt.INDEX));
+		}
+	}
+
+	private static class CounterPackLoadListener implements PackLoadListener {
+		final Map<PackExt, Integer> callsPerExt = new HashMap<>();
+
+		@Override
+		public void onIndexLoad(String packName, PackSource src, PackExt ext, long size,
+				Object loadedIdx) {
+			callsPerExt.merge(ext, 1, Integer::sum);
+		}
+
+		@Override
+		public void onBlockLoad(String packName, PackSource src, PackExt ext,
+				long size, DfsBlockData dfsBlockData) {
+		}
+	}
+
 	private ObjectId insertBlobWithSize(int size)
 			throws IOException {
 		TestRng testRng = new TestRng(JGitTestUtil.getName());
 		ObjectId oid;
 		try (ObjectInserter ins = db.newObjectInserter()) {
-				oid = ins.insert(Constants.OBJ_BLOB,
+				oid = ins.insert(OBJ_BLOB,
 						testRng.nextBytes(size));
 			ins.flush();
 		}