Pack: getter for the indexed object size (when available)

In some cases (e.g. filtering objects in a partial clone), the caller
prefers the faster (but optional) indexed size rather than the slower
read from storage.

Expose the indexed size in Pack. The caller must check that the pack
has index (with #hasObjectSizeIndex()) and ask only for objects in that
pack.

Change-Id: Ia04e40250a8fb4890ae556ace516944d191f24ef

3 files changed, 148 insertions, 2 deletions

diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
index 2d00a850e5..c546ae9082 100644
--- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
+++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java
@@ -73,6 +73,7 @@ import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.revwalk.RevTag;
 import org.eclipse.jgit.revwalk.RevTree;
 import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
 import org.eclipse.jgit.util.ChangeIdUtil;
@@ -987,7 +988,7 @@ public class TestRepository<R extends Repository> implements AutoCloseable {
 			ObjectDirectory odb = (ObjectDirectory) db.getObjectDatabase();
 			NullProgressMonitor m = NullProgressMonitor.INSTANCE;
 
-			final PackFile pack, idx;
+			PackFile pack;
 			try (PackWriter pw = new PackWriter(db)) {
 				Set<ObjectId> all = new HashSet<>();
 				for (Ref r : db.getRefDatabase().getRefs())
@@ -1002,12 +1003,22 @@ public class TestRepository<R extends Repository> implements AutoCloseable {
 				}
 				pack.setReadOnly();
 
-				idx = pack.create(PackExt.INDEX);
+				PackFile idx = pack.create(PackExt.INDEX);
 				try (OutputStream out =
 						new BufferedOutputStream(new FileOutputStream(idx))) {
 					pw.writeIndex(out);
 				}
 				idx.setReadOnly();
+
+				PackConfig pc = new PackConfig(db);
+				if (pc.getMinBytesForObjSizeIndex() >= 0) {
+					PackFile oidx = pack.create(PackExt.OBJECT_SIZE_INDEX);
+					try (OutputStream out = new BufferedOutputStream(
+							new FileOutputStream(oidx))) {
+						pw.writeObjectSizeIndex(out);
+					}
+					oidx.setReadOnly();
+				}
 			}
 
 			odb.openPack(pack);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
index e1509456e5..016a6afd70 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java
@@ -10,6 +10,7 @@
 
 package org.eclipse.jgit.internal.storage.file;
 
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -29,6 +30,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 import java.util.zip.Deflater;
 
 import org.eclipse.jgit.errors.LargeObjectException;
@@ -39,6 +41,7 @@ import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.junit.TestRng;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.ObjectId;
@@ -47,6 +50,7 @@ import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.ObjectStream;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.storage.file.WindowCacheConfig;
 import org.eclipse.jgit.transport.PackParser;
 import org.eclipse.jgit.transport.PackedObjectInfo;
@@ -295,6 +299,29 @@ public class PackTest extends LocalDiskRepositoryTestCase {
 		}
 	}
 
+	@Test
+	public void testObjectSize() throws Exception {
+		byte[] data = getRng().nextBytes(300);
+		RevBlob aBlob = tr.blob(data);
+		RevCommit aCommit = tr.branch("master").commit().add("A", aBlob).create();
+		repo.getConfig().setInt(CONFIG_PACK_SECTION, null, ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0);
+		tr.packAndPrune();
+
+		List<Pack> packs = repo.getObjectDatabase().getPacks().stream().collect(Collectors.toList());
+		assertEquals(1, packs.size());
+		// Indexed object
+		assertEquals(300, packs.get(0).getIndexedObjectSize(aBlob));
+		assertEquals(300, packs.get(0).getObjectSize(wc, aBlob));
+		// Non indexed object
+		assertEquals(-1, packs.get(0).getIndexedObjectSize(aCommit));
+		assertEquals(168, packs.get(0).getObjectSize(wc, aCommit));
+		// Object not in pack
+		assertEquals(-1, packs.get(0).getObjectSize(wc,
+				ObjectId.fromString("1111111111111111111111111111111111111111")));
+		assertEquals(-1, packs.get(0).getIndexedObjectSize(
+				ObjectId.fromString("1111111111111111111111111111111111111111")));
+	}
+
 	private static byte[] clone(int first, byte[] base) {
 		byte[] r = new byte[base.length];
 		System.arraycopy(base, 1, r, 1, r.length - 1);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
index 5813d39e9a..f2f54947af 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
 
 import java.io.EOFException;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -126,6 +128,10 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
 
 	private Optionally<PackReverseIndex> reverseIdx = Optionally.empty();
 
+	private volatile PackObjectSizeIndex loadedObjSizeIdx;
+
+	private volatile boolean attemptLoadObjSizeIdx;
+
 	private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty();
 
 	/**
@@ -210,6 +216,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
 		}
 	}
 
+	private PackObjectSizeIndex objectSizeIndex() throws IOException {
+		if (loadedObjSizeIdx != null) {
+			return loadedObjSizeIdx;
+		}
+
+		if (attemptLoadObjSizeIdx) {
+			return null;
+		}
+
+		synchronized (this) {
+			if (loadedObjSizeIdx != null) {
+				return loadedObjSizeIdx;
+			}
+
+			PackObjectSizeIndex sizeIdx;
+			try {
+				long start = System.currentTimeMillis();
+				PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX);
+				if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) {
+					attemptLoadObjSizeIdx = true;
+					return null;
+				}
+				sizeIdx = PackObjectSizeIndexLoader.load(
+						new FileInputStream(sizeIdxFile.getAbsoluteFile()));
+				if (LOG.isDebugEnabled()) {
+					LOG.debug(String.format(
+							"Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$
+							sizeIdxFile.getAbsolutePath(),
+							Float.valueOf(
+									sizeIdxFile.length() / (1024f * 1024)),
+							Long.valueOf(System.currentTimeMillis() - start)));
+				}
+
+				loadedObjSizeIdx = sizeIdx;
+			} catch (InterruptedIOException e) {
+				// don't invalidate the pack, we are interrupted from
+				// another thread
+				return null;
+			} finally {
+				attemptLoadObjSizeIdx = true;
+			}
+		}
+
+		return loadedObjSizeIdx;
+	}
+
 	/**
 	 * Get the File object which locates this pack on disk.
 	 *
@@ -231,6 +283,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
 	}
 
 	/**
+	 * Get the object size index for this pack file
+	 *
+	 * @return the object size index for this pack file if it exists (null
+	 *         otherwise)
+	 * @throws IOException
+	 *             problem reading the index
+	 */
+	public boolean hasObjectSizeIndex() throws IOException {
+		return objectSizeIndex() != null;
+	}
+
+	/**
+	 * Number of objects in the object-size index of this pack
+	 *
+	 * @return number of objects in the index (0 if either the index is empty or
+	 *         it doesn't exist)
+	 * @throws IOException
+	 *             if an IO error occurred while reading the index
+	 */
+	public long getObjectSizeIndexCount() throws IOException {
+		if (!hasObjectSizeIndex()) {
+			return 0;
+		}
+
+		return objectSizeIndex().getObjectCount();
+	}
+
+	/**
+	 * Return the size of the object from the object-size index.
+	 *
+	 * Caller MUST check that the pack has object-size index
+	 * ({@link #hasObjectSizeIndex()}) and that the pack contains the object.
+	 *
+	 * @param id
+	 *            object id of an object in the pack
+	 * @return size of the object from the index. Negative if the object is not
+	 *         in the index.
+	 * @throws IOException
+	 *             if an IO error occurred while reading the index
+	 */
+	public long getIndexedObjectSize(AnyObjectId id) throws IOException {
+		int idxPos = idx().findPosition(id);
+		if (idxPos < 0) {
+			return -1;
+		}
+
+		PackObjectSizeIndex sizeIdx = objectSizeIndex();
+		if (sizeIdx == null) {
+			throw new IllegalStateException(
+					"Asking indexed size from a pack without object size index"); //$NON-NLS-1$
+		}
+
+		return sizeIdx.getSize(idxPos);
+	}
+
+	/**
 	 * Get name extracted from {@code pack-*.pack} pattern.
 	 *
 	 * @return name extracted from {@code pack-*.pack} pattern.