PackInserter: write object-size index with the pack

Create an object-index when writing a pack with PackInserter if config
says so.

Change-Id: Ic7a3a92b8f8afc4d89e37b9bf8091f438fa9b8c1

3 files changed, 70 insertions, 5 deletions

diff --git a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
index 92371aec69..01df135c5f 100644
--- a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF
@@ -20,6 +20,7 @@ Import-Package: org.eclipse.jgit.annotations;version="[7.4.0,7.5.0)",
  org.eclipse.jgit.merge;version="[7.4.0,7.5.0)",
  org.eclipse.jgit.revwalk;version="[7.4.0,7.5.0)",
  org.eclipse.jgit.storage.file;version="[7.4.0,7.5.0)",
+ org.eclipse.jgit.storage.pack;version="[7.4.0,7.5.0)",
  org.eclipse.jgit.transport;version="7.4.0",
  org.eclipse.jgit.treewalk;version="[7.4.0,7.5.0)",
  org.eclipse.jgit.treewalk.filter;version="[7.4.0,7.5.0)",
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
index 85043034aa..cc43d3c2bb 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java
@@ -53,6 +53,7 @@ import static org.hamcrest.Matchers.lessThan;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
@@ -77,12 +78,14 @@ import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lib.CommitBuilder;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.lib.ObjectStream;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.storage.file.WindowCacheConfig;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
 import org.eclipse.jgit.util.IO;
@@ -489,6 +492,38 @@ public class PackInserterTest extends RepositoryTestCase {
 		}
 	}
 
+	@Test
+	public void createsObjectSizeIndex() throws Exception {
+		FileBasedConfig jGitConfig = mockSystemReader.getJGitConfig();
+		jGitConfig.setInt(
+				ConfigConstants.CONFIG_PACK_SECTION,
+				null,
+				ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 10);
+		jGitConfig.save();
+		byte[] oneBlob = Constants.encode("a blob with some content");
+		byte[] anotherBlob = Constants.encode("some more contents");
+		byte[] streamMeBlob = Constants.encode("some more content to write");
+
+		ObjectId oneBlobOid, anotherBlobOid, streamMeBlobOid;
+		try (PackInserter ins = newInserter()) {
+			oneBlobOid = ins.insert(OBJ_BLOB, oneBlob);
+			anotherBlobOid = ins.insert(OBJ_BLOB, anotherBlob);
+			streamMeBlobOid = ins.insert(OBJ_BLOB, streamMeBlob.length,
+					new ByteArrayInputStream(streamMeBlob));
+			ins.flush();
+		}
+
+		List<Pack> listPacks = listPacks(db);
+		assertEquals(1, listPacks.size());
+		Pack thePack = listPacks.get(0);
+		assertTrue(thePack.hasObjectSizeIndex());
+		assertEquals(oneBlob.length, thePack.getIndexedObjectSize(oneBlobOid));
+		assertEquals(anotherBlob.length,
+				thePack.getIndexedObjectSize(anotherBlobOid));
+		assertEquals(streamMeBlob.length,
+				thePack.getIndexedObjectSize(streamMeBlobOid));
+	}
+
 	private List<Pack> listPacks() throws Exception {
 		List<Pack> fromOpenDb = listPacks(db);
 		List<Pack> reopened;
@@ -549,7 +584,8 @@ public class PackInserterTest extends RepositoryTestCase {
 	}
 
 	private void assertPacksOnly() throws Exception {
-		new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
+		new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")
+				&& !f.endsWith(".objsize"))
 				.assertNoBadFiles(db.getObjectDatabase().getDirectory());
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
index 55e047bd43..97a854b8cd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
@@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter {
 		long offset = beginObject(type, len);
 		packOut.compress.write(data, off, len);
 		packOut.compress.finish();
-		return endObject(id, offset);
+		return endObject(id, offset, len, type);
 	}
 
 	@Override
@@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter {
 			len -= n;
 		}
 		packOut.compress.finish();
-		return endObject(md.toObjectId(), offset);
+		return endObject(md.toObjectId(), offset, len, type);
 	}
 
 	private long beginObject(int type, long len) throws IOException {
@@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter {
 		return offset;
 	}
 
-	private ObjectId endObject(ObjectId id, long offset) {
+	private ObjectId endObject(ObjectId id, long offset, long len, int type) {
 		PackedObjectInfo obj = new PackedObjectInfo(id);
+		obj.setType(type);
 		obj.setOffset(offset);
 		obj.setCRC((int) packOut.crc32.getValue());
+		obj.setFullSize(len);
 		objectList.add(obj);
 		objectMap.addIfAbsent(obj);
 		return id;
@@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter {
 				p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$
 	}
 
+	private static File getFileFor(File packFile, PackExt ext) {
+		String p = packFile.getName();
+		return new File(packFile.getParentFile(),
+				p.substring(0, p.lastIndexOf('.')) + ext.getExtension());
+	}
+
 	private void beginPack() throws IOException {
 		objectList = new BlockList<>();
 		objectMap = new ObjectIdOwnerMap<>();
@@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter {
 		Collections.sort(objectList);
 		File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile?
 		writePackIndex(tmpIdx, packHash, objectList);
-
+		File tmpObjSizeIdx = null;
+		if (pconfig.isWriteObjSizeIndex()) {
+			tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX);
+			writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig);
+		}
 		PackFile realPack = new PackFile(db.getPackDirectory(),
 				computeName(objectList), PackExt.PACK);
 		db.closeAllPackHandles(realPack);
@@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter {
 					realIdx), e);
 		}
 
+		if (tmpObjSizeIdx != null) {
+			PackFile realObjSizeIdx = realPack
+					.create(PackExt.OBJECT_SIZE_INDEX);
+			tmpObjSizeIdx.setReadOnly();
+                        FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE);
+		}
+
 		boolean interrupted = false;
 		try {
 			FileSnapshot snapshot = FileSnapshot.save(realPack);
@@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter {
 		}
 	}
 
+	private static void writeObjectSizeIndex(File objIdx,
+			List<PackedObjectInfo> list, PackConfig cfg) throws IOException {
+		try (OutputStream os = new FileOutputStream(objIdx)) {
+			PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter
+					.createWriter(os, cfg.getMinBytesForObjSizeIndex());
+			w.write(list);
+		}
+	}
+
 	private ObjectId computeName(List<PackedObjectInfo> list) {
 		SHA1 md = digest().reset();
 		byte[] buf = buffer();