PackWriter: make PackWriter.writeIndex() take a PackIndexWriter

Previously, the PackWriter implementation required that indexes and
their extensions be writable to an OutputStream with a fixed binary
format.  To support more general index storage formats, allow
PackWriter to accept an PackIndexWriter interface which accepts only
the objects to store. This allows implementors to choose their storage
format.

The implementation will be provided by the DfsObjectDatabase. The
DfsObjectDatabase is already responsible for providing the OutputStream
that was previously used to write indexes. Having it provide a writing
interface would be a natural generalization.

This idea was previously implemented for PackBitmapIndex writing in
https://gerrithub.io/c/eclipse-jgit/jgit/+/1177722.

Change-Id: I582d2f3d25d6adb2da243d6d0d7bc405a97d6183

4 files changed, 57 insertions, 24 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
index a177669788..b933e942d9 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
@@ -18,7 +18,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
 import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
@@ -687,14 +686,7 @@ public class DfsGarbageCollector {
 			pack.setBlockSize(PACK, out.blockSize());
 		}
 
-		try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) {
-			CountingOutputStream cnt = new CountingOutputStream(out);
-			pw.writeIndex(cnt);
-			pack.addFileExt(INDEX);
-			pack.setFileSize(INDEX, cnt.getCount());
-			pack.setBlockSize(INDEX, out.blockSize());
-			pack.setIndexVersion(pw.getIndexVersion());
-		}
+		pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion()));
 
 		if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) {
 			try (DfsOutputStream out = objdb.writeFile(pack,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
index 616563ffdd..efd666ff27 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
@@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs;
 
 import static java.util.stream.Collectors.joining;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -27,7 +28,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter;
 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1;
+import org.eclipse.jgit.internal.storage.pack.PackIndexWriter;
 import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.lib.AnyObjectId;
@@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 			}
 		};
 	}
+
+	/**
+	 * Returns a writer to store the pack index in this object database.
+	 *
+	 * @param pack
+	 *            Pack file to which the index is associated.
+	 * @param indexVersion
+	 *            which version of the index to write
+	 * @return a writer to store the index associated with the pack
+	 * @throws IOException
+	 *             when some I/O problem occurs while creating or writing to
+	 *             output stream
+	 */
+	public PackIndexWriter getPackIndexWriter(
+			DfsPackDescription pack, int indexVersion)
+			throws IOException {
+		return (objectsToStore, packDataChecksum) -> {
+			try (DfsOutputStream out = writeFile(pack, INDEX);
+					CountingOutputStream cnt = new CountingOutputStream(out)) {
+				final PackIndexWriter iw = BasePackIndexWriter
+						.createVersion(cnt,
+						indexVersion);
+				iw.write(objectsToStore, packDataChecksum);
+				pack.addFileExt(INDEX);
+				pack.setFileSize(INDEX, cnt.getCount());
+				pack.setBlockSize(INDEX, out.blockSize());
+				pack.setIndexVersion(indexVersion);
+			}
+		};
+	}
+
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
index 86144b389c..b32cddae77 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
@@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
@@ -45,7 +44,6 @@ import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 import org.eclipse.jgit.util.BlockList;
-import org.eclipse.jgit.util.io.CountingOutputStream;
 
 /**
  * Combine several pack files into one pack.
@@ -458,14 +456,7 @@ public class DfsPackCompactor {
 	private static void writeIndex(DfsObjDatabase objdb,
 			DfsPackDescription pack,
 			PackWriter pw) throws IOException {
-		try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) {
-			CountingOutputStream cnt = new CountingOutputStream(out);
-			pw.writeIndex(cnt);
-			pack.addFileExt(INDEX);
-			pack.setFileSize(INDEX, cnt.getCount());
-			pack.setBlockSize(INDEX, out.blockSize());
-			pack.setIndexVersion(pw.getIndexVersion());
-		}
+		pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion()));
 	}
 
 	static ReftableConfig configureReftable(ReftableConfig cfg,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 4fd2eb5798..4c262b0e3a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer;
  * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with
  * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the
  * pack is being stored as a file the matching index can be written out after
- * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap
+ * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap
  * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)}
  * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}.
  * </p>
@@ -1099,12 +1099,28 @@ public class PackWriter implements AutoCloseable {
 	 *             the index data could not be written to the supplied stream.
 	 */
 	public void writeIndex(OutputStream indexStream) throws IOException {
+		writeIndex(BasePackIndexWriter.createVersion(indexStream,
+				getIndexVersion()));
+	}
+
+	/**
+	 * Create an index file to match the pack file just written.
+	 * <p>
+	 * Called after
+	 * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}.
+	 * <p>
+	 * Writing an index is only required for local pack storage. Packs sent on
+	 * the network do not need to create an index.
+	 *
+	 * @param iw
+	 *            an @code{PackIndexWriter} instance to write the index
+	 * @throws java.io.IOException
+	 *             the index data could not be written to the supplied stream.
+	 */
+	public void writeIndex(PackIndexWriter iw) throws IOException {
 		if (isIndexDisabled())
 			throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);
-
 		long writeStart = System.currentTimeMillis();
-		PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream,
-				getIndexVersion());
 		iw.write(sortByName(), packcsum);
 		stats.timeWriting += System.currentTimeMillis() - writeStart;
 	}
@@ -2448,7 +2464,7 @@ public class PackWriter implements AutoCloseable {
 	 * object graph at selected commits. Writing a bitmap index is an optional
 	 * feature that not all pack users may require.
 	 * <p>
-	 * Called after {@link #writeIndex(OutputStream)}.
+	 * Called after {@link #writeIndex(PackIndexWriter)}.
 	 * <p>
 	 * To reduce memory internal state is cleared during this method, rendering
 	 * the PackWriter instance useless for anything further than a call to write