PackWriter: make PackWriter.writeIndex() take a PackIndexWriter

Previously, the PackWriter implementation required that indexes and
their extensions be writable to an OutputStream with a fixed binary
format.  To support more general index storage formats, allow
PackWriter to accept an PackIndexWriter interface which accepts only
the objects to store. This allows implementors to choose their storage
format.

The implementation will be provided by the DfsObjectDatabase. The
DfsObjectDatabase is already responsible for providing the OutputStream
that was previously used to write indexes. Having it provide a writing
interface would be a natural generalization.

This idea was previously implemented for PackBitmapIndex writing in
https://gerrithub.io/c/eclipse-jgit/jgit/+/1177722.

Change-Id: I582d2f3d25d6adb2da243d6d0d7bc405a97d6183

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
index a17766978854e44e7e276ece24a7b2f99d3c726a..b933e942d9e16fdc94abebb1b537d85f6b4804b5 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
@@ -18,7 +18,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
 import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
@@ -687,14 +686,7 @@ public class DfsGarbageCollector {
                        pack.setBlockSize(PACK, out.blockSize());
                }
 
-               try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) {
-                       CountingOutputStream cnt = new CountingOutputStream(out);
-                       pw.writeIndex(cnt);
-                       pack.addFileExt(INDEX);
-                       pack.setFileSize(INDEX, cnt.getCount());
-                       pack.setBlockSize(INDEX, out.blockSize());
-                       pack.setIndexVersion(pw.getIndexVersion());
-               }
+               pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion()));
 
                if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) {
                        try (DfsOutputStream out = objdb.writeFile(pack,

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
index 616563ffddf49d607bfcff606cb0e6895efe85f6..efd666ff27d0ce68f8f92f6e53436106e1860c0d 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java
@@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs;
 
 import static java.util.stream.Collectors.joining;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -27,7 +28,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter;
 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1;
+import org.eclipse.jgit.internal.storage.pack.PackIndexWriter;
 import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.lib.AnyObjectId;
@@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
                        }
                };
        }
+
+       /**
+        * Returns a writer to store the pack index in this object database.
+        *
+        * @param pack
+        *            Pack file to which the index is associated.
+        * @param indexVersion
+        *            which version of the index to write
+        * @return a writer to store the index associated with the pack
+        * @throws IOException
+        *             when some I/O problem occurs while creating or writing to
+        *             output stream
+        */
+       public PackIndexWriter getPackIndexWriter(
+                       DfsPackDescription pack, int indexVersion)
+                       throws IOException {
+               return (objectsToStore, packDataChecksum) -> {
+                       try (DfsOutputStream out = writeFile(pack, INDEX);
+                                       CountingOutputStream cnt = new CountingOutputStream(out)) {
+                               final PackIndexWriter iw = BasePackIndexWriter
+                                               .createVersion(cnt,
+                                               indexVersion);
+                               iw.write(objectsToStore, packDataChecksum);
+                               pack.addFileExt(INDEX);
+                               pack.setFileSize(INDEX, cnt.getCount());
+                               pack.setBlockSize(INDEX, out.blockSize());
+                               pack.setIndexVersion(indexVersion);
+                       }
+               };
+       }
+
 }

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
index 86144b389c1496f962544688861d88cbea2cb5c2..b32cddae77120fc83fb609dc5e77156d2acde181 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
@@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
-import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
@@ -45,7 +44,6 @@ import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 import org.eclipse.jgit.util.BlockList;
-import org.eclipse.jgit.util.io.CountingOutputStream;
 
 /**
  * Combine several pack files into one pack.
@@ -458,14 +456,7 @@ public class DfsPackCompactor {
        private static void writeIndex(DfsObjDatabase objdb,
                        DfsPackDescription pack,
                        PackWriter pw) throws IOException {
-               try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) {
-                       CountingOutputStream cnt = new CountingOutputStream(out);
-                       pw.writeIndex(cnt);
-                       pack.addFileExt(INDEX);
-                       pack.setFileSize(INDEX, cnt.getCount());
-                       pack.setBlockSize(INDEX, out.blockSize());
-                       pack.setIndexVersion(pw.getIndexVersion());
-               }
+               pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion()));
        }
 
        static ReftableConfig configureReftable(ReftableConfig cfg,

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 4fd2eb5798c7801dce6ef51b942e1fc813b3c232..4c262b0e3a02d0e785dde4901ba441527cfe9439 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer;
  * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with
  * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the
  * pack is being stored as a file the matching index can be written out after
- * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap
+ * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap
  * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)}
  * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}.
  * </p>
@@ -1099,12 +1099,28 @@ public class PackWriter implements AutoCloseable {
         *             the index data could not be written to the supplied stream.
         */
        public void writeIndex(OutputStream indexStream) throws IOException {
+               writeIndex(BasePackIndexWriter.createVersion(indexStream,
+                               getIndexVersion()));
+       }
+
+       /**
+        * Create an index file to match the pack file just written.
+        * <p>
+        * Called after
+        * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}.
+        * <p>
+        * Writing an index is only required for local pack storage. Packs sent on
+        * the network do not need to create an index.
+        *
+        * @param iw
+        *            an @code{PackIndexWriter} instance to write the index
+        * @throws java.io.IOException
+        *             the index data could not be written to the supplied stream.
+        */
+       public void writeIndex(PackIndexWriter iw) throws IOException {
                if (isIndexDisabled())
                        throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);
-
                long writeStart = System.currentTimeMillis();
-               PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream,
-                               getIndexVersion());
                iw.write(sortByName(), packcsum);
                stats.timeWriting += System.currentTimeMillis() - writeStart;
        }
@@ -2448,7 +2464,7 @@ public class PackWriter implements AutoCloseable {
         * object graph at selected commits. Writing a bitmap index is an optional
         * feature that not all pack users may require.
         * <p>
-        * Called after {@link #writeIndex(OutputStream)}.
+        * Called after {@link #writeIndex(PackIndexWriter)}.
         * <p>
         * To reduce memory internal state is cleared during this method, rendering
         * the PackWriter instance useless for anything further than a call to write