From: Sam Delmerico Date: Mon, 7 Oct 2024 21:34:03 +0000 (-0700) Subject: PackWriter: make PackWriter.writeIndex() take a PackIndexWriter X-Git-Tag: v7.1.0.202411121450-m3~1^2~23^2 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=refs%2Fchanges%2F74%2F1202274%2F22;p=jgit.git PackWriter: make PackWriter.writeIndex() take a PackIndexWriter Previously, the PackWriter implementation required that indexes and their extensions be writable to an OutputStream with a fixed binary format. To support more general index storage formats, allow PackWriter to accept an PackIndexWriter interface which accepts only the objects to store. This allows implementors to choose their storage format. The implementation will be provided by the DfsObjectDatabase. The DfsObjectDatabase is already responsible for providing the OutputStream that was previously used to write indexes. Having it provide a writing interface would be a natural generalization. This idea was previously implemented for PackBitmapIndex writing in https://gerrithub.io/c/eclipse-jgit/jgit/+/1177722. Change-Id: I582d2f3d25d6adb2da243d6d0d7bc405a97d6183 --- diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index a177669788..b933e942d9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -18,7 +18,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; @@ -687,14 +686,7 @@ public class DfsGarbageCollector { pack.setBlockSize(PACK, out.blockSize()); } - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) { try (DfsOutputStream out = objdb.writeFile(pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 616563ffdd..efd666ff27 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static java.util.stream.Collectors.joining; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.FileNotFoundException; import java.io.IOException; @@ -27,7 +28,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; @@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } }; } + + /** + * Returns a writer to store the pack index in this object database. + * + * @param pack + * Pack file to which the index is associated. + * @param indexVersion + * which version of the index to write + * @return a writer to store the index associated with the pack + * @throws IOException + * when some I/O problem occurs while creating or writing to + * output stream + */ + public PackIndexWriter getPackIndexWriter( + DfsPackDescription pack, int indexVersion) + throws IOException { + return (objectsToStore, packDataChecksum) -> { + try (DfsOutputStream out = writeFile(pack, INDEX); + CountingOutputStream cnt = new CountingOutputStream(out)) { + final PackIndexWriter iw = BasePackIndexWriter + .createVersion(cnt, + indexVersion); + iw.write(objectsToStore, packDataChecksum); + pack.addFileExt(INDEX); + pack.setFileSize(INDEX, cnt.getCount()); + pack.setBlockSize(INDEX, out.blockSize()); + pack.setIndexVersion(indexVersion); + } + }; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 86144b389c..b32cddae77 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; @@ -45,7 +44,6 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.util.BlockList; -import org.eclipse.jgit.util.io.CountingOutputStream; /** * Combine several pack files into one pack. @@ -458,14 +456,7 @@ public class DfsPackCompactor { private static void writeIndex(DfsObjDatabase objdb, DfsPackDescription pack, PackWriter pw) throws IOException { - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); } static ReftableConfig configureReftable(ReftableConfig cfg, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4fd2eb5798..4c262b0e3a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer; * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the * pack is being stored as a file the matching index can be written out after - * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap + * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}. *

@@ -1099,12 +1099,28 @@ public class PackWriter implements AutoCloseable { * the index data could not be written to the supplied stream. */ public void writeIndex(OutputStream indexStream) throws IOException { + writeIndex(BasePackIndexWriter.createVersion(indexStream, + getIndexVersion())); + } + + /** + * Create an index file to match the pack file just written. + *

+ * Called after + * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. + *

+ * Writing an index is only required for local pack storage. Packs sent on + * the network do not need to create an index. + * + * @param iw + * an @code{PackIndexWriter} instance to write the index + * @throws java.io.IOException + * the index data could not be written to the supplied stream. + */ + public void writeIndex(PackIndexWriter iw) throws IOException { if (isIndexDisabled()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); - long writeStart = System.currentTimeMillis(); - PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream, - getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } @@ -2448,7 +2464,7 @@ public class PackWriter implements AutoCloseable { * object graph at selected commits. Writing a bitmap index is an optional * feature that not all pack users may require. *

- * Called after {@link #writeIndex(OutputStream)}. + * Called after {@link #writeIndex(PackIndexWriter)}. *

* To reduce memory internal state is cleared during this method, rendering * the PackWriter instance useless for anything further than a call to write