From 52405358a588b6453c7568e7430456826f2ce3b4 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 7 Oct 2024 15:20:26 -0700 Subject: [PATCH] PackIndexWriter: create interface to write indexes PackWriter assumes that the primary index goes to a file in a well-known format. This cannot accomodate implementations in other storages or formats (e.g. in a database). Create an interface to write the index (PackIndexWriter). This interface will be implemented by the existing pack index writer classes (PackIndexWriterV1 etc.). As the "PackIndexWriter" name was used by the previous superclass of the file writers, we rename that class to "BasePackIndexWriter". Change-Id: Ia7348395315e458fc7adc75a8db5dcb903e2a4a1 --- .../storage/file/AbbreviationTest.java | 1 + ...riterTest.java => BasePackWriterTest.java} | 2 +- .../internal/storage/dfs/DfsInserter.java | 4 +- ...exWriter.java => BasePackIndexWriter.java} | 6 ++- .../file/ObjectDirectoryPackParser.java | 7 ++-- .../jgit/internal/storage/file/PackIndex.java | 2 +- .../storage/file/PackIndexWriterV1.java | 4 +- .../storage/file/PackIndexWriterV2.java | 4 +- .../internal/storage/file/PackInserter.java | 4 +- .../storage/pack/PackIndexWriter.java | 40 +++++++++++++++++++ .../internal/storage/pack/PackWriter.java | 8 ++-- .../eclipse/jgit/storage/pack/PackConfig.java | 6 +-- 12 files changed, 67 insertions(+), 21 deletions(-) rename org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/{PackWriterTest.java => BasePackWriterTest.java} (99%) rename org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/{PackIndexWriter.java => BasePackIndexWriter.java} (98%) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java index bd36337f35..41a33df0e4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java @@ -29,6 +29,7 @@ import java.util.List; import org.eclipse.jgit.errors.AmbiguousObjectException; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AbbreviatedObjectId; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java similarity index 99% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java index 24a81b6715..92d7465376 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java @@ -66,7 +66,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -public class PackWriterTest extends SampleDataRepositoryTestCase { +public class BasePackWriterTest extends SampleDataRepositoryTestCase { private static final List EMPTY_LIST_REVS = Collections . emptyList(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 86fbd445be..8f09261674 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -42,7 +42,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackIndex; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; @@ -333,7 +333,7 @@ public class DfsInserter extends ObjectInserter { private static void index(OutputStream out, byte[] packHash, List list) throws IOException { - PackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); + BasePackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); } void writeObjectSizeIndex(DfsPackDescription pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java similarity index 98% rename from org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java index 87e0b44d46..b89cc1ebf4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java @@ -19,6 +19,7 @@ import java.text.MessageFormat; import java.util.List; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.NB; @@ -31,7 +32,7 @@ import org.eclipse.jgit.util.NB; * random access to any object in the pack by associating an ObjectId to the * byte offset within the pack where the object's data can be read. */ -public abstract class PackIndexWriter { +public abstract class BasePackIndexWriter implements PackIndexWriter { /** Magic constant indicating post-version 1 format. */ protected static final byte[] TOC = { -1, 't', 'O', 'c' }; @@ -147,7 +148,7 @@ public abstract class PackIndexWriter { * the stream this instance outputs to. If not already buffered * it will be automatically wrapped in a buffered stream. */ - protected PackIndexWriter(OutputStream dst) { + protected BasePackIndexWriter(OutputStream dst) { out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst : new BufferedOutputStream(dst), Constants.newMessageDigest()); @@ -172,6 +173,7 @@ public abstract class PackIndexWriter { * an error occurred while writing to the output stream, or this * index format cannot store the object data supplied. */ + @Override public void write(final List toStore, final byte[] packDataChecksum) throws IOException { entries = toStore; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 9f27f4bd6e..746e124e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -28,6 +28,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -110,7 +111,7 @@ public class ObjectDirectoryPackParser extends PackParser { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; @@ -386,9 +387,9 @@ public class ObjectDirectoryPackParser extends PackParser { try (FileOutputStream os = new FileOutputStream(tmpIdx)) { final PackIndexWriter iw; if (indexVersion <= 0) - iw = PackIndexWriter.createOldestPossible(os, list); + iw = BasePackIndexWriter.createOldestPossible(os, list); else - iw = PackIndexWriter.createVersion(os, indexVersion); + iw = BasePackIndexWriter.createVersion(os, indexVersion); iw.write(list, packHash); os.getChannel().force(true); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index dfea5c1c80..7189ce20a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -109,7 +109,7 @@ public interface PackIndex } private static boolean isTOC(byte[] h) { - final byte[] toc = PackIndexWriter.TOC; + final byte[] toc = BasePackIndexWriter.TOC; for (int i = 0; i < toc.length; i++) if (h[i] != toc[i]) return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java index 7e28b5eb2b..f0b6193066 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java @@ -21,10 +21,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 1 (old style) pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV1 */ -class PackIndexWriterV1 extends PackIndexWriter { +class PackIndexWriterV1 extends BasePackIndexWriter { static boolean canStore(PackedObjectInfo oe) { // We are limited to 4 GB per pack as offset is 32 bit unsigned int. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java index fc5ef61912..b72b35a464 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java @@ -19,10 +19,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 2 pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV2 */ -class PackIndexWriterV2 extends PackIndexWriter { +class PackIndexWriterV2 extends BasePackIndexWriter { private static final int MAX_OFFSET_32 = 0x7fffffff; private static final int IS_OFFSET_64 = 0x80000000; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 1b092a3332..55e047bd43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -77,6 +77,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -320,7 +321,8 @@ public class PackInserter extends ObjectInserter { private static void writePackIndex(File idx, byte[] packHash, List list) throws IOException { try (OutputStream os = new FileOutputStream(idx)) { - PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + PackIndexWriter w = BasePackIndexWriter.createVersion(os, + INDEX_VERSION); w.write(list, packHash); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java new file mode 100644 index 0000000000..f69e68d4ba --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.transport.PackedObjectInfo; + +/** + * Represents a function that accepts a collection of objects to write into a + * primary pack index storage format. + */ +public interface PackIndexWriter { + /** + * Write all object entries to the index stream. + * + * @param toStore + * sorted list of objects to store in the index. The caller must + * have previously sorted the list using + * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native + * {@link java.lang.Comparable} implementation. + * @param packDataChecksum + * checksum signature of the entire pack data content. This is + * traditionally the last 20 bytes of the pack file's own stream. + * @throws java.io.IOException + * an error occurred while writing to the output stream, or the + * underlying format cannot store the object data supplied. + */ + void write(List toStore, + byte[] packDataChecksum) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index d3e30f3f6c..4fd2eb5798 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -58,7 +58,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; @@ -1078,7 +1078,7 @@ public class PackWriter implements AutoCloseable { if (indexVersion <= 0) { for (BlockList objs : objectsLists) indexVersion = Math.max(indexVersion, - PackIndexWriter.oldestPossibleFormat(objs)); + BasePackIndexWriter.oldestPossibleFormat(objs)); } return indexVersion; } @@ -1103,8 +1103,8 @@ public class PackWriter implements AutoCloseable { throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); long writeStart = System.currentTimeMillis(); - final PackIndexWriter iw = PackIndexWriter.createVersion( - indexStream, getIndexVersion()); + PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream, + getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 8373d6809a..863b79466a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -50,7 +50,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.zip.Deflater; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Repository; @@ -995,7 +995,7 @@ public class PackConfig { * * @return the index version, the special version 0 designates the oldest * (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public int getIndexVersion() { return indexVersion; @@ -1009,7 +1009,7 @@ public class PackConfig { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; -- 2.39.5