diff options
5 files changed, 106 insertions, 6 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java index 4f1314057f..070d666ee5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java @@ -290,6 +290,31 @@ public class DfsBlockCacheTest { assertEquals(1, cache.getMissCount()[0]); } + @SuppressWarnings("resource") + @Test + public void highConcurrencyParallelReads_oneRepoParallelReverseIndex() + throws Exception { + InMemoryRepository r1 = createRepoWithBitmap("test"); + resetCache(); + + DfsReader reader = (DfsReader) r1.newObjectReader(); + reader.getOptions().setLoadRevIndexInParallel(true); + for (DfsPackFile pack : r1.getObjectDatabase().getPacks()) { + // Only load non-garbage pack with bitmap. + if (pack.isGarbage()) { + continue; + } + asyncRun(() -> pack.getBitmapIndex(reader)); + asyncRun(() -> pack.getPackIndex(reader)); + asyncRun(() -> pack.getBitmapIndex(reader)); + } + waitForExecutorPoolTermination(); + + assertEquals(1, cache.getMissCount()[PackExt.BITMAP_INDEX.ordinal()]); + assertEquals(1, cache.getMissCount()[PackExt.INDEX.ordinal()]); + assertEquals(1, cache.getMissCount()[0]); + } + private void resetCache() { resetCache(32); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index bb76df1d5d..f7a2c94d48 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -1061,7 +1061,8 @@ public final class DfsPackFile extends BlockBasedFile { } in = new BufferedInputStream(in, bs); bmidx = PackBitmapIndex.read(in, () -> idx(ctx), - () -> getReverseIdx(ctx)); + () -> getReverseIdx(ctx), + ctx.getOptions().shouldLoadRevIndexInParallel()); } finally { size = rc.position(); ctx.stats.readBitmapIdxBytes += size; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index 89de53460c..146f76167d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java @@ -34,6 +34,8 @@ public class DfsReaderOptions { private int streamPackBufferSize; + private boolean loadRevIndexInParallel; + /** * Create a default reader configuration. */ @@ -113,6 +115,28 @@ public class DfsReaderOptions { } /** + * Check if reverse index should be loaded in parallel. + * + * @return true if reverse index is loaded in parallel for bitmap index. + */ + public boolean shouldLoadRevIndexInParallel() { + return loadRevIndexInParallel; + } + + /** + * Enable (or disable) parallel loading of reverse index. + * + * @param loadRevIndexInParallel + * whether to load reverse index in parallel. + * @return {@code this} + */ + public DfsReaderOptions setLoadRevIndexInParallel( + boolean loadRevIndexInParallel) { + this.loadRevIndexInParallel = loadRevIndexInParallel; + return this; + } + + /** * Update properties by setting fields from the configuration. * <p> * If a property is not defined in the configuration, then it is left diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java index 8401f0718a..8fb17fcf21 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java @@ -95,7 +95,7 @@ public abstract class PackBitmapIndex { */ public static PackBitmapIndex read(InputStream fd, PackIndex packIndex, PackReverseIndex reverseIndex) throws IOException { - return new PackBitmapIndexV1(fd, () -> packIndex, () -> reverseIndex); + return new PackBitmapIndexV1(fd, packIndex, reverseIndex); } /** @@ -114,6 +114,8 @@ public abstract class PackBitmapIndex { * @param reverseIndexSupplier * the supplier for pack reverse index for the corresponding pack * file. + * @param loadParallelRevIndex + * whether reverse index should be loaded in parallel * @return a copy of the index in-memory. * @throws java.io.IOException * the stream cannot be read. @@ -122,10 +124,11 @@ public abstract class PackBitmapIndex { */ public static PackBitmapIndex read(InputStream fd, SupplierWithIOException<PackIndex> packIndexSupplier, - SupplierWithIOException<PackReverseIndex> reverseIndexSupplier) + SupplierWithIOException<PackReverseIndex> reverseIndexSupplier, + boolean loadParallelRevIndex) throws IOException { return new PackBitmapIndexV1(fd, packIndexSupplier, - reverseIndexSupplier); + reverseIndexSupplier, loadParallelRevIndex); } /** Footer checksum applied on the bottom of the pack file. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java index 6846e3bcad..21aba3e6a3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java @@ -17,6 +17,12 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; @@ -40,6 +46,23 @@ class PackBitmapIndexV1 extends BasePackBitmapIndex { private static final int MAX_XOR_OFFSET = 126; + private static final ExecutorService executor = Executors + .newCachedThreadPool(new ThreadFactory() { + private final ThreadFactory baseFactory = Executors + .defaultThreadFactory(); + + private final AtomicInteger threadNumber = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = baseFactory.newThread(runnable); + thread.setName("JGit-PackBitmapIndexV1-" //$NON-NLS-1$ + + threadNumber.getAndIncrement()); + thread.setDaemon(true); + return thread; + } + }); + private final PackIndex packIndex; private final PackReverseIndex reverseIndex; private final EWAHCompressedBitmap commits; @@ -49,15 +72,28 @@ class PackBitmapIndexV1 extends BasePackBitmapIndex { private final ObjectIdOwnerMap<StoredBitmap> bitmaps; + PackBitmapIndexV1(final InputStream fd, PackIndex packIndex, + PackReverseIndex reverseIndex) throws IOException { + this(fd, () -> packIndex, () -> reverseIndex, false); + } + PackBitmapIndexV1(final InputStream fd, SupplierWithIOException<PackIndex> packIndexSupplier, - SupplierWithIOException<PackReverseIndex> reverseIndexSupplier) + SupplierWithIOException<PackReverseIndex> reverseIndexSupplier, + boolean loadParallelRevIndex) throws IOException { // An entry is object id, xor offset, flag byte, and a length encoded // bitmap. The object id is an int32 of the nth position sorted by name. super(new ObjectIdOwnerMap<StoredBitmap>()); this.bitmaps = getBitmaps(); + // Optionally start loading reverse index in parallel to loading bitmap + // from storage. + Future<PackReverseIndex> reverseIndexFuture = null; + if (loadParallelRevIndex) { + reverseIndexFuture = executor.submit(reverseIndexSupplier::get); + } + final byte[] scratch = new byte[32]; IO.readFully(fd, scratch, 0, scratch.length); @@ -164,7 +200,18 @@ class PackBitmapIndexV1 extends BasePackBitmapIndex { bitmaps.add(sb); } - this.reverseIndex = reverseIndexSupplier.get(); + PackReverseIndex computedReverseIndex; + if (loadParallelRevIndex && reverseIndexFuture != null) { + try { + computedReverseIndex = reverseIndexFuture.get(); + } catch (InterruptedException | ExecutionException e) { + // Fallback to loading reverse index through a supplier. + computedReverseIndex = reverseIndexSupplier.get(); + } + } else { + computedReverseIndex = reverseIndexSupplier.get(); + } + this.reverseIndex = computedReverseIndex; } /** {@inheritDoc} */ |