--- /dev/null
+/*
+ * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com>
+ * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.eclipse.jgit.errors.CorruptObjectException;
+import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
+import org.eclipse.jgit.junit.JGitTestUtil;
+import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PackReverseIndexComputedTest extends RepositoryTestCase {
+
+ private PackIndex idx;
+
+ private PackReverseIndex reverseIdx;
+
+ /**
+ * Set up tested class instance, test constructor by the way.
+ */
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ // index with both small (< 2^31) and big offsets
+ idx = PackIndex.open(JGitTestUtil.getTestResourceFile("pack-huge.idx"));
+ reverseIdx = PackReverseIndexFactory.computeFromIndex(idx);
+ }
+
+ /**
+ * Test findObject() for all index entries.
+ */
+ @Test
+ public void testFindObject() {
+ for (MutableEntry me : idx)
+ assertEquals(me.toObjectId(), reverseIdx.findObject(me.getOffset()));
+ }
+
+ /**
+ * Test findObject() with illegal argument.
+ */
+ @Test
+ public void testFindObjectWrongOffset() {
+ assertNull(reverseIdx.findObject(0));
+ }
+
+ /**
+ * Test findNextOffset() for all index entries.
+ *
+ * @throws CorruptObjectException
+ */
+ @Test
+ public void testFindNextOffset() throws CorruptObjectException {
+ long offset = findFirstOffset();
+ assertTrue(offset > 0);
+ for (int i = 0; i < idx.getObjectCount(); i++) {
+ long newOffset = reverseIdx.findNextOffset(offset, Long.MAX_VALUE);
+ assertTrue(newOffset > offset);
+ if (i == idx.getObjectCount() - 1)
+ assertEquals(newOffset, Long.MAX_VALUE);
+ else
+ assertEquals(newOffset, idx.findOffset(reverseIdx
+ .findObject(newOffset)));
+ offset = newOffset;
+ }
+ }
+
+ /**
+ * Test findNextOffset() with wrong illegal argument as offset.
+ */
+ @Test
+ public void testFindNextOffsetWrongOffset() {
+ try {
+ reverseIdx.findNextOffset(0, Long.MAX_VALUE);
+ fail("findNextOffset() should throw exception");
+ } catch (CorruptObjectException x) {
+ // expected
+ }
+ }
+
+ private long findFirstOffset() {
+ long min = Long.MAX_VALUE;
+ for (MutableEntry me : idx)
+ min = Math.min(min, me.getOffset());
+ return min;
+ }
+}
+++ /dev/null
-/*
- * Copyright (C) 2008, Imran M Yousuf <imyousuf@smartitengineering.com>
- * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> and others
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Distribution License v. 1.0 which is available at
- * https://www.eclipse.org/org/documents/edl-v10.php.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-package org.eclipse.jgit.internal.storage.file;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.eclipse.jgit.errors.CorruptObjectException;
-import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
-import org.eclipse.jgit.junit.JGitTestUtil;
-import org.eclipse.jgit.junit.RepositoryTestCase;
-import org.junit.Before;
-import org.junit.Test;
-
-public class PackReverseIndexTest extends RepositoryTestCase {
-
- private PackIndex idx;
-
- private PackReverseIndex reverseIdx;
-
- /**
- * Set up tested class instance, test constructor by the way.
- */
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- // index with both small (< 2^31) and big offsets
- idx = PackIndex.open(JGitTestUtil.getTestResourceFile("pack-huge.idx"));
- reverseIdx = PackReverseIndex.computeFromIndex(idx);
- }
-
- /**
- * Test findObject() for all index entries.
- */
- @Test
- public void testFindObject() {
- for (MutableEntry me : idx)
- assertEquals(me.toObjectId(), reverseIdx.findObject(me.getOffset()));
- }
-
- /**
- * Test findObject() with illegal argument.
- */
- @Test
- public void testFindObjectWrongOffset() {
- assertNull(reverseIdx.findObject(0));
- }
-
- /**
- * Test findNextOffset() for all index entries.
- *
- * @throws CorruptObjectException
- */
- @Test
- public void testFindNextOffset() throws CorruptObjectException {
- long offset = findFirstOffset();
- assertTrue(offset > 0);
- for (int i = 0; i < idx.getObjectCount(); i++) {
- long newOffset = reverseIdx.findNextOffset(offset, Long.MAX_VALUE);
- assertTrue(newOffset > offset);
- if (i == idx.getObjectCount() - 1)
- assertEquals(newOffset, Long.MAX_VALUE);
- else
- assertEquals(newOffset, idx.findOffset(reverseIdx
- .findObject(newOffset)));
- offset = newOffset;
- }
- }
-
- /**
- * Test findNextOffset() with wrong illegal argument as offset.
- */
- @Test
- public void testFindNextOffsetWrongOffset() {
- try {
- reverseIdx.findNextOffset(0, Long.MAX_VALUE);
- fail("findNextOffset() should throw exception");
- } catch (CorruptObjectException x) {
- // expected
- }
- }
-
- private long findFirstOffset() {
- long min = Long.MAX_VALUE;
- for (MutableEntry me : idx)
- min = Math.min(min, me.getOffset());
- return min;
- }
-}
import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
+import org.eclipse.jgit.internal.storage.file.PackReverseIndexFactory;
import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
ctx.stats.readReverseIdx++;
long start = System.nanoTime();
- PackReverseIndex revidx = PackReverseIndex.computeFromIndex(idx);
+ PackReverseIndex revidx = PackReverseIndexFactory.computeFromIndex(idx);
reverseIndex = revidx;
ctx.stats.readReverseIdxMicros += elapsedMicros(start);
return new DfsBlockCache.Ref<>(
private synchronized PackReverseIndex getReverseIdx() throws IOException {
if (reverseIdx == null)
- reverseIdx = PackReverseIndex.computeFromIndex(idx());
+ reverseIdx = PackReverseIndexFactory.computeFromIndex(idx());
return reverseIdx;
}
package org.eclipse.jgit.internal.storage.file;
-import java.text.MessageFormat;
-
import org.eclipse.jgit.errors.CorruptObjectException;
-import org.eclipse.jgit.internal.JGitText;
-import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
import org.eclipse.jgit.lib.ObjectId;
/**
* @see PackIndex
* @see Pack
*/
-public class PackReverseIndex {
- /** Index we were created from, and that has our ObjectId data. */
- private final PackIndex index;
-
- /** The number of bytes per entry in the offsetIndex. */
- private final long bucketSize;
-
- /**
- * An index into the nth mapping, where the value is the position after the
- * the last index that contains the values of the bucket. For example given
- * offset o (and bucket = o / bucketSize), the offset will be contained in
- * the range nth[offsetIndex[bucket - 1]] inclusive to
- * nth[offsetIndex[bucket]] exclusive.
- *
- * See {@link #binarySearch}
- */
- private final int[] offsetIndex;
-
- /** Mapping from indices in offset order to indices in SHA-1 order. */
- private final int[] nth;
-
- /**
- * Compute an in-memory pack reverse index from the in-memory pack forward
- * index. This computation uses insertion sort, which has a quadratic
- * runtime on average.
- *
- * @param packIndex
- * the forward index to compute from
- * @return the reverse index instance
- */
- public static PackReverseIndex computeFromIndex(PackIndex packIndex) {
- return new PackReverseIndex(packIndex);
- }
-
- /**
- * Create reverse index from straight/forward pack index, by indexing all
- * its entries.
- *
- * @param packIndex
- * forward index - entries to (reverse) index.
- */
- private PackReverseIndex(PackIndex packIndex) {
- index = packIndex;
-
- final long cnt = index.getObjectCount();
- if (cnt + 1 > Integer.MAX_VALUE)
- throw new IllegalArgumentException(
- JGitText.get().hugeIndexesAreNotSupportedByJgitYet);
-
- if (cnt == 0) {
- bucketSize = Long.MAX_VALUE;
- offsetIndex = new int[1];
- nth = new int[0];
- return;
- }
-
- final long[] offsetsBySha1 = new long[(int) cnt];
-
- long maxOffset = 0;
- int ith = 0;
- for (MutableEntry me : index) {
- final long o = me.getOffset();
- offsetsBySha1[ith++] = o;
- if (o > maxOffset)
- maxOffset = o;
- }
-
- bucketSize = maxOffset / cnt + 1;
- int[] bucketIndex = new int[(int) cnt];
- int[] bucketValues = new int[(int) cnt + 1];
- for (int oi = 0; oi < offsetsBySha1.length; oi++) {
- final long o = offsetsBySha1[oi];
- final int bucket = (int) (o / bucketSize);
- final int bucketValuesPos = oi + 1;
- final int current = bucketIndex[bucket];
- bucketIndex[bucket] = bucketValuesPos;
- bucketValues[bucketValuesPos] = current;
- }
-
- int nthByOffset = 0;
- nth = new int[offsetsBySha1.length];
- offsetIndex = bucketIndex; // Reuse the allocation
- for (int bi = 0; bi < bucketIndex.length; bi++) {
- final int start = nthByOffset;
- // Insertion sort of the values in the bucket.
- for (int vi = bucketIndex[bi]; vi > 0; vi = bucketValues[vi]) {
- final int nthBySha1 = vi - 1;
- final long o = offsetsBySha1[nthBySha1];
- int insertion = nthByOffset++;
- for (; start < insertion; insertion--) {
- if (o > offsetsBySha1[nth[insertion - 1]])
- break;
- nth[insertion] = nth[insertion - 1];
- }
- nth[insertion] = nthBySha1;
- }
- offsetIndex[bi] = nthByOffset;
- }
- }
-
+public interface PackReverseIndex {
/**
* Search for object id with the specified start offset in this pack
* (reverse) index.
* start offset of object to find.
* @return object id for this offset, or null if no object was found.
*/
- public ObjectId findObject(long offset) {
- final int ith = binarySearch(offset);
- if (ith < 0)
- return null;
- return index.getObjectId(nth[ith]);
- }
+ ObjectId findObject(long offset);
/**
* Search for the next offset to the specified offset in this pack (reverse)
* @throws org.eclipse.jgit.errors.CorruptObjectException
* when there is no object with the provided offset.
*/
- public long findNextOffset(long offset, long maxOffset)
- throws CorruptObjectException {
- final int ith = binarySearch(offset);
- if (ith < 0)
- throw new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().cantFindObjectInReversePackIndexForTheSpecifiedOffset,
- Long.valueOf(offset)));
-
- if (ith + 1 == nth.length)
- return maxOffset;
- return index.getOffset(nth[ith + 1]);
- }
-
- int findPosition(long offset) {
- return binarySearch(offset);
- }
+ long findNextOffset(long offset, long maxOffset)
+ throws CorruptObjectException;
- private int binarySearch(long offset) {
- int bucket = (int) (offset / bucketSize);
- int low = bucket == 0 ? 0 : offsetIndex[bucket - 1];
- int high = offsetIndex[bucket];
- while (low < high) {
- final int mid = (low + high) >>> 1;
- final long o = index.getOffset(nth[mid]);
- if (offset < o)
- high = mid;
- else if (offset == o)
- return mid;
- else
- low = mid + 1;
- }
- return -1;
- }
+ /**
+ * Find the position in the primary index of the object at the given pack
+ * offset.
+ *
+ * @param offset
+ * the pack offset of the object
+ * @return the position in the primary index of the object
+ */
+ int findPosition(long offset);
- ObjectId findObjectByPosition(int nthPosition) {
- return index.getObjectId(nth[nthPosition]);
- }
+ /**
+ * Find the object that is in the given position in the primary index.
+ *
+ * @param nthPosition
+ * the position of the object in the primary index
+ * @return the object in that position
+ */
+ ObjectId findObjectByPosition(int nthPosition);
}
--- /dev/null
+/*
+ * Copyright (C) 2023, Google LLC and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import java.text.MessageFormat;
+
+import org.eclipse.jgit.errors.CorruptObjectException;
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
+import org.eclipse.jgit.lib.ObjectId;
+
+/**
+ * Reverse index for forward pack index which is computed from the forward pack
+ * index.
+ * <p>
+ * Creating an instance uses an insertion sort of the entries in the forward
+ * index, so it runs in quadratic time on average.
+ */
+final class PackReverseIndexComputed implements PackReverseIndex {
+ /**
+ * Index we were created from, and that has our ObjectId data.
+ */
+ private final PackIndex index;
+
+ /**
+ * The number of bytes per entry in the offsetIndex.
+ */
+ private final long bucketSize;
+
+ /**
+ * An index into the nth mapping, where the value is the position after the
+ * the last index that contains the values of the bucket. For example given
+ * offset o (and bucket = o / bucketSize), the offset will be contained in
+ * the range nth[offsetIndex[bucket - 1]] inclusive to
+ * nth[offsetIndex[bucket]] exclusive.
+ * <p>
+ * See {@link #binarySearch}
+ */
+ private final int[] offsetIndex;
+
+ /**
+ * Mapping from indices in offset order to indices in SHA-1 order.
+ */
+ private final int[] nth;
+
+ /**
+ * Create reverse index from straight/forward pack index, by indexing all
+ * its entries.
+ *
+ * @param packIndex
+ * forward index - entries to (reverse) index.
+ */
+ PackReverseIndexComputed(PackIndex packIndex) {
+ index = packIndex;
+
+ final long cnt = index.getObjectCount();
+ if (cnt + 1 > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException(
+ JGitText.get().hugeIndexesAreNotSupportedByJgitYet);
+ }
+
+ if (cnt == 0) {
+ bucketSize = Long.MAX_VALUE;
+ offsetIndex = new int[1];
+ nth = new int[0];
+ return;
+ }
+
+ final long[] offsetsBySha1 = new long[(int) cnt];
+
+ long maxOffset = 0;
+ int ith = 0;
+ for (MutableEntry me : index) {
+ final long o = me.getOffset();
+ offsetsBySha1[ith++] = o;
+ if (o > maxOffset) {
+ maxOffset = o;
+ }
+ }
+
+ bucketSize = maxOffset / cnt + 1;
+ int[] bucketIndex = new int[(int) cnt];
+ int[] bucketValues = new int[(int) cnt + 1];
+ for (int oi = 0; oi < offsetsBySha1.length; oi++) {
+ final long o = offsetsBySha1[oi];
+ final int bucket = (int) (o / bucketSize);
+ final int bucketValuesPos = oi + 1;
+ final int current = bucketIndex[bucket];
+ bucketIndex[bucket] = bucketValuesPos;
+ bucketValues[bucketValuesPos] = current;
+ }
+
+ int nthByOffset = 0;
+ nth = new int[offsetsBySha1.length];
+ offsetIndex = bucketIndex; // Reuse the allocation
+ for (int bi = 0; bi < bucketIndex.length; bi++) {
+ final int start = nthByOffset;
+ // Insertion sort of the values in the bucket.
+ for (int vi = bucketIndex[bi]; vi > 0; vi = bucketValues[vi]) {
+ final int nthBySha1 = vi - 1;
+ final long o = offsetsBySha1[nthBySha1];
+ int insertion = nthByOffset++;
+ for (; start < insertion; insertion--) {
+ if (o > offsetsBySha1[nth[insertion - 1]]) {
+ break;
+ }
+ nth[insertion] = nth[insertion - 1];
+ }
+ nth[insertion] = nthBySha1;
+ }
+ offsetIndex[bi] = nthByOffset;
+ }
+ }
+
+ @Override
+ public ObjectId findObject(long offset) {
+ final int ith = binarySearch(offset);
+ if (ith < 0) {
+ return null;
+ }
+ return index.getObjectId(nth[ith]);
+ }
+
+ @Override
+ public long findNextOffset(long offset, long maxOffset)
+ throws CorruptObjectException {
+ final int ith = binarySearch(offset);
+ if (ith < 0) {
+ throw new CorruptObjectException(MessageFormat.format(JGitText
+ .get().cantFindObjectInReversePackIndexForTheSpecifiedOffset,
+ Long.valueOf(offset)));
+ }
+
+ if (ith + 1 == nth.length) {
+ return maxOffset;
+ }
+ return index.getOffset(nth[ith + 1]);
+ }
+
+ @Override
+ public int findPosition(long offset) {
+ return binarySearch(offset);
+ }
+
+ private int binarySearch(long offset) {
+ int bucket = (int) (offset / bucketSize);
+ int low = bucket == 0 ? 0 : offsetIndex[bucket - 1];
+ int high = offsetIndex[bucket];
+ while (low < high) {
+ final int mid = (low + high) >>> 1;
+ final long o = index.getOffset(nth[mid]);
+ if (offset < o) {
+ high = mid;
+ } else if (offset == o) {
+ return mid;
+ } else {
+ low = mid + 1;
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public ObjectId findObjectByPosition(int nthPosition) {
+ return index.getObjectId(nth[nthPosition]);
+ }
+}
--- /dev/null
+/*
+ * Copyright (C) 2023, Google LLC and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+/**
+ * Factory for creating instances of {@link PackReverseIndex}.
+ */
+public final class PackReverseIndexFactory {
+ /**
+ * Compute an in-memory pack reverse index from the in-memory pack forward
+ * index. This computation uses insertion sort, which has a quadratic
+ * runtime on average.
+ *
+ * @param packIndex
+ * the forward index to compute from
+ * @return the reverse index instance
+ */
+ public static PackReverseIndex computeFromIndex(PackIndex packIndex) {
+ return new PackReverseIndexComputed(packIndex);
+ }
+}
projects / jgit.git / commitdiff