Added read/write support for pack bitmap index.

A pack bitmap index is an additional index of compressed
bitmaps of the object graph. Furthermore, a logical API of the index
functionality is included, as it is expected to be used by the
PackWriter.

Compressed bitmaps are created using the javaewah library, which is a
word-aligned compressed variant of the Java bitset class based on
run-length encoding. The library only works with positive integer
values. Thus, the maximum number of ObjectIds in a pack file that
this index can currently support is limited to Integer.MAX_VALUE.

Every ObjectId is given an integer mapping. The integer is the
position of the ObjectId in the complete ObjectId list, sorted
by offset, for the pack file. That integer is what the bitmaps
use to reference the ObjectId. Currently, the new index format can
only be used with pack files that contain a complete closure of the
object graph e.g. the result of a garbage collection.

The index file includes four bitmaps for the Git object types i.e.
commits, trees, blobs, and tags. In addition, a collection of
bitmaps keyed by an ObjectId is also included. The bitmap for each entry
in the collection represents the full closure of ObjectIds reachable
from the keyed ObjectId (including the keyed ObjectId itself). The
bitmaps are further compressed by XORing the current bitmaps against
prior bitmaps in the index, and selecting the smallest representation.
The XOR'd bitmap and offset from the current entry to the position
of the bitmap to XOR against is the actual representation of the entry
in the index file. Each entry contains one byte, which is currently
used to note whether the bitmap should be blindly reused.

Change-Id: Id328724bf6b4c8366a088233098c18643edcf40f

org.eclipse.jgit.test/META-INF/MANIFEST.MF

@@ -7,7 +7,8 @@ Bundle-Localization: plugin
 Bundle-Vendor: %provider_name
 Bundle-ActivationPolicy: lazy
 Bundle-RequiredExecutionEnvironment: J2SE-1.5
-Import-Package: org.eclipse.jgit.api;version="[2.4.0,2.5.0)",
+Import-Package: javaewah;version="0.5.6",
+ org.eclipse.jgit.api;version="[2.4.0,2.5.0)",
  org.eclipse.jgit.api.errors;version="[2.4.0,2.5.0)",
  org.eclipse.jgit.awtui;version="[2.4.0,2.5.0)",
  org.eclipse.jgit.blame;version="[2.4.0,2.5.0)",

org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/InflatingBitSetTest.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.junit.Test;
+
+public class InflatingBitSetTest {
+
+	@Test
+	public void testMaybeContains() {
+		EWAHCompressedBitmap ecb = new EWAHCompressedBitmap();
+		ecb.set(63);
+		ecb.set(64);
+		ecb.set(128);
+
+		InflatingBitSet ibs = new InflatingBitSet(ecb);
+		assertTrue(ibs.maybeContains(0));
+		assertFalse(ibs.contains(0)); // Advance
+		assertFalse(ibs.maybeContains(0));
+		assertTrue(ibs.maybeContains(63));
+		assertTrue(ibs.maybeContains(64));
+		assertTrue(ibs.maybeContains(65));
+		assertFalse(ibs.maybeContains(129));
+	}
+
+	@Test
+	public void testContainsMany() {
+		EWAHCompressedBitmap ecb = new EWAHCompressedBitmap();
+		ecb.set(64);
+		ecb.set(65);
+		ecb.set(1024);
+
+		InflatingBitSet ibs = new InflatingBitSet(ecb);
+		assertFalse(ibs.contains(0));
+		assertTrue(ibs.contains(64));
+		assertTrue(ibs.contains(65));
+		assertFalse(ibs.contains(66));
+		assertTrue(ibs.contains(1024));
+		assertFalse(ibs.contains(1025));
+	}
+
+	@Test
+	public void testContainsOne() {
+		EWAHCompressedBitmap ecb = new EWAHCompressedBitmap();
+		ecb.set(64);
+
+		InflatingBitSet ibs = new InflatingBitSet(ecb);
+		assertTrue(ibs.contains(64));
+		assertTrue(ibs.contains(64));
+		assertFalse(ibs.contains(65));
+		assertFalse(ibs.contains(63));
+	}
+
+	@Test
+	public void testContainsEmpty() {
+		InflatingBitSet ibs = new InflatingBitSet(new EWAHCompressedBitmap());
+		assertFalse(ibs.maybeContains(0));
+		assertFalse(ibs.contains(0));
+	}
+}

org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/StoredBitmapTest.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import static org.junit.Assert.*;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.storage.file.BasePackBitmapIndex.StoredBitmap;
+import org.junit.Test;
+
+public class StoredBitmapTest {
+
+	@Test
+	public void testGetBitmapWithoutXor() {
+		EWAHCompressedBitmap b = bitmapOf(100);
+		StoredBitmap sb = newStoredBitmap(bitmapOf(100));
+		assertEquals(b, sb.getBitmap());
+	}
+
+	@Test
+	public void testGetBitmapWithOneXor() {
+		StoredBitmap sb = newStoredBitmap(bitmapOf(100), bitmapOf(100, 101));
+		assertEquals(bitmapOf(101), sb.getBitmap());
+	}
+
+	@Test
+	public void testGetBitmapWithThreeXor() {
+		StoredBitmap sb = newStoredBitmap(
+				bitmapOf(100),
+				bitmapOf(90, 101),
+				bitmapOf(100, 101),
+				bitmapOf(50));
+		assertEquals(bitmapOf(50, 90), sb.getBitmap());
+		assertEquals(bitmapOf(50, 90), sb.getBitmap());
+	}
+
+	private static final StoredBitmap newStoredBitmap(
+			EWAHCompressedBitmap... bitmaps) {
+		StoredBitmap sb = null;
+		for (EWAHCompressedBitmap bitmap : bitmaps)
+			sb = new StoredBitmap(ObjectId.zeroId(), bitmap, sb, 0);
+		return sb;
+	}
+
+	private static final EWAHCompressedBitmap bitmapOf(int... bits) {
+		EWAHCompressedBitmap b = new EWAHCompressedBitmap();
+		for (int bit : bits)
+			b.set(bit);
+		return b;
+	}
+}

org.eclipse.jgit/META-INF/MANIFEST.MF

@@ -36,7 +36,8 @@ Export-Package: org.eclipse.jgit.api;version="2.4.0",
 Bundle-ActivationPolicy: lazy
 Bundle-RequiredExecutionEnvironment: J2SE-1.5
 Require-Bundle: com.jcraft.jsch;bundle-version="[0.1.37,0.2.0)"
-Import-Package: javax.crypto,
+Import-Package: javaewah;version="0.5.6",
+ javax.crypto,
  javax.net.ssl,
  org.xml.sax,
  org.xml.sax.helpers

org.eclipse.jgit/pom.xml

@@ -73,6 +73,11 @@
       <groupId>com.jcraft</groupId>
       <artifactId>jsch</artifactId>
     </dependency>
+
+    <dependency>
+      <groupId>com.googlecode.javaewah</groupId>
+      <artifactId>JavaEWAH</artifactId>
+    </dependency>
   </dependencies>
 
   <build>

org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@@ -196,6 +196,7 @@ expectedBooleanStringValue=Expected boolean string value
 expectedCharacterEncodingGuesses=Expected {0} character encoding guesses
 expectedEOFReceived=expected EOF; received ''{0}'' instead
 expectedGot=expected ''{0}'', got ''{1}''
+expectedLessThanGot=expected less than ''{0}'', got ''{1}''
 expectedPktLineWithService=expected pkt-line with '# service=-', got ''{0}''
 expectedReceivedContentType=expected Content-Type {0}; received Content-Type {1}
 expectedReportForRefNotReceived={0}: expected report for ref {1} not received
@@ -336,6 +337,7 @@ objectAtHasBadZlibStream=Object at {0} in {1} has bad zlib stream
 objectAtPathDoesNotHaveId=Object at path "{0}" does not have an id assigned. All object ids must be assigned prior to writing a tree.
 objectIsCorrupt=Object {0} is corrupt: {1}
 objectIsNotA=Object {0} is not a {1}.
+objectNotFound=Object {0} not found.
 objectNotFoundIn=Object {0} not found in {1}.
 obtainingCommitsForCherryPick=Obtaining commits that need to be cherry-picked
 offsetWrittenDeltaBaseForObjectNotFoundInAPack=Offset-written delta base for object not found in a pack

org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

@@ -258,6 +258,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String expectedCharacterEncodingGuesses;
 	/***/ public String expectedEOFReceived;
 	/***/ public String expectedGot;
+	/***/ public String expectedLessThanGot;
 	/***/ public String expectedPktLineWithService;
 	/***/ public String expectedReceivedContentType;
 	/***/ public String expectedReportForRefNotReceived;
@@ -398,6 +399,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String objectAtPathDoesNotHaveId;
 	/***/ public String objectIsCorrupt;
 	/***/ public String objectIsNotA;
+	/***/ public String objectNotFound;
 	/***/ public String objectNotFoundIn;
 	/***/ public String obtainingCommitsForCherryPick;
 	/***/ public String offsetWrittenDeltaBaseForObjectNotFoundInAPack;

org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+import java.util.Iterator;
+
+import org.eclipse.jgit.storage.file.PackBitmapIndex;
+
+/** A compressed bitmap representation of the entire object graph. */
+public interface BitmapIndex {
+	/**
+	 * Get the bitmap for the id. The returned bitmap is immutable and the
+	 * bitwise operations return the result of the operation in a new Bitmap.
+	 *
+	 * @param objectId
+	 *            the object ID
+	 * @return the Bitmap for the objectId or null, if one does not exist.
+	 */
+	Bitmap getBitmap(AnyObjectId objectId);
+
+	/** @return a new {@code BitmapBuilder} based on the values in the index. */
+	BitmapBuilder newBitmapBuilder();
+
+	/**
+	 * A bitmap representation of ObjectIds that can be iterated to return the
+	 * underlying {@code ObjectId}s or operated on with other {@code Bitmap}s.
+	 */
+	public interface Bitmap extends Iterable<BitmapObject> {
+		/**
+		 * Bitwise-OR the current bitmap with the value from the other
+		 * bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return a bitmap that is the bitwise-OR.
+		 */
+		Bitmap or(Bitmap other);
+
+		/**
+		 * Bitwise-AND-NOT the current bitmap with the value from the other
+		 * bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return a bitmap that is the bitwise-AND-NOT.
+		 */
+		Bitmap andNot(Bitmap other);
+
+		/**
+		 * Bitwise-XOR the current bitmap with the value from the other
+		 * bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return a bitmap that is the bitwise-XOR.
+		 */
+		Bitmap xor(Bitmap other);
+
+		/**
+		 * Returns an iterator over a set of elements of type BitmapObject. The
+		 * BitmapObject instance is reused across calls to
+		 * {@link Iterator#next()} for performance reasons.
+		 *
+		 * @return an Iterator.
+		 */
+		Iterator<BitmapObject> iterator();
+	}
+
+	/**
+	 * A builder for a bitmap. The bitwise operations update the builder and
+	 * return a reference to the current builder.
+	 */
+	public interface BitmapBuilder extends Bitmap {
+		/**
+		 * Adds the id and the existing bitmap for the id, if one exists, to the
+		 * bitmap.
+		 *
+		 * @param objectId
+		 *            the object ID
+		 * @param type
+		 *            the Git object type. See {@link Constants}.
+		 * @return true if the value was not contained or able to be loaded.
+		 */
+		boolean add(AnyObjectId objectId, int type);
+
+		/**
+		 * Whether the bitmap has the id set.
+		 *
+		 * @param objectId
+		 *            the object ID
+		 * @return whether the bitmap currently contains the object ID
+		 */
+		boolean contains(AnyObjectId objectId);
+
+		/**
+		 * Remove the id from the bitmap.
+		 *
+		 * @param objectId
+		 *            the object ID
+		 */
+		void remove(AnyObjectId objectId);
+
+		/**
+		 * Bitwise-OR the current bitmap with the value from the other bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return the current builder.
+		 */
+		BitmapBuilder or(Bitmap other);
+
+		/**
+		 * Bitwise-AND-NOT the current bitmap with the value from the other
+		 * bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return the current builder.
+		 */
+		BitmapBuilder andNot(Bitmap other);
+
+		/**
+		 * Bitwise-XOR the current bitmap with the value from the other bitmap.
+		 *
+		 * @param other
+		 *            the other bitmap
+		 * @return the current builder.
+		 */
+		BitmapBuilder xor(Bitmap other);
+
+		/** @return the fully built immutable bitmap */
+		Bitmap build();
+
+		/**
+		 * Determines if the entire bitmap index is contained in the bitmap. If
+		 * it is, the matching bits are removed from the bitmap and true is
+		 * returned. If the bitmap index is null, false is returned.
+		 *
+		 * @param bitmapIndex
+		 *            the bitmap index to check if it is completely contained
+		 *            inside of the current bitmap.
+		 * @return {@code true} if the bitmap index was a complete match.
+		 */
+		boolean removeAllOrNone(PackBitmapIndex bitmapIndex);
+
+		/** @return the number of elements in the bitmap. */
+		int cardinality();
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapObject.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+/** Base object type accessed during bitmap expansion. */
+public abstract class BitmapObject {
+	/**
+	 * Get Git object type. See {@link Constants}.
+	 *
+	 * @return object type
+	 */
+	public abstract int getType();
+
+	/**
+	 * Get the name of this object.
+	 *
+	 * @return unique hash of this object.
+	 */
+	public abstract ObjectId getObjectId();
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java

@@ -436,6 +436,17 @@ public abstract class ObjectReader {
 		// Do nothing by default, most readers don't want or need advice.
 	}
 
+	/**
+	 * An index that can be used to speed up ObjectWalks.
+	 *
+	 * @return the index or null if one does not exist.
+	 * @throws IOException
+	 *             when the index fails to load
+	 */
+	public BitmapIndex getBitmapIndex() throws IOException {
+		return null;
+	}
+
 	/**
 	 * Release any resources used by this reader.
 	 * <p>

org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java

@@ -69,7 +69,7 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 
 	private long lastModified;
 
-	private Map<PackExt, Long> sizeMap;
+	private final Map<PackExt, Long> sizeMap;
 
 	private long objectCount;
 
@@ -79,6 +79,8 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 
 	private PackWriter.Statistics stats;
 
+	private int extensions;
+
 	/**
 	 * Initialize a description by pack name and repository.
 	 * <p>
@@ -98,7 +100,7 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 		this.repoDesc = repoDesc;
 		int dot = name.lastIndexOf('.');
 		this.packName = (dot < 0) ? name : name.substring(0, dot);
-		this.sizeMap = new HashMap<PackExt, Long>(5);
+		this.sizeMap = new HashMap<PackExt, Long>(PackExt.values().length * 2);
 	}
 
 	/** @return description of the repository. */
@@ -106,11 +108,30 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 		return repoDesc;
 	}
 
+	/**
+	 * Adds the pack file extension to the known list.
+	 *
+	 * @param ext
+	 *            the file extension
+	 */
+	public void addFileExt(PackExt ext) {
+		extensions |= ext.getBit();
+	}
+
+	/**
+	 * @param ext
+	 *            the file extension
+	 * @return whether the pack file extensions is known to exist.
+	 */
+	public boolean hasFileExt(PackExt ext) {
+		return (extensions & ext.getBit()) != 0;
+	}
+
 	/**
 	 * @param ext
 	 *            the file extension
 	 * @return name of the file.
-	 * */
+	 */
 	public String getFileName(PackExt ext) {
 		return packName + '.' + ext.getExtension();
 	}

org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackFile.java

@@ -45,6 +45,7 @@
 
 package org.eclipse.jgit.storage.dfs;
 
+import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
 import static org.eclipse.jgit.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
 
@@ -71,9 +72,11 @@ import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.PackBitmapIndex;
 import org.eclipse.jgit.storage.file.PackIndex;
 import org.eclipse.jgit.storage.file.PackReverseIndex;
 import org.eclipse.jgit.storage.pack.BinaryDelta;
+import org.eclipse.jgit.storage.pack.PackExt;
 import org.eclipse.jgit.storage.pack.PackOutputStream;
 import org.eclipse.jgit.storage.pack.StoredObjectRepresentation;
 import org.eclipse.jgit.util.IO;
@@ -99,6 +102,9 @@ public final class DfsPackFile {
 	/** Offset used to cache {@link #reverseIndex}. See {@link #POS_INDEX}. */
 	private static final long POS_REVERSE_INDEX = -2;
 
+	/** Offset used to cache {@link #bitmapIndex}. See {@link #POS_INDEX}. */
+	private static final long POS_BITMAP_INDEX = -3;
+
 	/** Cache that owns this pack file and its data. */
 	private final DfsBlockCache cache;
 
@@ -141,6 +147,9 @@ public final class DfsPackFile {
 	/** Reverse version of {@link #index} mapping position to {@link ObjectId}. */
 	private volatile DfsBlockCache.Ref<PackReverseIndex> reverseIndex;
 
+	/** Index of compressed bitmap mapping entire object graph. */
+	private volatile DfsBlockCache.Ref<PackBitmapIndex> bitmapIndex;
+
 	/**
 	 * Objects we have tried to read, and discovered to be corrupt.
 	 * <p>
@@ -267,6 +276,64 @@ public final class DfsPackFile {
 		}
 	}
 
+	PackBitmapIndex getPackBitmapIndex(DfsReader ctx) throws IOException {
+		DfsBlockCache.Ref<PackBitmapIndex> idxref = bitmapIndex;
+		if (idxref != null) {
+			PackBitmapIndex idx = idxref.get();
+			if (idx != null)
+				return idx;
+		}
+
+		if (!packDesc.hasFileExt(PackExt.BITMAP_INDEX))
+			return null;
+
+		synchronized (initLock) {
+			idxref = bitmapIndex;
+			if (idxref != null) {
+				PackBitmapIndex idx = idxref.get();
+				if (idx != null)
+					return idx;
+			}
+
+			long size;
+			PackBitmapIndex idx;
+			try {
+				ReadableChannel rc = ctx.db.openFile(packDesc, BITMAP_INDEX);
+				try {
+					InputStream in = Channels.newInputStream(rc);
+					int wantSize = 8192;
+					int bs = rc.blockSize();
+					if (0 < bs && bs < wantSize)
+						bs = (wantSize / bs) * bs;
+					else if (bs <= 0)
+						bs = wantSize;
+					in = new BufferedInputStream(in, bs);
+					idx = PackBitmapIndex.read(
+							in, idx(ctx), getReverseIdx(ctx));
+				} finally {
+					size = rc.position();
+					rc.close();
+				}
+			} catch (EOFException e) {
+				IOException e2 = new IOException(MessageFormat.format(
+						DfsText.get().shortReadOfIndex,
+						packDesc.getFileName(BITMAP_INDEX)));
+				e2.initCause(e);
+				throw e2;
+			} catch (IOException e) {
+				IOException e2 = new IOException(MessageFormat.format(
+						DfsText.get().cannotReadIndex,
+						packDesc.getFileName(BITMAP_INDEX)));
+				e2.initCause(e);
+				throw e2;
+			}
+
+			bitmapIndex = cache.put(key, POS_BITMAP_INDEX,
+					(int) Math.min(size, Integer.MAX_VALUE), idx);
+			return idx;
+		}
+	}
+
 	private PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
 		DfsBlockCache.Ref<PackReverseIndex> revref = reverseIndex;
 		if (revref != null) {

org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsReader.java

@@ -75,6 +75,7 @@ import org.eclipse.jgit.lib.AbbreviatedObjectId;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
 import org.eclipse.jgit.lib.AsyncObjectSizeQueue;
+import org.eclipse.jgit.lib.BitmapIndex;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.ObjectId;
@@ -84,6 +85,8 @@ import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.revwalk.ObjectWalk;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.file.BitmapIndexImpl;
+import org.eclipse.jgit.storage.file.PackBitmapIndex;
 import org.eclipse.jgit.storage.pack.CachedPack;
 import org.eclipse.jgit.storage.pack.ObjectReuseAsIs;
 import org.eclipse.jgit.storage.pack.ObjectToPack;
@@ -139,6 +142,16 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 		return new DfsReader(db);
 	}
 
+	@Override
+	public BitmapIndex getBitmapIndex() throws IOException {
+		for (DfsPackFile pack : db.getPacks()) {
+			PackBitmapIndex bitmapIndex = pack.getPackBitmapIndex(this);
+			if (bitmapIndex != null)
+				return new BitmapIndexImpl(bitmapIndex);
+		}
+		return null;
+	}
+
 	@Override
 	public Collection<ObjectId> resolve(AbbreviatedObjectId id)
 			throws IOException {

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/BasePackBitmapIndex.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectIdOwnerMap;
+
+/**
+ * Base implementation of the PackBitmapIndex.
+ */
+abstract class BasePackBitmapIndex extends PackBitmapIndex {
+	private final ObjectIdOwnerMap<StoredBitmap> bitmaps;
+
+	BasePackBitmapIndex(ObjectIdOwnerMap<StoredBitmap> bitmaps) {
+		this.bitmaps = bitmaps;
+	}
+
+	public EWAHCompressedBitmap getBitmap(AnyObjectId objectId) {
+		StoredBitmap sb = bitmaps.get(objectId);
+		return sb != null ? sb.getBitmap() : null;
+	}
+
+	ObjectIdOwnerMap<StoredBitmap> getBitmaps() {
+		return bitmaps;
+	}
+
+	/**
+	 * Data representation of the bitmap entry restored from a pack index. The
+	 * commit of the bitmap is the map key.
+	 */
+	static final class StoredBitmap extends ObjectIdOwnerMap.Entry {
+		private volatile Object bitmapContainer;
+		private final int flags;
+
+		StoredBitmap(AnyObjectId objectId, EWAHCompressedBitmap bitmap,
+				StoredBitmap xorBitmap, int flags) {
+			super(objectId);
+			this.bitmapContainer = xorBitmap == null
+					? bitmap
+					: new XorCompressedBitmap(bitmap, xorBitmap);
+			this.flags = flags;
+		}
+
+		/**
+		 * Computes and returns the full bitmap.
+		 *
+		 * @return the full bitmap
+		 */
+		EWAHCompressedBitmap getBitmap() {
+			// Fast path to immediately return the expanded result.
+			Object r = bitmapContainer;
+			if (r instanceof EWAHCompressedBitmap)
+				return (EWAHCompressedBitmap) r;
+
+			// Expand the bitmap and cache the result.
+			XorCompressedBitmap xb = (XorCompressedBitmap) r;
+			EWAHCompressedBitmap out = xb.bitmap;
+			for (;;) {
+				r = xb.xorBitmap.bitmapContainer;
+				if (r instanceof EWAHCompressedBitmap) {
+					out = out.xor((EWAHCompressedBitmap) r);
+					bitmapContainer = out;
+					return out;
+				}
+				xb = (XorCompressedBitmap) r;
+				out = out.xor(xb.bitmap);
+			}
+		}
+
+		/** @return the flags associated with the bitmap */
+		int getFlags() {
+			return flags;
+		}
+	}
+
+	private static final class XorCompressedBitmap {
+		final EWAHCompressedBitmap bitmap;
+		final StoredBitmap xorBitmap;
+
+		XorCompressedBitmap(EWAHCompressedBitmap b, StoredBitmap xb) {
+			bitmap = b;
+			xorBitmap = xb;
+		}
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/BitSet.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.util.Arrays;
+
+import javaewah.EWAHCompressedBitmap;
+
+/**
+ * A random access BitSet to supports efficient conversions to
+ * EWAHCompressedBitmap.
+ */
+final class BitSet {
+
+	private long[] words;
+
+	BitSet(int initialCapacity) {
+		words = new long[block(initialCapacity) + 1];
+	}
+
+	final void clear() {
+		Arrays.fill(words, 0);
+	}
+
+	final void set(int position) {
+		int block = block(position);
+		if (block >= words.length) {
+			long[] buf = new long[2 * block(position)];
+			System.arraycopy(words, 0, buf, 0, words.length);
+			words = buf;
+		}
+		words[block] |= mask(position);
+	}
+
+	final void clear(int position) {
+		int block = block(position);
+		if (block < words.length)
+			words[block] &= ~mask(position);
+	}
+
+	final boolean get(int position) {
+		int block = block(position);
+		return block < words.length && (words[block] & mask(position)) != 0;
+	}
+
+	final EWAHCompressedBitmap toEWAHCompressedBitmap() {
+		EWAHCompressedBitmap compressed = new EWAHCompressedBitmap(
+				words.length);
+		int runningEmptyWords = 0;
+		long lastNonEmptyWord = 0;
+		for (long word : words) {
+			if (word == 0) {
+				runningEmptyWords++;
+				continue;
+			}
+
+			if (lastNonEmptyWord != 0)
+				compressed.add(lastNonEmptyWord);
+
+			if (runningEmptyWords > 0) {
+				compressed.addStreamOfEmptyWords(false, runningEmptyWords);
+				runningEmptyWords = 0;
+			}
+
+			lastNonEmptyWord = word;
+		}
+		int bitsThatMatter = 64 - Long.numberOfLeadingZeros(lastNonEmptyWord);
+		if (bitsThatMatter > 0)
+			compressed.add(lastNonEmptyWord, bitsThatMatter);
+		return compressed;
+	}
+
+	private static final int block(int position) {
+		return position >> 6;
+	}
+
+	private static final long mask(int position) {
+		return 1L << position;
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/BitmapIndexImpl.java

@@ -0,0 +1,484 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.text.MessageFormat;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import javaewah.EWAHCompressedBitmap;
+import javaewah.IntIterator;
+
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BitmapIndex;
+import org.eclipse.jgit.lib.BitmapObject;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdOwnerMap;
+import org.eclipse.jgit.util.BlockList;
+
+/** A compressed bitmap representation of the entire object graph. */
+public class BitmapIndexImpl implements BitmapIndex {
+	private static final int EXTRA_BITS = 10 * 1024;
+
+	private final PackBitmapIndex packIndex;
+
+	private final MutableBitmapIndex mutableIndex;
+
+	private final int indexObjectCount;
+
+	/**
+	 * Creates a BitmapIndex that is back by Compressed bitmaps.
+	 *
+	 * @param packIndex
+	 *            the bitmap index for the pack.
+	 */
+	public BitmapIndexImpl(PackBitmapIndex packIndex) {
+		this.packIndex = packIndex;
+		mutableIndex = new MutableBitmapIndex();
+		indexObjectCount = packIndex.getObjectCount();
+	}
+
+	PackBitmapIndex getPackBitmapIndex() {
+		return packIndex;
+	}
+
+	public CompressedBitmap getBitmap(AnyObjectId objectId) {
+		EWAHCompressedBitmap compressed = packIndex.getBitmap(objectId);
+		if (compressed == null)
+			return null;
+		return new CompressedBitmap(compressed);
+	}
+
+	public CompressedBitmapBuilder newBitmapBuilder() {
+		return new CompressedBitmapBuilder();
+	}
+
+	private int findPosition(AnyObjectId objectId) {
+		int position = packIndex.findPosition(objectId);
+		if (position < 0) {
+			position = mutableIndex.findPosition(objectId);
+			if (position >= 0)
+				position += indexObjectCount;
+		}
+		return position;
+	}
+
+	private int addObject(AnyObjectId objectId, int type) {
+		int position = findPosition(objectId);
+		if (position < 0) {
+			position = mutableIndex.addObject(objectId, type);
+			position += indexObjectCount;
+		}
+		return position;
+	}
+
+	private static final class ComboBitset {
+		private InflatingBitSet inflatingBitmap;
+
+		private BitSet toAdd;
+
+		private BitSet toRemove;
+
+		private ComboBitset() {
+			this(new EWAHCompressedBitmap());
+		}
+
+		private ComboBitset(EWAHCompressedBitmap bitmap) {
+			this.inflatingBitmap = new InflatingBitSet(bitmap);
+		}
+
+		EWAHCompressedBitmap combine() {
+			EWAHCompressedBitmap toAddCompressed = null;
+			if (toAdd != null) {
+				toAddCompressed = toAdd.toEWAHCompressedBitmap();
+				toAdd = null;
+			}
+
+			EWAHCompressedBitmap toRemoveCompressed = null;
+			if (toRemove != null) {
+				toRemoveCompressed = toRemove.toEWAHCompressedBitmap();
+				toRemove = null;
+			}
+
+			if (toAddCompressed != null)
+				or(toAddCompressed);
+			if (toRemoveCompressed != null)
+				andNot(toRemoveCompressed);
+			return inflatingBitmap.getBitmap();
+		}
+
+		void or(EWAHCompressedBitmap inbits) {
+			if (toRemove != null)
+				combine();
+			inflatingBitmap = inflatingBitmap.or(inbits);
+		}
+
+		void andNot(EWAHCompressedBitmap inbits) {
+			if (toAdd != null || toRemove != null)
+				combine();
+			inflatingBitmap = inflatingBitmap.andNot(inbits);
+		}
+
+		void xor(EWAHCompressedBitmap inbits) {
+			if (toAdd != null || toRemove != null)
+				combine();
+			inflatingBitmap = inflatingBitmap.xor(inbits);
+		}
+
+		boolean contains(int position) {
+			if (toRemove != null && toRemove.get(position))
+				return false;
+			if (toAdd != null && toAdd.get(position))
+				return true;
+			return inflatingBitmap.contains(position);
+		}
+
+		void remove(int position) {
+			if (toAdd != null)
+				toAdd.clear(position);
+
+			if (inflatingBitmap.maybeContains(position)) {
+				if (toRemove == null)
+					toRemove = new BitSet(position + EXTRA_BITS);
+				toRemove.set(position);
+			}
+		}
+
+		void set(int position) {
+			if (toRemove != null)
+				toRemove.clear(position);
+
+			if (toAdd == null)
+				toAdd = new BitSet(position + EXTRA_BITS);
+			toAdd.set(position);
+		}
+	}
+
+	private final class CompressedBitmapBuilder implements BitmapBuilder {
+		private ComboBitset bitset = new ComboBitset();
+
+		public boolean add(AnyObjectId objectId, int type) {
+			int position = addObject(objectId, type);
+			if (bitset.contains(position))
+				return false;
+
+			Bitmap entry = getBitmap(objectId);
+			if (entry != null) {
+				or(entry);
+				return false;
+			}
+
+			bitset.set(position);
+			return true;
+		}
+
+		public boolean contains(AnyObjectId objectId) {
+			int position = findPosition(objectId);
+			return 0 <= position && bitset.contains(position);
+		}
+
+		public void remove(AnyObjectId objectId) {
+			int position = findPosition(objectId);
+			if (0 <= position)
+				bitset.remove(position);
+		}
+
+		public CompressedBitmapBuilder or(Bitmap other) {
+			if (isSameCompressedBitmap(other)) {
+				bitset.or(((CompressedBitmap) other).bitmap);
+			} else if (isSameCompressedBitmapBuilder(other)) {
+				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
+				bitset.or(b.bitset.combine());
+			} else {
+				throw new IllegalArgumentException();
+			}
+			return this;
+		}
+
+		public CompressedBitmapBuilder andNot(Bitmap other) {
+			if (isSameCompressedBitmap(other)) {
+				bitset.andNot(((CompressedBitmap) other).bitmap);
+			} else if (isSameCompressedBitmapBuilder(other)) {
+				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
+				bitset.andNot(b.bitset.combine());
+			} else {
+				throw new IllegalArgumentException();
+			}
+			return this;
+		}
+
+		public CompressedBitmapBuilder xor(Bitmap other) {
+			if (isSameCompressedBitmap(other)) {
+				bitset.xor(((CompressedBitmap) other).bitmap);
+			} else if (isSameCompressedBitmapBuilder(other)) {
+				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
+				bitset.xor(b.bitset.combine());
+			} else {
+				throw new IllegalArgumentException();
+			}
+			return this;
+		}
+
+		/** @return the fully built immutable bitmap */
+		public CompressedBitmap build() {
+			return new CompressedBitmap(bitset.combine());
+		}
+
+		public Iterator<BitmapObject> iterator() {
+			return build().iterator();
+		}
+
+		public int cardinality() {
+			return bitset.combine().cardinality();
+		}
+
+		public boolean removeAllOrNone(PackBitmapIndex index) {
+			if (!packIndex.equals(index))
+				return false;
+
+			EWAHCompressedBitmap curr = bitset.combine()
+					.xor(ones(indexObjectCount));
+
+			IntIterator ii = curr.intIterator();
+			if (ii.hasNext() && ii.next() < indexObjectCount)
+				return false;
+			bitset = new ComboBitset(curr);
+			return true;
+		}
+
+		private BitmapIndexImpl getBitmapIndex() {
+			return BitmapIndexImpl.this;
+		}
+	}
+
+	final class CompressedBitmap implements Bitmap {
+		private final EWAHCompressedBitmap bitmap;
+
+		private CompressedBitmap(EWAHCompressedBitmap bitmap) {
+			this.bitmap = bitmap;
+		}
+
+		public CompressedBitmap or(Bitmap other) {
+			return new CompressedBitmap(bitmap.or(bitmapOf(other)));
+		}
+
+		public CompressedBitmap andNot(Bitmap other) {
+			return new CompressedBitmap(bitmap.andNot(bitmapOf(other)));
+		}
+
+		public CompressedBitmap xor(Bitmap other) {
+			return new CompressedBitmap(bitmap.xor(bitmapOf(other)));
+		}
+
+		private EWAHCompressedBitmap bitmapOf(Bitmap other) {
+			if (isSameCompressedBitmap(other))
+				return ((CompressedBitmap) other).bitmap;
+			if (isSameCompressedBitmapBuilder(other))
+				return ((CompressedBitmapBuilder) other).build().bitmap;
+			CompressedBitmapBuilder builder = newBitmapBuilder();
+			builder.or(other);
+			return builder.build().bitmap;
+		}
+
+		private final IntIterator ofObjectType(int type) {
+			return packIndex.ofObjectType(bitmap, type).intIterator();
+		}
+
+		public Iterator<BitmapObject> iterator() {
+			final IntIterator dynamic = bitmap.andNot(ones(indexObjectCount))
+					.intIterator();
+			final IntIterator commits = ofObjectType(Constants.OBJ_COMMIT);
+			final IntIterator trees = ofObjectType(Constants.OBJ_TREE);
+			final IntIterator blobs = ofObjectType(Constants.OBJ_BLOB);
+			final IntIterator tags = ofObjectType(Constants.OBJ_TAG);
+			return new Iterator<BitmapObject>() {
+				private final BitmapObjectImpl out = new BitmapObjectImpl();
+				private int type;
+				private IntIterator cached = dynamic;
+
+				public boolean hasNext() {
+					if (!cached.hasNext()) {
+						if (commits.hasNext()) {
+							type = Constants.OBJ_COMMIT;
+							cached = commits;
+						} else if (trees.hasNext()) {
+							type = Constants.OBJ_TREE;
+							cached = trees;
+						} else if (blobs.hasNext()) {
+							type = Constants.OBJ_BLOB;
+							cached = blobs;
+						} else if (tags.hasNext()) {
+							type = Constants.OBJ_TAG;
+							cached = tags;
+						} else {
+							return false;
+						}
+					}
+					return true;
+				}
+
+				public BitmapObject next() {
+					if (!hasNext())
+						throw new NoSuchElementException();
+
+					int position = cached.next();
+					if (position < indexObjectCount) {
+						out.type = type;
+						out.objectId = packIndex.getObject(position);
+					} else {
+						position -= indexObjectCount;
+						MutableEntry entry = mutableIndex.getObject(position);
+						out.type = entry.type;
+						out.objectId = entry;
+					}
+					return out;
+				}
+
+				public void remove() {
+					throw new UnsupportedOperationException();
+				}
+			};
+		}
+
+		EWAHCompressedBitmap getEwahCompressedBitmap() {
+			return bitmap;
+		}
+
+		private BitmapIndexImpl getPackBitmapIndex() {
+			return BitmapIndexImpl.this;
+		}
+	}
+
+	private static final class MutableBitmapIndex {
+		private final ObjectIdOwnerMap<MutableEntry>
+				revMap = new ObjectIdOwnerMap<MutableEntry>();
+
+		private final BlockList<MutableEntry>
+				revList = new BlockList<MutableEntry>();
+
+		int findPosition(AnyObjectId objectId) {
+			MutableEntry entry = revMap.get(objectId);
+			if (entry == null)
+				return -1;
+			return entry.position;
+		}
+
+		MutableEntry getObject(int position) {
+			try {
+				MutableEntry entry = revList.get(position);
+				if (entry == null)
+					throw new IllegalArgumentException(MessageFormat.format(
+							JGitText.get().objectNotFound,
+							String.valueOf(position)));
+				return entry;
+			} catch (IndexOutOfBoundsException ex) {
+				throw new IllegalArgumentException(ex);
+			}
+		}
+
+		int addObject(AnyObjectId objectId, int type) {
+			MutableEntry entry = new MutableEntry(
+					objectId, type, revList.size());
+			revList.add(entry);
+			revMap.add(entry);
+			return entry.position;
+		}
+	}
+
+	private static final class MutableEntry extends ObjectIdOwnerMap.Entry {
+		private final int type;
+
+		private final int position;
+
+		MutableEntry(AnyObjectId objectId, int type, int position) {
+			super(objectId);
+			this.type = type;
+			this.position = position;
+		}
+	}
+
+	private static final class BitmapObjectImpl extends BitmapObject {
+		private ObjectId objectId;
+
+		private int type;
+
+		@Override
+		public ObjectId getObjectId() {
+			return objectId;
+		}
+
+		@Override
+		public int getType() {
+			return type;
+		}
+	}
+
+	private boolean isSameCompressedBitmap(Bitmap other) {
+		if (other instanceof CompressedBitmap) {
+			CompressedBitmap b = (CompressedBitmap) other;
+			return this == b.getPackBitmapIndex();
+		}
+		return false;
+	}
+
+	private boolean isSameCompressedBitmapBuilder(Bitmap other) {
+		if (other instanceof CompressedBitmapBuilder) {
+			CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
+			return this == b.getBitmapIndex();
+		}
+		return false;
+	}
+
+	private static final EWAHCompressedBitmap ones(int sizeInBits) {
+		EWAHCompressedBitmap mask = new EWAHCompressedBitmap();
+		mask.addStreamOfEmptyWords(
+				true, sizeInBits / EWAHCompressedBitmap.wordinbits);
+		int remaining = sizeInBits % EWAHCompressedBitmap.wordinbits;
+		if (remaining > 0)
+			mask.add((1L << remaining) - 1, remaining);
+		return mask;
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java

@@ -262,6 +262,11 @@ class CachedObjectDirectory extends FileObjectDatabase {
 		wrapped.selectObjectRepresentation(packer, otp, curs);
 	}
 
+	@Override
+	Collection<PackFile> getPacks() {
+		return wrapped.getPacks();
+	}
+
 	private static class UnpackedObjectId extends ObjectIdOwnerMap.Entry {
 		UnpackedObjectId(AnyObjectId id) {
 			super(id);

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java

@@ -292,6 +292,8 @@ abstract class FileObjectDatabase extends ObjectDatabase {
 
 	abstract FileObjectDatabase newCachedFileObjectDatabase();
 
+	abstract Collection<PackFile> getPacks();
+
 	static class AlternateHandle {
 		final FileObjectDatabase db;
 

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/InflatingBitSet.java

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import javaewah.EWAHCompressedBitmap;
+import javaewah.IntIterator;
+
+/**
+ * A wrapper around the EWAHCompressedBitmap optimized for the contains
+ * operation.
+ */
+final class InflatingBitSet {
+	private static final long[] EMPTY = new long[0];
+
+	private final EWAHCompressedBitmap bitmap;
+
+	private IntIterator iterator;
+
+	private long[] inflated;
+
+	private int nextPosition = -1;
+
+	private final int sizeInBits;
+
+	InflatingBitSet(EWAHCompressedBitmap bitmap) {
+		this(bitmap, EMPTY);
+	}
+
+	private InflatingBitSet(EWAHCompressedBitmap orBitmap, long[] inflated) {
+		this.bitmap = orBitmap;
+		this.inflated = inflated;
+		this.sizeInBits = bitmap.sizeInBits();
+	}
+
+	final boolean maybeContains(int position) {
+		if (get(position))
+			return true;
+		return nextPosition <= position && position < sizeInBits;
+	}
+
+	final boolean contains(int position) {
+		if (get(position))
+			return true;
+		if (position <= nextPosition || position >= sizeInBits)
+			return position == nextPosition;
+
+		if (iterator == null) {
+			iterator = bitmap.intIterator();
+			if (iterator.hasNext())
+				nextPosition = iterator.next();
+			else
+				return false;
+		} else if (!iterator.hasNext())
+			return false;
+
+		int positionBlock = block(position);
+		if (positionBlock >= inflated.length) {
+			long[] tmp = new long[block(sizeInBits) + 1];
+			System.arraycopy(inflated, 0, tmp, 0, inflated.length);
+			inflated = tmp;
+		}
+
+		int block = block(nextPosition);
+		long word = mask(nextPosition);
+		int end = Math.max(nextPosition, position) | 63;
+		while (iterator.hasNext()) {
+			nextPosition = iterator.next();
+			if (end < nextPosition)
+				break;
+
+			int b = block(nextPosition);
+			long m = mask(nextPosition);
+			if (block == b) {
+				word |= m;
+			} else {
+				inflated[block] = word;
+				block = b;
+				word = m;
+			}
+		}
+		inflated[block] = word;
+		return block == positionBlock && (word & mask(position)) != 0;
+	}
+
+	private final boolean get(int position) {
+		int b = block(position);
+		return b < inflated.length && (inflated[b] & mask(position)) != 0;
+	}
+
+	private static final int block(int position) {
+		return position >> 6;
+	}
+
+	private static final long mask(int position) {
+		return 1L << position;
+	}
+
+	private final boolean isEmpty() {
+		return sizeInBits == 0;
+	}
+
+	final InflatingBitSet or(EWAHCompressedBitmap other) {
+		if (other.sizeInBits() == 0)
+			return this;
+		return new InflatingBitSet(bitmap.or(other), inflated);
+	}
+
+	final InflatingBitSet andNot(EWAHCompressedBitmap other) {
+		if (isEmpty())
+			return this;
+		return new InflatingBitSet(bitmap.andNot(other));
+	}
+
+	final InflatingBitSet xor(EWAHCompressedBitmap other) {
+		if (isEmpty()) {
+			if (other.sizeInBits() == 0)
+				return this;
+			return new InflatingBitSet(other);
+		}
+		return new InflatingBitSet(bitmap.xor(other));
+	}
+
+	final EWAHCompressedBitmap getBitmap() {
+		return bitmap;
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java

@@ -241,6 +241,7 @@ public class ObjectDirectory extends FileObjectDatabase {
 	 *         containing objects referenced by commits further back in the
 	 *         history of the repository.
 	 */
+	@Override
 	public Collection<PackFile> getPacks() {
 		PackList list = packList.get();
 		if (list == NO_PACKS)

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackBitmapIndex.java

@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.MessageFormat;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.errors.CorruptObjectException;
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+
+/**
+ * Logical representation of the bitmap data stored in the pack index.
+ * {@link ObjectId}s are encoded as a single integer in the range [0,
+ * {@link #getObjectCount()}). Compressed bitmaps are available at certain
+ * {@code ObjectId}s, which represent all of the objects reachable from that
+ * {@code ObjectId} (include the {@code ObjectId} itself). The meaning of the
+ * positions in the bitmaps can be decoded using {@link #getObject(int)} and
+ * {@link #ofObjectType(EWAHCompressedBitmap, int)}. Furthermore,
+ * {@link #findPosition(AnyObjectId)} can be used to build other bitmaps that a
+ * compatible with the encoded bitmaps available from the index.
+ */
+public abstract class PackBitmapIndex {
+	/** Flag bit denoting the bitmap should be reused during index creation. */
+	public static final int FLAG_REUSE = 1;
+
+	/**
+	 * Read an existing pack bitmap index file from a buffered stream.
+	 * <p>
+	 * The format of the file will be automatically detected and a proper access
+	 * implementation for that format will be constructed and returned to the
+	 * caller. The file may or may not be held open by the returned instance.
+	 *
+	 * @param idxFile
+	 *            existing pack .bitmap to read.
+	 * @param packIndex
+	 *            the pack index for the corresponding pack file.
+	 * @param reverseIndex
+	 *            the pack reverse index for the corresponding pack file.
+	 * @return a copy of the index in-memory.
+	 * @throws IOException
+	 *             the stream cannot be read.
+	 * @throws CorruptObjectException
+	 *             the stream does not contain a valid pack bitmap index.
+	 */
+	public static PackBitmapIndex open(
+			File idxFile, PackIndex packIndex, PackReverseIndex reverseIndex)
+			throws IOException {
+		final FileInputStream fd = new FileInputStream(idxFile);
+		try {
+			return read(fd, packIndex, reverseIndex);
+		} catch (IOException ioe) {
+			final String path = idxFile.getAbsolutePath();
+			final IOException err;
+			err = new IOException(MessageFormat.format(
+					JGitText.get().unreadablePackIndex, path));
+			err.initCause(ioe);
+			throw err;
+		} finally {
+			try {
+				fd.close();
+			} catch (IOException err2) {
+				// ignore
+			}
+		}
+	}
+
+	/**
+	 * Read an existing pack bitmap index file from a buffered stream.
+	 * <p>
+	 * The format of the file will be automatically detected and a proper access
+	 * implementation for that format will be constructed and returned to the
+	 * caller. The file may or may not be held open by the returned instance.
+	 *
+	 * @param fd
+	 *            stream to read the bitmap index file from. The stream must be
+	 *            buffered as some small IOs are performed against the stream.
+	 *            The caller is responsible for closing the stream.
+	 * @param packIndex
+	 *            the pack index for the corresponding pack file.
+	 * @param reverseIndex
+	 *            the pack reverse index for the corresponding pack file.
+	 * @return a copy of the index in-memory.
+	 * @throws IOException
+	 *             the stream cannot be read.
+	 * @throws CorruptObjectException
+	 *             the stream does not contain a valid pack bitmap index.
+	 */
+	public static PackBitmapIndex read(
+			InputStream fd, PackIndex packIndex, PackReverseIndex reverseIndex)
+			throws IOException {
+		return new PackBitmapIndexV1(fd, packIndex, reverseIndex);
+	}
+
+	/** Footer checksum applied on the bottom of the pack file. */
+	byte[] packChecksum;
+
+	/**
+	 * Finds the position in the bitmap of the object.
+	 *
+	 * @param objectId
+	 *            the id for which the bitmap position will be found.
+	 * @return the bitmap id or -1 if the object was not found.
+	 */
+	public abstract int findPosition(AnyObjectId objectId);
+
+	/**
+	 * Get the object at the bitmap position.
+	 *
+	 * @param position
+	 *            the id for which the object will be found.
+	 * @return the ObjectId.
+	 * @throws IllegalArgumentException
+	 *             when the item is not found.
+	 */
+	public abstract ObjectId getObject(int position) throws IllegalArgumentException;
+
+	/**
+	 * Returns a bitmap containing positions for objects that have the given Git
+	 * type.
+	 *
+	 * @param bitmap
+	 *            the object bitmap.
+	 * @param type
+	 *            the Git type.
+	 * @return the object bitmap with only objects of the Git type.
+	 */
+	public abstract EWAHCompressedBitmap ofObjectType(
+			EWAHCompressedBitmap bitmap, int type);
+
+	/**
+	 * Returns the previously constructed bitmap for the object.
+	 *
+	 * @param objectId
+	 *            the id for which the bitmap will be found.
+	 * @return the bitmap or null if the object was not found.
+	 */
+	public abstract EWAHCompressedBitmap getBitmap(AnyObjectId objectId);
+
+	/**
+	 * Obtain the total number of objects described by this index.
+	 * {@code getObjectCount() - 1} is the largest bit that will be set in a
+	 * bitmap.
+	 *
+	 * @return number of objects in this index, and likewise in the associated
+	 *         pack that this index was generated from.
+	 */
+	public abstract int getObjectCount();
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackBitmapIndexBuilder.java

@@ -0,0 +1,349 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BitmapIndex.Bitmap;
+import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdOwnerMap;
+import org.eclipse.jgit.storage.file.BitmapIndexImpl.CompressedBitmap;
+import org.eclipse.jgit.storage.pack.ObjectToPack;
+import org.eclipse.jgit.util.BlockList;
+
+/**
+ * Helper for constructing {@link PackBitmapIndex}es.
+ */
+public class PackBitmapIndexBuilder extends BasePackBitmapIndex {
+	private static final int MAX_XOR_OFFSET_SEARCH = 10;
+
+	private final EWAHCompressedBitmap commits;
+	private final EWAHCompressedBitmap trees;
+	private final EWAHCompressedBitmap blobs;
+	private final EWAHCompressedBitmap tags;
+	private final ObjectToPack[] byOffset;
+	private final BlockList<StoredBitmap>
+			byAddOrder = new BlockList<StoredBitmap>();
+	private final ObjectIdOwnerMap<PositionEntry>
+			positionEntries = new ObjectIdOwnerMap<PositionEntry>();
+
+	/**
+	 * Creates a PackBitmapIndex used for building the contents of an index
+	 * file.
+	 *
+	 * @param byName
+	 *            objects sorted by name.
+	 */
+	public PackBitmapIndexBuilder(List<ObjectToPack> byName) {
+		super(new ObjectIdOwnerMap<StoredBitmap>());
+		byOffset = sortByOffset(byName);
+
+		int sizeInWords = Math.max(byOffset.length / 64, 4);
+		commits = new EWAHCompressedBitmap(sizeInWords);
+		trees = new EWAHCompressedBitmap(sizeInWords);
+		blobs = new EWAHCompressedBitmap(sizeInWords);
+		tags = new EWAHCompressedBitmap(sizeInWords);
+		for (int i = 0; i < byOffset.length; i++) {
+			int type = byOffset[i].getType();
+			switch (type) {
+			case Constants.OBJ_COMMIT:
+				commits.set(i);
+				break;
+			case Constants.OBJ_TREE:
+				trees.set(i);
+				break;
+			case Constants.OBJ_BLOB:
+				blobs.set(i);
+				break;
+			case Constants.OBJ_TAG:
+				tags.set(i);
+				break;
+			default:
+				throw new IllegalArgumentException(MessageFormat.format(
+						JGitText.get().badObjectType, String.valueOf(type)));
+			}
+		}
+	}
+
+	private ObjectToPack[] sortByOffset(List<ObjectToPack> entries) {
+		ObjectToPack[] result = new ObjectToPack[entries.size()];
+		for (int i = 0; i < result.length; i++) {
+			result[i] = entries.get(i);
+			positionEntries.add(new PositionEntry(result[i], i));
+		}
+		Arrays.sort(result, new Comparator<ObjectToPack>() {
+			public int compare(ObjectToPack a, ObjectToPack b) {
+				return Long.signum(a.getOffset() - b.getOffset());
+			}
+		});
+		for (int i = 0; i < result.length; i++)
+			positionEntries.get(result[i]).offsetPosition = i;
+		return result;
+	}
+
+	/**
+	 * Stores the bitmap for the objectId.
+	 *
+	 * @param objectId
+	 *            the object id key for the bitmap.
+	 * @param bitmap
+	 *            the bitmap
+	 * @param flags
+	 *            the flags to be stored with the bitmap
+	 */
+	public void addBitmap(AnyObjectId objectId, Bitmap bitmap, int flags) {
+		if (bitmap instanceof BitmapBuilder)
+			bitmap = ((BitmapBuilder) bitmap).build();
+
+		EWAHCompressedBitmap compressed;
+		if (bitmap instanceof CompressedBitmap)
+			compressed = ((CompressedBitmap) bitmap).getEwahCompressedBitmap();
+		else
+			throw new IllegalArgumentException(bitmap.getClass().toString());
+
+		addBitmap(objectId, compressed, flags);
+	}
+
+	/**
+	 * Stores the bitmap for the objectId.
+	 *
+	 * @param objectId
+	 *            the object id key for the bitmap.
+	 * @param bitmap
+	 *            the bitmap
+	 * @param flags
+	 *            the flags to be stored with the bitmap
+	 */
+	public void addBitmap(
+			AnyObjectId objectId, EWAHCompressedBitmap bitmap, int flags) {
+		StoredBitmap result = new StoredBitmap(objectId, bitmap, null, flags);
+		getBitmaps().add(result);
+		byAddOrder.add(result);
+	}
+
+	@Override
+	public EWAHCompressedBitmap ofObjectType(
+			EWAHCompressedBitmap bitmap, int type) {
+		switch (type) {
+		case Constants.OBJ_BLOB:
+			return getBlobs().and(bitmap);
+		case Constants.OBJ_TREE:
+			return getTrees().and(bitmap);
+		case Constants.OBJ_COMMIT:
+			return getCommits().and(bitmap);
+		case Constants.OBJ_TAG:
+			return getTags().and(bitmap);
+		}
+		throw new IllegalArgumentException();
+	}
+
+	@Override
+	public int findPosition(AnyObjectId objectId) {
+		PositionEntry entry = positionEntries.get(objectId);
+		if (entry == null)
+			return -1;
+		return entry.offsetPosition;
+	}
+
+	@Override
+	public ObjectId getObject(int position) throws IllegalArgumentException {
+		ObjectId objectId = byOffset[position];
+		if (objectId == null)
+			throw new IllegalArgumentException();
+		return objectId;
+	}
+
+	/** @return the commit object bitmap. */
+	public EWAHCompressedBitmap getCommits() {
+		return commits;
+	}
+
+	/** @return the tree object bitmap. */
+	public EWAHCompressedBitmap getTrees() {
+		return trees;
+	}
+
+	/** @return the blob object bitmap. */
+	public EWAHCompressedBitmap getBlobs() {
+		return blobs;
+	}
+
+	/** @return the tag object bitmap. */
+	public EWAHCompressedBitmap getTags() {
+		return tags;
+	}
+
+	/** @return the index storage options. */
+	public int getOptions() {
+		return PackBitmapIndexV1.OPT_FULL;
+	}
+
+	/** @return the number of bitmaps. */
+	public int getBitmapCount() {
+		return getBitmaps().size();
+	}
+
+	/** Removes all the bitmaps entries added. */
+	public void clearBitmaps() {
+		byAddOrder.clear();
+		getBitmaps().clear();
+	}
+
+	@Override
+	public int getObjectCount() {
+		return byOffset.length;
+	}
+
+	/** @return an iterator over the xor compressed entries. */
+	public Iterable<StoredEntry> getCompressedBitmaps() {
+		// Add order is from oldest to newest. The reverse add order is the
+		// output order.
+		return new Iterable<StoredEntry>() {
+			public Iterator<StoredEntry> iterator() {
+				return new Iterator<StoredEntry>() {
+					private int index = byAddOrder.size() - 1;
+
+					public boolean hasNext() {
+						return index >= 0;
+					}
+
+					public StoredEntry next() {
+						if (!hasNext())
+							throw new NoSuchElementException();
+						StoredBitmap item = byAddOrder.get(index);
+						int bestXorOffset = 0;
+						EWAHCompressedBitmap bestBitmap = item.getBitmap();
+
+						// Attempt to compress the bitmap with an XOR of the
+						// previously written entries.
+						for (int i = 1; i <= MAX_XOR_OFFSET_SEARCH; i++) {
+							int curr = i + index;
+							if (curr >= byAddOrder.size())
+								break;
+
+							StoredBitmap other = byAddOrder.get(curr);
+							EWAHCompressedBitmap bitmap = other.getBitmap()
+									.xor(item.getBitmap());
+
+							if (bitmap.sizeInBytes()
+									< bestBitmap.sizeInBytes()) {
+								bestBitmap = bitmap;
+								bestXorOffset = i;
+							}
+						}
+						index--;
+
+						PositionEntry entry = positionEntries.get(item);
+						if (entry == null)
+							throw new IllegalStateException();
+						return new StoredEntry(entry.namePosition, bestBitmap,
+								bestXorOffset, item.getFlags());
+					}
+
+					public void remove() {
+						throw new UnsupportedOperationException();
+					}
+				};
+			}
+		};
+	}
+
+	/** Data object for the on disk representation of a bitmap entry. */
+	public static final class StoredEntry {
+		private final long objectId;
+		private final EWAHCompressedBitmap bitmap;
+		private final int xorOffset;
+		private final int flags;
+
+		private StoredEntry(long objectId, EWAHCompressedBitmap bitmap,
+				int xorOffset, int flags) {
+			this.objectId = objectId;
+			this.bitmap = bitmap;
+			this.xorOffset = xorOffset;
+			this.flags = flags;
+		}
+
+		/** @return the bitmap */
+		public EWAHCompressedBitmap getBitmap() {
+			return bitmap;
+		}
+
+		/** @return the xorOffset */
+		public int getXorOffset() {
+			return xorOffset;
+		}
+
+		/** @return the flags */
+		public int getFlags() {
+			return flags;
+		}
+
+		/** @return the ObjectId */
+		public long getObjectId() {
+			return objectId;
+		}
+	}
+
+	private static final class PositionEntry extends ObjectIdOwnerMap.Entry {
+		private final int namePosition;
+
+		private int offsetPosition;
+
+		private PositionEntry(AnyObjectId objectId, int namePosition) {
+			super(objectId);
+			this.namePosition = namePosition;
+		}
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackBitmapIndexV1.java

@@ -0,0 +1,244 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.MessageFormat;
+import java.util.Arrays;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdOwnerMap;
+import org.eclipse.jgit.util.IO;
+import org.eclipse.jgit.util.NB;
+
+/**
+ * Support for the pack bitmap index v1 format, which contains experimental
+ * support for bitmaps.
+ *
+ * @see PackBitmapIndex
+ */
+class PackBitmapIndexV1 extends BasePackBitmapIndex {
+	static final byte[] MAGIC = { 'B', 'I', 'T', 'M' };
+	static final int OPT_FULL = 1;
+
+	private static final int MAX_XOR_OFFSET = 126;
+
+	private final PackIndex packIndex;
+	private final PackReverseIndex reverseIndex;
+	private final EWAHCompressedBitmap commits;
+	private final EWAHCompressedBitmap trees;
+	private final EWAHCompressedBitmap blobs;
+	private final EWAHCompressedBitmap tags;
+
+	private final ObjectIdOwnerMap<StoredBitmap> bitmaps;
+
+	PackBitmapIndexV1(final InputStream fd, PackIndex packIndex,
+			PackReverseIndex reverseIndex) throws IOException {
+		super(new ObjectIdOwnerMap<StoredBitmap>());
+		this.packIndex = packIndex;
+		this.reverseIndex = reverseIndex;
+		this.bitmaps = getBitmaps();
+
+		final byte[] scratch = new byte[32];
+		IO.readFully(fd, scratch, 0, scratch.length);
+
+		// Check the magic bytes
+		for (int i = 0; i < MAGIC.length; i++) {
+			if (scratch[i] != MAGIC[i]) {
+				byte[] actual = new byte[MAGIC.length];
+				System.arraycopy(scratch, 0, actual, 0, MAGIC.length);
+				throw new IOException(MessageFormat.format(
+						JGitText.get().expectedGot, Arrays.toString(MAGIC),
+						Arrays.toString(actual)));
+			}
+		}
+
+		// Read the version (2 bytes)
+		final int version = NB.decodeUInt16(scratch, 4);
+		if (version != 1)
+			throw new IOException(MessageFormat.format(
+					JGitText.get().unsupportedPackIndexVersion,
+					Integer.valueOf(version)));
+
+		// Read the options (2 bytes)
+		final int opts = NB.decodeUInt16(scratch, 6);
+		switch (opts) {
+		case OPT_FULL:
+			// Bitmaps are self contained within this file.
+			break;
+		default:
+			throw new IOException(MessageFormat.format(
+					JGitText.get().expectedGot, Integer.valueOf(OPT_FULL),
+					Integer.valueOf(opts)));
+		}
+
+		// Read the number of entries (1 int32)
+		long numEntries = NB.decodeUInt32(scratch, 8);
+		if (numEntries > Integer.MAX_VALUE)
+			throw new IOException(JGitText.get().indexFileIsTooLargeForJgit);
+
+		// Checksum applied on the bottom of the corresponding pack file.
+		this.packChecksum = new byte[20];
+		System.arraycopy(scratch, 12, packChecksum, 0, packChecksum.length);
+
+		// Read the bitmaps for the Git types
+		SimpleDataInput dataInput = new SimpleDataInput(fd);
+		this.commits = readBitmap(dataInput);
+		this.trees = readBitmap(dataInput);
+		this.blobs = readBitmap(dataInput);
+		this.tags = readBitmap(dataInput);
+
+		// An entry is object id, xor offset, flag byte, and a length encoded
+		// bitmap. The object id is an int32 of the nth position sorted by name.
+		// The xor offset is a single byte offset back in the list of entries.
+		StoredBitmap[] recentBitmaps = new StoredBitmap[MAX_XOR_OFFSET];
+		for (int i = 0; i < (int) numEntries; i++) {
+			IO.readFully(fd, scratch, 0, 6);
+			int nthObjectId = NB.decodeInt32(scratch, 0);
+			int xorOffset = scratch[4];
+			int flags = scratch[5];
+			EWAHCompressedBitmap bitmap = readBitmap(dataInput);
+
+			if (nthObjectId < 0)
+				throw new IOException(MessageFormat.format(
+						JGitText.get().invalidId, String.valueOf(nthObjectId)));
+			if (xorOffset < 0)
+				throw new IOException(MessageFormat.format(
+						JGitText.get().invalidId, String.valueOf(xorOffset)));
+			if (xorOffset > MAX_XOR_OFFSET)
+				throw new IOException(MessageFormat.format(
+						JGitText.get().expectedLessThanGot,
+						String.valueOf(MAX_XOR_OFFSET),
+						String.valueOf(xorOffset)));
+			if (xorOffset > i)
+				throw new IOException(MessageFormat.format(
+						JGitText.get().expectedLessThanGot, String.valueOf(i),
+						String.valueOf(xorOffset)));
+
+			ObjectId objectId = packIndex.getObjectId(nthObjectId);
+			StoredBitmap xorBitmap = null;
+			if (xorOffset > 0) {
+				int index = (i - xorOffset);
+				xorBitmap = recentBitmaps[index % recentBitmaps.length];
+				if (xorBitmap == null)
+					throw new IOException(MessageFormat.format(
+							JGitText.get().invalidId,
+							String.valueOf(xorOffset)));
+			}
+
+			StoredBitmap sb = new StoredBitmap(
+					objectId, bitmap, xorBitmap, flags);
+			bitmaps.add(sb);
+			recentBitmaps[i % recentBitmaps.length] = sb;
+		}
+	}
+
+	@Override
+	public int findPosition(AnyObjectId objectId) {
+		long offset = packIndex.findOffset(objectId);
+		if (offset == -1)
+			return -1;
+		return reverseIndex.findPostion(offset);
+	}
+
+	@Override
+	public ObjectId getObject(int position) throws IllegalArgumentException {
+		ObjectId objectId = reverseIndex.findObjectByPosition(position);
+		if (objectId == null)
+			throw new IllegalArgumentException();
+		return objectId;
+	}
+
+	@Override
+	public int getObjectCount() {
+		return (int) packIndex.getObjectCount();
+	}
+
+	@Override
+	public EWAHCompressedBitmap ofObjectType(
+			EWAHCompressedBitmap bitmap, int type) {
+		switch (type) {
+		case Constants.OBJ_BLOB:
+			return blobs.and(bitmap);
+		case Constants.OBJ_TREE:
+			return trees.and(bitmap);
+		case Constants.OBJ_COMMIT:
+			return commits.and(bitmap);
+		case Constants.OBJ_TAG:
+			return tags.and(bitmap);
+		}
+		throw new IllegalArgumentException();
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		// TODO(cranger): compare the pack checksum?
+		if (o instanceof PackBitmapIndexV1)
+			return getPackIndex() == ((PackBitmapIndexV1) o).getPackIndex();
+		return false;
+	}
+
+	@Override
+	public int hashCode() {
+		return getPackIndex().hashCode();
+	}
+
+	PackIndex getPackIndex() {
+		return packIndex;
+	}
+
+	private static EWAHCompressedBitmap readBitmap(DataInput dataInput)
+			throws IOException {
+		EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap();
+		bitmap.deserialize(dataInput);
+		return bitmap;
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackBitmapIndexWriterV1.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.io.BufferedOutputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.security.DigestOutputStream;
+import java.text.MessageFormat;
+
+import javaewah.EWAHCompressedBitmap;
+
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.storage.file.PackBitmapIndexBuilder.StoredEntry;
+import org.eclipse.jgit.util.io.SafeBufferedOutputStream;
+
+/**
+ * Creates the version 1 pack bitmap index files.
+ *
+ * @see PackBitmapIndexV1
+ */
+class PackBitmapIndexWriterV1 {
+	private final DigestOutputStream out;
+	private final DataOutput dataOutput;
+
+	PackBitmapIndexWriterV1(final OutputStream dst) {
+		out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst
+				: new SafeBufferedOutputStream(dst),
+				Constants.newMessageDigest());
+		dataOutput = new SimpleDataOutput(out);
+	}
+
+	/**
+	 * Write all object entries to the index stream.
+	 * <p>
+	 * After writing the stream passed to the factory is flushed but remains
+	 * open. Callers are always responsible for closing the output stream.
+	 *
+	 * @param bitmaps
+	 *            the index data for the bitmaps
+	 * @param packDataChecksum
+	 *            checksum signature of the entire pack data content. This is
+	 *            traditionally the last 20 bytes of the pack file's own stream.
+	 * @throws IOException
+	 *             an error occurred while writing to the output stream, or this
+	 *             index format cannot store the object data supplied.
+	 */
+	public void write(PackBitmapIndexBuilder bitmaps, byte[] packDataChecksum)
+			throws IOException {
+		if (bitmaps == null || packDataChecksum.length != 20)
+			throw new IllegalStateException();
+
+		writeHeader(bitmaps.getOptions(), bitmaps.getBitmapCount(),
+				packDataChecksum);
+		writeBody(bitmaps);
+		writeFooter();
+
+		out.flush();
+	}
+
+	private void writeHeader(
+			int options, int bitmapCount, byte[] packDataChecksum)
+			throws IOException {
+		out.write(PackBitmapIndexV1.MAGIC);
+		dataOutput.writeShort(1);
+		dataOutput.writeShort(options);
+		dataOutput.writeInt(bitmapCount);
+		out.write(packDataChecksum);
+	}
+
+	private void writeBody(PackBitmapIndexBuilder bitmaps) throws IOException {
+		writeBitmap(bitmaps.getCommits());
+		writeBitmap(bitmaps.getTrees());
+		writeBitmap(bitmaps.getBlobs());
+		writeBitmap(bitmaps.getTags());
+		writeBitmaps(bitmaps);
+	}
+
+	private void writeBitmap(EWAHCompressedBitmap bitmap) throws IOException {
+		bitmap.serialize(dataOutput);
+	}
+
+	private void writeBitmaps(PackBitmapIndexBuilder bitmaps)
+			throws IOException {
+		int bitmapCount = 0;
+		for (StoredEntry entry : bitmaps.getCompressedBitmaps()) {
+			writeBitmapEntry(entry);
+			bitmapCount++;
+		}
+
+		int expectedBitmapCount = bitmaps.getBitmapCount();
+		if (expectedBitmapCount != bitmapCount)
+			throw new IOException(MessageFormat.format(
+					JGitText.get().expectedGot,
+					String.valueOf(expectedBitmapCount),
+					String.valueOf(bitmapCount)));
+	}
+
+	private void writeBitmapEntry(StoredEntry entry) throws IOException {
+		// Write object, xor offset, and bitmap
+		dataOutput.writeInt((int) entry.getObjectId());
+		out.write(entry.getXorOffset());
+		out.write(entry.getFlags());
+		writeBitmap(entry.getBitmap());
+	}
+
+	private void writeFooter() throws IOException {
+		out.on(false);
+		out.write(out.getMessageDigest().digest());
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java

@@ -45,6 +45,7 @@
 
 package org.eclipse.jgit.storage.file;
 
+import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
 import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
 
 import java.io.EOFException;
@@ -97,6 +98,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 	private final File packFile;
 
+	private final int extensions;
+
 	private File keepFile;
 
 	private volatile String packName;
@@ -124,6 +127,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 	private PackReverseIndex reverseIdx;
 
+	private PackBitmapIndex bitmapIdx;
+
 	/**
 	 * Objects we have tried to read, and discovered to be corrupt.
 	 * <p>
@@ -144,6 +149,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 	public PackFile(final File packFile, int extensions) {
 		this.packFile = packFile;
 		this.packLastModified = (int) (packFile.lastModified() >> 10);
+		this.extensions = extensions;
 
 		// Multiply by 31 here so we can more directly combine with another
 		// value in WindowCache.hash(), without doing the multiply there.
@@ -1050,6 +1056,22 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 		return getReverseIdx().findNextOffset(startOffset, maxOffset);
 	}
 
+	synchronized PackBitmapIndex getBitmapIndex() throws IOException {
+		if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
+			final PackBitmapIndex idx = PackBitmapIndex.open(
+					extFile(BITMAP_INDEX), idx(), getReverseIdx());
+
+			if (packChecksum == null)
+				packChecksum = idx.packChecksum;
+			else if (!Arrays.equals(packChecksum, idx.packChecksum))
+				throw new PackMismatchException(
+						JGitText.get().packChecksumMismatch);
+
+			bitmapIdx = idx;
+		}
+		return bitmapIdx;
+	}
+
 	private synchronized PackReverseIndex getReverseIdx() throws IOException {
 		if (reverseIdx == null)
 			reverseIdx = new PackReverseIndex(idx());
@@ -1087,4 +1109,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 		String b = (dot < 0) ? p : p.substring(0, dot);
 		return new File(packFile.getParentFile(), b + '.' + ext.getExtension());
 	}
+
+	private boolean hasExt(PackExt ext) {
+		return (extensions & ext.getBit()) != 0;
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackReverseIndex.java

@@ -193,4 +193,25 @@ public class PackReverseIndex {
 			return offsets64[i64 + 1];
 		}
 	}
+
+	int findPostion(long offset) {
+		if (offset <= Integer.MAX_VALUE) {
+			final int i32 = Arrays.binarySearch(offsets32, (int) offset);
+			if (i32 < 0)
+				return -1;
+			return i32;
+		} else {
+			final int i64 = Arrays.binarySearch(offsets64, offset);
+			if (i64 < 0)
+				return -1;
+			return nth32.length + i64;
+		}
+	}
+
+	ObjectId findObjectByPosition(int nthPosition) {
+		if (nthPosition < nth32.length)
+			return index.getObjectId(nth32[nthPosition]);
+		final int i64 = nthPosition - nth32.length;
+		return index.getObjectId(nth64[i64]);
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/SimpleDataInput.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.eclipse.jgit.util.IO;
+import org.eclipse.jgit.util.NB;
+
+/**
+ * An implementation of DataInput that only handles readInt() and readLong()
+ * using the Git conversion utilities for network byte order handling. This is
+ * needed to read {@link javaewah.EWAHCompressedBitmap}s.
+ */
+class SimpleDataInput implements DataInput {
+	private final InputStream fd;
+
+	private final byte[] buf = new byte[8];
+
+	SimpleDataInput(InputStream fd) {
+		this.fd = fd;
+	}
+
+	public int readInt() throws IOException {
+		readFully(buf, 0, 4);
+		return NB.decodeInt32(buf, 0);
+	}
+
+	public long readLong() throws IOException {
+		readFully(buf, 0, 8);
+		return NB.decodeInt64(buf, 0);
+	}
+
+	public long readUnsignedInt() throws IOException {
+		readFully(buf, 0, 4);
+		return NB.decodeUInt32(buf, 0);
+	}
+
+	public void readFully(byte[] b) throws IOException {
+		readFully(b, 0, b.length);
+	}
+
+	public void readFully(byte[] b, int off, int len) throws IOException {
+		IO.readFully(fd, b, off, len);
+	}
+
+	public int skipBytes(int n) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public boolean readBoolean() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public byte readByte() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public int readUnsignedByte() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public short readShort() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public int readUnsignedShort() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public char readChar() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public float readFloat() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public double readDouble() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public String readLine() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public String readUTF() throws IOException {
+		throw new UnsupportedOperationException();
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/SimpleDataOutput.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2012, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.eclipse.jgit.util.NB;
+
+/**
+ * An implementation of {@link DataOutput} that only handles
+ * {@link #writeInt(int)} and {@link #writeLong(long)} using the Git conversion
+ * utilities for network byte order handling. This is needed to write
+ * {@link javaewah.EWAHCompressedBitmap}s.
+ */
+class SimpleDataOutput implements DataOutput {
+	private final OutputStream fd;
+
+	private final byte[] buf = new byte[8];
+
+	SimpleDataOutput(OutputStream fd) {
+		this.fd = fd;
+	}
+
+	public void writeShort(int v) throws IOException {
+		NB.encodeInt16(buf, 0, v);
+		fd.write(buf, 0, 2);
+	}
+
+	public void writeInt(int v) throws IOException {
+		NB.encodeInt32(buf, 0, v);
+		fd.write(buf, 0, 4);
+	}
+
+	public void writeLong(long v) throws IOException {
+		NB.encodeInt64(buf, 0, v);
+		fd.write(buf, 0, 8);
+	}
+
+	public void write(int b) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void write(byte[] b) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void write(byte[] b, int off, int len) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeBoolean(boolean v) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeByte(int v) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeChar(int v) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeFloat(float v) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeDouble(double v) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeBytes(String s) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeChars(String s) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+
+	public void writeUTF(String s) throws IOException {
+		throw new UnsupportedOperationException();
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java

@@ -62,6 +62,7 @@ import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.AbbreviatedObjectId;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.BitmapIndex;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.ObjectId;
@@ -102,6 +103,16 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs {
 		return new WindowCursor(db);
 	}
 
+	@Override
+	public BitmapIndex getBitmapIndex() throws IOException {
+		for (PackFile pack : db.getPacks()) {
+			PackBitmapIndex index = pack.getBitmapIndex();
+			if (index != null)
+				return new BitmapIndexImpl(index);
+		}
+		return null;
+	}
+
 	@Override
 	public Collection<ObjectId> resolve(AbbreviatedObjectId id)
 			throws IOException {

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackExt.java

@@ -53,6 +53,9 @@ public class PackExt {
 	/** A pack index file extension. */
 	public static final PackExt INDEX = newPackExt("idx"); //$NON-NLS-1$
 
+	/** A pack bitmap index file extension. */
+	public static final PackExt BITMAP_INDEX = newPackExt("bitmap"); //$NON-NLS-1$
+
 	/** @return all of the PackExt values. */
 	public static PackExt[] values() {
 		return VALUES;

org.eclipse.jgit/src/org/eclipse/jgit/util/NB.java

@@ -102,6 +102,39 @@ public final class NB {
 		return (r << 8) | (intbuf[offset + 3] & 0xff);
 	}
 
+	/**
+	 * Convert sequence of 8 bytes (network byte order) into signed value.
+	 *
+	 * @param intbuf
+	 *            buffer to acquire the 8 bytes of data from.
+	 * @param offset
+	 *            position within the buffer to begin reading from. This
+	 *            position and the next 7 bytes after it (for a total of 8
+	 *            bytes) will be read.
+	 * @return signed integer value that matches the 64 bits read.
+	 */
+	public static long decodeInt64(final byte[] intbuf, final int offset) {
+		long r = intbuf[offset] << 8;
+
+		r |= intbuf[offset + 1] & 0xff;
+		r <<= 8;
+
+		r |= intbuf[offset + 2] & 0xff;
+		r <<= 8;
+
+		r |= intbuf[offset + 3] & 0xff;
+		r <<= 8;
+
+		r |= intbuf[offset + 4] & 0xff;
+		r <<= 8;
+
+		r |= intbuf[offset + 5] & 0xff;
+		r <<= 8;
+
+		r |= intbuf[offset + 6] & 0xff;
+		return (r << 8) | (intbuf[offset + 7] & 0xff);
+	}
+
 	/**
 	 * Convert sequence of 4 bytes (network byte order) into unsigned value.
 	 *
@@ -186,7 +219,7 @@ public final class NB {
 	 * Write a 64 bit integer as a sequence of 8 bytes (network byte order).
 	 *
 	 * @param intbuf
-	 *            buffer to write the 48bytes of data into.
+	 *            buffer to write the 8 bytes of data into.
 	 * @param offset
 	 *            position within the buffer to begin writing to. This position
 	 *            and the next 7 bytes after it (for a total of 8 bytes) will be

pom.xml

@@ -177,6 +177,7 @@
 
     <jgit-last-release-version>2.3.1.201302201838-r</jgit-last-release-version>
     <jsch-version>0.1.46</jsch-version>
+    <javaewah-version>0.5.6</javaewah-version>
     <junit-version>4.5</junit-version>
     <!-- TODO: update Maven dependency for args4j to 2.0.21 as soon as available on Maven Central -->
     <args4j-version>2.0.12</args4j-version>
@@ -405,6 +406,12 @@
         <version>${jsch-version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.googlecode.javaewah</groupId>
+        <artifactId>JavaEWAH</artifactId>
+        <version>${javaewah-version}</version>
+      </dependency>
+
       <dependency>
         <groupId>args4j</groupId>
         <artifactId>args4j</artifactId>