From 3b325917a5c928caadd88a0ec718b1632f088fd5 Mon Sep 17 00:00:00 2001 From: Colby Ranger Date: Tue, 28 Aug 2012 09:08:20 -0700 Subject: Added read/write support for pack bitmap index. A pack bitmap index is an additional index of compressed bitmaps of the object graph. Furthermore, a logical API of the index functionality is included, as it is expected to be used by the PackWriter. Compressed bitmaps are created using the javaewah library, which is a word-aligned compressed variant of the Java bitset class based on run-length encoding. The library only works with positive integer values. Thus, the maximum number of ObjectIds in a pack file that this index can currently support is limited to Integer.MAX_VALUE. Every ObjectId is given an integer mapping. The integer is the position of the ObjectId in the complete ObjectId list, sorted by offset, for the pack file. That integer is what the bitmaps use to reference the ObjectId. Currently, the new index format can only be used with pack files that contain a complete closure of the object graph e.g. the result of a garbage collection. The index file includes four bitmaps for the Git object types i.e. commits, trees, blobs, and tags. In addition, a collection of bitmaps keyed by an ObjectId is also included. The bitmap for each entry in the collection represents the full closure of ObjectIds reachable from the keyed ObjectId (including the keyed ObjectId itself). The bitmaps are further compressed by XORing the current bitmaps against prior bitmaps in the index, and selecting the smallest representation. The XOR'd bitmap and offset from the current entry to the position of the bitmap to XOR against is the actual representation of the entry in the index file. Each entry contains one byte, which is currently used to note whether the bitmap should be blindly reused. Change-Id: Id328724bf6b4c8366a088233098c18643edcf40f --- org.eclipse.jgit.test/META-INF/MANIFEST.MF | 3 +- .../jgit/storage/file/InflatingBitSetTest.java | 106 +++++++++++++++++++++ .../jgit/storage/file/StoredBitmapTest.java | 94 ++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/InflatingBitSetTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/StoredBitmapTest.java (limited to 'org.eclipse.jgit.test') diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index 4de17bdcb9..dbdf0c9cdf 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -7,7 +7,8 @@ Bundle-Localization: plugin Bundle-Vendor: %provider_name Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: J2SE-1.5 -Import-Package: org.eclipse.jgit.api;version="[2.4.0,2.5.0)", +Import-Package: javaewah;version="0.5.6", + org.eclipse.jgit.api;version="[2.4.0,2.5.0)", org.eclipse.jgit.api.errors;version="[2.4.0,2.5.0)", org.eclipse.jgit.awtui;version="[2.4.0,2.5.0)", org.eclipse.jgit.blame;version="[2.4.0,2.5.0)", diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/InflatingBitSetTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/InflatingBitSetTest.java new file mode 100644 index 0000000000..811096e3db --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/InflatingBitSetTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2012, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.storage.file; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import javaewah.EWAHCompressedBitmap; + +import org.junit.Test; + +public class InflatingBitSetTest { + + @Test + public void testMaybeContains() { + EWAHCompressedBitmap ecb = new EWAHCompressedBitmap(); + ecb.set(63); + ecb.set(64); + ecb.set(128); + + InflatingBitSet ibs = new InflatingBitSet(ecb); + assertTrue(ibs.maybeContains(0)); + assertFalse(ibs.contains(0)); // Advance + assertFalse(ibs.maybeContains(0)); + assertTrue(ibs.maybeContains(63)); + assertTrue(ibs.maybeContains(64)); + assertTrue(ibs.maybeContains(65)); + assertFalse(ibs.maybeContains(129)); + } + + @Test + public void testContainsMany() { + EWAHCompressedBitmap ecb = new EWAHCompressedBitmap(); + ecb.set(64); + ecb.set(65); + ecb.set(1024); + + InflatingBitSet ibs = new InflatingBitSet(ecb); + assertFalse(ibs.contains(0)); + assertTrue(ibs.contains(64)); + assertTrue(ibs.contains(65)); + assertFalse(ibs.contains(66)); + assertTrue(ibs.contains(1024)); + assertFalse(ibs.contains(1025)); + } + + @Test + public void testContainsOne() { + EWAHCompressedBitmap ecb = new EWAHCompressedBitmap(); + ecb.set(64); + + InflatingBitSet ibs = new InflatingBitSet(ecb); + assertTrue(ibs.contains(64)); + assertTrue(ibs.contains(64)); + assertFalse(ibs.contains(65)); + assertFalse(ibs.contains(63)); + } + + @Test + public void testContainsEmpty() { + InflatingBitSet ibs = new InflatingBitSet(new EWAHCompressedBitmap()); + assertFalse(ibs.maybeContains(0)); + assertFalse(ibs.contains(0)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/StoredBitmapTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/StoredBitmapTest.java new file mode 100644 index 0000000000..9170e6e3c4 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/StoredBitmapTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2012, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.storage.file; + +import static org.junit.Assert.*; + +import javaewah.EWAHCompressedBitmap; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.storage.file.BasePackBitmapIndex.StoredBitmap; +import org.junit.Test; + +public class StoredBitmapTest { + + @Test + public void testGetBitmapWithoutXor() { + EWAHCompressedBitmap b = bitmapOf(100); + StoredBitmap sb = newStoredBitmap(bitmapOf(100)); + assertEquals(b, sb.getBitmap()); + } + + @Test + public void testGetBitmapWithOneXor() { + StoredBitmap sb = newStoredBitmap(bitmapOf(100), bitmapOf(100, 101)); + assertEquals(bitmapOf(101), sb.getBitmap()); + } + + @Test + public void testGetBitmapWithThreeXor() { + StoredBitmap sb = newStoredBitmap( + bitmapOf(100), + bitmapOf(90, 101), + bitmapOf(100, 101), + bitmapOf(50)); + assertEquals(bitmapOf(50, 90), sb.getBitmap()); + assertEquals(bitmapOf(50, 90), sb.getBitmap()); + } + + private static final StoredBitmap newStoredBitmap( + EWAHCompressedBitmap... bitmaps) { + StoredBitmap sb = null; + for (EWAHCompressedBitmap bitmap : bitmaps) + sb = new StoredBitmap(ObjectId.zeroId(), bitmap, sb, 0); + return sb; + } + + private static final EWAHCompressedBitmap bitmapOf(int... bits) { + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + for (int bit : bits) + b.set(bit); + return b; + } +} -- cgit v1.2.3