]> source.dussan.org Git - jgit.git/commitdiff
Enable writing pack indexes with bitmaps in the GC. 42/7942/21
authorColby Ranger <cranger@google.com>
Wed, 26 Sep 2012 18:50:05 +0000 (11:50 -0700)
committerColby Ranger <cranger@google.com>
Tue, 5 Mar 2013 19:15:19 +0000 (11:15 -0800)
Update the dfs and file GC implementations to prepare and write
bitmaps on the packs that contain the full closure of the object
graph. Update the DfsPackDescription to include the index version.

Change-Id: I3f1421e9cd90fe93e7e2ef2b8179ae2f1ba819ed

org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsGarbageCollector.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsInserter.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackCompactor.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/GC.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

index 76fb521a62d8d7d45efdfa1f9daa972380450c0c..db7a195e1583d3c2ec2bc0c9ec0e0dd044256dc7 100644 (file)
@@ -45,6 +45,7 @@ package org.eclipse.jgit.storage.dfs;
 
 import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.GC;
 import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
+import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
 import static org.eclipse.jgit.storage.pack.PackExt.PACK;
 import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
 
@@ -329,10 +330,22 @@ public class DfsGarbageCollector {
                        CountingOutputStream cnt = new CountingOutputStream(out);
                        pw.writeIndex(cnt);
                        pack.setFileSize(INDEX, cnt.getCount());
+                       pack.setIndexVersion(pw.getIndexVersion());
                } finally {
                        out.close();
                }
 
+               if (pw.prepareBitmapIndex(pm)) {
+                       out = objdb.writeFile(pack, BITMAP_INDEX);
+                       try {
+                               CountingOutputStream cnt = new CountingOutputStream(out);
+                               pw.writeIndex(cnt);
+                               pack.setFileSize(BITMAP_INDEX, cnt.getCount());
+                       } finally {
+                               out.close();
+                       }
+               }
+
                final ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> packedObjs = pw
                                .getObjectSet();
                newPackObj.add(new PackWriter.ObjectIdSet() {
index 90bd444724a3cf980b4d9463d26bddf7f784a01a..0851db26977db74ebae1bb1c74ea08677cebe293 100644 (file)
@@ -240,6 +240,7 @@ public class DfsInserter extends ObjectInserter {
 
        PackIndex writePackIndex(DfsPackDescription pack, byte[] packHash,
                        List<PackedObjectInfo> list) throws IOException {
+               pack.setIndexVersion(INDEX_VERSION);
                pack.setObjectCount(list.size());
 
                // If there are less than 58,000 objects, the entire index fits in under
index c17c8633f93191f406680eb33bb1f7f0cd078882..904ba4efc03e116c119212d8a67375136996367d 100644 (file)
@@ -303,6 +303,7 @@ public class DfsPackCompactor {
                        CountingOutputStream cnt = new CountingOutputStream(out);
                        pw.writeIndex(cnt);
                        pack.setFileSize(INDEX, cnt.getCount());
+                       pack.setIndexVersion(pw.getIndexVersion());
                } finally {
                        out.close();
                }
index 7aa3b5093b16910392149464209345e756232cf0..2aa81d9a2573a54fe50a92da148d293292a65cbd 100644 (file)
@@ -81,6 +81,8 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 
        private int extensions;
 
+       private int indexVersion;
+
        /**
         * Initialize a description by pack name and repository.
         * <p>
@@ -259,6 +261,21 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
                return this;
        }
 
+       /** @return the version of the index file written. */
+       public int getIndexVersion() {
+               return indexVersion;
+       }
+
+       /**
+        * @param version
+        *            the version of the index file written.
+        * @return {@code this}
+        */
+       public DfsPackDescription setIndexVersion(int version) {
+               indexVersion = version;
+               return this;
+       }
+
        @Override
        public int hashCode() {
                return packName.hashCode();
index 45f382316a7f8b8034973fba37d703f6fc08725c..b93ce2bb36a32090a2c3c2d88637d5cf2c9ef893 100644 (file)
@@ -43,6 +43,9 @@
  */
 package org.eclipse.jgit.storage.file;
 
+import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
+import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -54,6 +57,7 @@ import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -63,6 +67,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.eclipse.jgit.dircache.DirCacheIterator;
 import org.eclipse.jgit.errors.CorruptObjectException;
@@ -83,6 +88,7 @@ import org.eclipse.jgit.lib.RefDatabase;
 import org.eclipse.jgit.revwalk.ObjectWalk;
 import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.pack.PackExt;
 import org.eclipse.jgit.storage.pack.PackWriter;
 import org.eclipse.jgit.storage.pack.PackWriter.ObjectIdSet;
 import org.eclipse.jgit.treewalk.TreeWalk;
@@ -192,8 +198,10 @@ public class GC {
 
                        if (!oldPack.shouldBeKept()) {
                                oldPack.close();
-                               FileUtils.delete(nameFor(oldName, ".pack"), deleteOptions); //$NON-NLS-1$
-                               FileUtils.delete(nameFor(oldName, ".idx"), deleteOptions); //$NON-NLS-1$
+                               for (PackExt ext : PackExt.values()) {
+                                       File f = nameFor(oldName, "." + ext.getExtension()); //$NON-NLS-1$
+                                       FileUtils.delete(f, deleteOptions);
+                               }
                        }
                }
                // close the complete object database. Thats my only chance to force
@@ -636,7 +644,22 @@ public class GC {
                        Set<? extends ObjectId> have, Set<ObjectId> tagTargets,
                        List<ObjectIdSet> excludeObjects) throws IOException {
                File tmpPack = null;
-               File tmpIdx = null;
+               Map<PackExt, File> tmpExts = new TreeMap<PackExt, File>(
+                               new Comparator<PackExt>() {
+                                       public int compare(PackExt o1, PackExt o2) {
+                                               // INDEX entries must be returned last, so the pack
+                                               // scanner does pick up the new pack until all the
+                                               // PackExt entries have been written.
+                                               if (o1 == o2)
+                                                       return 0;
+                                               if (o1 == PackExt.INDEX)
+                                                       return 1;
+                                               if (o2 == PackExt.INDEX)
+                                                       return -1;
+                                               return Integer.signum(o1.hashCode() - o2.hashCode());
+                                       }
+
+                               });
                PackWriter pw = new PackWriter(repo);
                try {
                        // prepare the PackWriter
@@ -655,9 +678,10 @@ public class GC {
                        String id = pw.computeName().getName();
                        File packdir = new File(repo.getObjectsDirectory(), "pack"); //$NON-NLS-1$
                        tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
-                       tmpIdx = new File(packdir, tmpPack.getName().substring(0,
-                                       tmpPack.getName().lastIndexOf('.'))
-                                       + ".idx_tmp"); //$NON-NLS-1$
+                       final String tmpBase = tmpPack.getName()
+                                       .substring(0, tmpPack.getName().lastIndexOf('.'));
+                       File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
+                       tmpExts.put(INDEX, tmpIdx);
 
                        if (!tmpIdx.createNewFile())
                                throw new IOException(MessageFormat.format(
@@ -687,38 +711,70 @@ public class GC {
                                idxChannel.close();
                        }
 
+                       if (pw.prepareBitmapIndex(pm)) {
+                               File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
+                               tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
+
+                               if (!tmpBitmapIdx.createNewFile())
+                                       throw new IOException(MessageFormat.format(
+                                                       JGitText.get().cannotCreateIndexfile,
+                                                       tmpBitmapIdx.getPath()));
+
+                               idxChannel = new FileOutputStream(tmpBitmapIdx).getChannel();
+                               idxStream = Channels.newOutputStream(idxChannel);
+                               try {
+                                       pw.writeBitmapIndex(idxStream);
+                               } finally {
+                                       idxChannel.force(true);
+                                       idxStream.close();
+                                       idxChannel.close();
+                               }
+                       }
+
                        // rename the temporary files to real files
                        File realPack = nameFor(id, ".pack"); //$NON-NLS-1$
                        tmpPack.setReadOnly();
-                       File realIdx = nameFor(id, ".idx"); //$NON-NLS-1$
-                       realIdx.setReadOnly();
                        boolean delete = true;
                        try {
                                if (!tmpPack.renameTo(realPack))
                                        return null;
                                delete = false;
-                               if (!tmpIdx.renameTo(realIdx)) {
-                                       File newIdx = new File(realIdx.getParentFile(),
-                                                       realIdx.getName() + ".new"); //$NON-NLS-1$
-                                       if (!tmpIdx.renameTo(newIdx))
-                                               newIdx = tmpIdx;
-                                       throw new IOException(MessageFormat.format(
-                                                       JGitText.get().panicCantRenameIndexFile, newIdx,
-                                                       realIdx));
+                               for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
+                                       File tmpExt = tmpEntry.getValue();
+                                       tmpExt.setReadOnly();
+
+                                       File realExt = nameFor(
+                                                       id, "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$
+                                       if (!tmpExt.renameTo(realExt)) {
+                                               File newExt = new File(realExt.getParentFile(),
+                                                               realExt.getName() + ".new"); //$NON-NLS-1$
+                                               if (!tmpExt.renameTo(newExt))
+                                                       newExt = tmpExt;
+                                               throw new IOException(MessageFormat.format(
+                                                               JGitText.get().panicCantRenameIndexFile, newExt,
+                                                               realExt));
+                                       }
                                }
+
                        } finally {
-                               if (delete && tmpPack.exists())
-                                       tmpPack.delete();
-                               if (delete && tmpIdx.exists())
-                                       tmpIdx.delete();
+                               if (delete) {
+                                       if (tmpPack.exists())
+                                               tmpPack.delete();
+                                       for (File tmpExt : tmpExts.values()) {
+                                               if (tmpExt.exists())
+                                                       tmpExt.delete();
+                                       }
+                               }
                        }
                        return repo.getObjectDatabase().openPack(realPack);
                } finally {
                        pw.release();
                        if (tmpPack != null && tmpPack.exists())
                                tmpPack.delete();
-                       if (tmpIdx != null && tmpIdx.exists())
-                               tmpIdx.delete();
+                       for (File tmpExt : tmpExts.values()) {
+                               if (tmpExt.exists())
+                                       tmpExt.delete();
+                       }
                }
        }
 
index 470ffd343c6f83df04058c08b86844d00983e8cc..56c542cb6220fded839695ec26d9b1876144e22e 100644 (file)
@@ -866,7 +866,7 @@ public class PackWriter {
         * Create a bitmap index file to match the pack file just written.
         * <p>
         * This method can only be invoked after
-        * {@link #prepareBitmapIndex(ProgressMonitor, Set)} has been invoked and
+        * {@link #prepareBitmapIndex(ProgressMonitor)} has been invoked and
         * completed successfully. Writing a corresponding bitmap index is an
         * optional feature that not all pack users may require.
         *
@@ -2051,16 +2051,11 @@ public class PackWriter {
         *
         * @param pm
         *            progress monitor to report bitmap building work.
-        * @param want
-        *            collection of objects to be marked as interesting (start
-        *            points of graph traversal).
         * @return whether a bitmap index may be written.
         * @throws IOException
         *             when some I/O problem occur during reading objects.
         */
-       public boolean prepareBitmapIndex(
-                       ProgressMonitor pm, Set<? extends ObjectId> want)
-                       throws IOException {
+       public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException {
                if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE
                                || !cachedPacks.isEmpty())
                        return false;
@@ -2069,8 +2064,8 @@ public class PackWriter {
                        pm = NullProgressMonitor.INSTANCE;
 
                writeBitmaps = new PackBitmapIndexBuilder(sortByName());
-               PackWriterBitmapPreparer bitmapPreparer =
-                               new PackWriterBitmapPreparer(reader, writeBitmaps, pm, want);
+               PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
+                               reader, writeBitmaps, pm, stats.interestingObjects);
 
                int numCommits = objectsLists[Constants.OBJ_COMMIT].size();
                Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits =