Allow object reuse selection to occur in parallel

ObjectReader implementations may wish to use multiple threads in
order to evaluate object reuse faster.  Let the reader make that
decision by passing the iteration down into the reader.

Because the work is pushed into the reader, it may need to locate a
given ObjectToPack given its ObjectId.  This can easily occur if the
reader has sent a list of ObjectIds to the object database and gets
back information keyed only by ObjectId, without the ObjectToPack
handle.  Expose lookup using the PackWriter's own internal map,
so the reader doesn't need to build a redundant copy to track the
assocation of ObjectId back to ObjectToPack.

Change-Id: I0c536405a55034881fb5db92a2d2a99534faed34
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java
index 1718ef30f2f24bfff4e86d20ffbde822e1873b18..0e0c73831ae17bd45fc5288e8fe125638aec263f 100644 (file)
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java
@@ -46,6 +46,7 @@ package org.eclipse.jgit.pgm.debug;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.zip.InflaterInputStream;
 
 import org.eclipse.jgit.errors.MissingObjectException;
@@ -103,7 +104,8 @@ class ShowPackDelta extends TextBuiltin {
                };
 
                ByteArrayOutputStream buf = new ByteArrayOutputStream();
-               asis.selectObjectRepresentation(pw, target);
+               asis.selectObjectRepresentation(pw, NullProgressMonitor.INSTANCE,
+                               Collections.singleton(target));
                asis.copyObjectAsIs(new PackOutputStream(NullProgressMonitor.INSTANCE,
                                buf, pw), target);
 

diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties
index 6744f548d3ed707c218611080ef1e60cca497cde..681d289c8416888bbe93a2c40b110a6c2713d6c7 100644 (file)
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties
@@ -321,6 +321,7 @@ repositoryState_rebaseOrApplyMailbox=Rebase/Apply mailbox
 repositoryState_rebaseWithMerge=Rebase w/merge
 requiredHashFunctionNotAvailable=Required hash function {0} not available.
 resolvingDeltas=Resolving deltas
+searchForReuse=Finding sources
 serviceNotPermitted={0} not permitted
 shortCompressedStreamAt=Short compressed stream at {0}
 shortReadOfBlock=Short read of block.

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java
index ece1c29e1d776d76e56b3efa786860a57a5418ab..193d7336790a139b1d9ce28fffd0a8ed753a1a6c 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java
@@ -380,6 +380,7 @@ public class JGitText extends TranslationBundle {
        /***/ public String repositoryState_rebaseWithMerge;
        /***/ public String requiredHashFunctionNotAvailable;
        /***/ public String resolvingDeltas;
+       /***/ public String searchForReuse;
        /***/ public String serviceNotPermitted;
        /***/ public String shortCompressedStreamAt;
        /***/ public String shortReadOfBlock;

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
index 5376d077ad4333a815133872c7066c8e79928014..72a719aa656697ba39d5f1194b7e0f72fd76defd 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
@@ -56,6 +56,7 @@ import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.storage.pack.ObjectReuseAsIs;
 import org.eclipse.jgit.storage.pack.ObjectToPack;
@@ -116,9 +117,13 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs {
                return new LocalObjectToPack(obj);
        }
 
-       public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp)
+       public void selectObjectRepresentation(PackWriter packer,
+                       ProgressMonitor monitor, Iterable<ObjectToPack> objects)
                        throws IOException, MissingObjectException {
-               db.selectObjectRepresentation(packer, otp, this);
+               for (ObjectToPack otp : objects) {
+                       db.selectObjectRepresentation(packer, otp, this);
+                       monitor.update(1);
+               }
        }
 
        public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp)

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
index a815e935420f69fa0d1965cf5776f939b95da95e..4bfaa43ebc49a80af8edc382448a0c9f18ff788c 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
@@ -48,6 +48,7 @@ import java.io.IOException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
 import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.revwalk.RevObject;
 
 /**
@@ -78,24 +79,32 @@ public interface ObjectReuseAsIs {
 
        /**
         * Select the best object representation for a packer.
-        * <p>
+        *
         * Implementations should iterate through all available representations of
         * an object, and pass them in turn to the PackWriter though
         * {@link PackWriter#select(ObjectToPack, StoredObjectRepresentation)} so
         * the writer can select the most suitable representation to reuse into the
         * output stream.
         *
+        * The implementation may choose to consider multiple objects at once on
+        * concurrent threads, but must evaluate all representations of an object
+        * within the same thread.
+        *
         * @param packer
         *            the packer that will write the object in the near future.
-        * @param otp
-        *            the object to pack.
+        * @param monitor
+        *            progress monitor, implementation should update the monitor
+        *            once for each item in the iteration when selection is done.
+        * @param objects
+        *            the objects that are being packed.
         * @throws MissingObjectException
         *             there is no representation available for the object, as it is
         *             no longer in the repository. Packing will abort.
         * @throws IOException
         *             the repository cannot be accessed. Packing will abort.
         */
-       public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp)
+       public void selectObjectRepresentation(PackWriter packer,
+                       ProgressMonitor monitor, Iterable<ObjectToPack> objects)
                        throws IOException, MissingObjectException;
 
        /**

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
index 1bb11c3a0d88f46fe3ed97042e1632bed9522a81..5f6bf4bfcd5c412a6ba6d65e62e6b8d721416fa7 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
@@ -388,7 +388,18 @@ public class PackWriter {
         * @return true if the object will appear in the output pack file.
         */
        public boolean willInclude(final AnyObjectId id) {
-               return objectsMap.get(id) != null;
+               return get(id) != null;
+       }
+
+       /**
+        * Lookup the ObjectToPack object for a given ObjectId.
+        *
+        * @param id
+        *            the object to find in the pack.
+        * @return the object we are packing, or null.
+        */
+       public ObjectToPack get(AnyObjectId id) {
+               return objectsMap.get(id);
        }
 
        /**
@@ -477,7 +488,7 @@ public class PackWriter {
                        writeMonitor = NullProgressMonitor.INSTANCE;
 
                if ((reuseDeltas || config.isReuseObjects()) && reuseSupport != null)
-                       searchForReuse();
+                       searchForReuse(compressMonitor);
                if (config.isDeltaCompress())
                        searchForDeltas(compressMonitor);
 
@@ -504,11 +515,11 @@ public class PackWriter {
                }
        }
 
-       private void searchForReuse() throws IOException {
-               for (List<ObjectToPack> list : objectsLists) {
-                       for (ObjectToPack otp : list)
-                               reuseSupport.selectObjectRepresentation(this, otp);
-               }
+       private void searchForReuse(ProgressMonitor monitor) throws IOException {
+               monitor.beginTask(JGitText.get().searchForReuse, getObjectsNumber());
+               for (List<ObjectToPack> list : objectsLists)
+                       reuseSupport.selectObjectRepresentation(this, monitor, list);
+               monitor.endTask();
        }
 
        private void searchForDeltas(ProgressMonitor monitor)
@@ -840,7 +851,8 @@ public class PackWriter {
                        MissingObjectException {
                otp.clearDeltaBase();
                otp.clearReuseAsIs();
-               reuseSupport.selectObjectRepresentation(this, otp);
+               reuseSupport.selectObjectRepresentation(this,
+                               NullProgressMonitor.INSTANCE, Collections.singleton(otp));
        }
 
        private void writeWholeObjectDeflate(PackOutputStream out,