From b85af06324644e699bd2cc7e2047ae1f65f35db5 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Jul 2010 18:12:23 -0700 Subject: [PATCH] Allow object reuse selection to occur in parallel ObjectReader implementations may wish to use multiple threads in order to evaluate object reuse faster. Let the reader make that decision by passing the iteration down into the reader. Because the work is pushed into the reader, it may need to locate a given ObjectToPack given its ObjectId. This can easily occur if the reader has sent a list of ObjectIds to the object database and gets back information keyed only by ObjectId, without the ObjectToPack handle. Expose lookup using the PackWriter's own internal map, so the reader doesn't need to build a redundant copy to track the assocation of ObjectId back to ObjectToPack. Change-Id: I0c536405a55034881fb5db92a2d2a99534faed34 Signed-off-by: Shawn O. Pearce --- .../eclipse/jgit/pgm/debug/ShowPackDelta.java | 4 ++- .../org/eclipse/jgit/JGitText.properties | 1 + .../src/org/eclipse/jgit/JGitText.java | 1 + .../jgit/storage/file/WindowCursor.java | 9 ++++-- .../jgit/storage/pack/ObjectReuseAsIs.java | 17 ++++++++--- .../eclipse/jgit/storage/pack/PackWriter.java | 28 +++++++++++++------ 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java index 1718ef30f2..0e0c73831a 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.pgm.debug; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.zip.InflaterInputStream; import org.eclipse.jgit.errors.MissingObjectException; @@ -103,7 +104,8 @@ class ShowPackDelta extends TextBuiltin { }; ByteArrayOutputStream buf = new ByteArrayOutputStream(); - asis.selectObjectRepresentation(pw, target); + asis.selectObjectRepresentation(pw, NullProgressMonitor.INSTANCE, + Collections.singleton(target)); asis.copyObjectAsIs(new PackOutputStream(NullProgressMonitor.INSTANCE, buf, pw), target); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties index 6744f548d3..681d289c84 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties @@ -321,6 +321,7 @@ repositoryState_rebaseOrApplyMailbox=Rebase/Apply mailbox repositoryState_rebaseWithMerge=Rebase w/merge requiredHashFunctionNotAvailable=Required hash function {0} not available. resolvingDeltas=Resolving deltas +searchForReuse=Finding sources serviceNotPermitted={0} not permitted shortCompressedStreamAt=Short compressed stream at {0} shortReadOfBlock=Short read of block. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java index ece1c29e1d..193d733679 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java @@ -380,6 +380,7 @@ public class JGitText extends TranslationBundle { /***/ public String repositoryState_rebaseWithMerge; /***/ public String requiredHashFunctionNotAvailable; /***/ public String resolvingDeltas; + /***/ public String searchForReuse; /***/ public String serviceNotPermitted; /***/ public String shortCompressedStreamAt; /***/ public String shortReadOfBlock; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java index 5376d077ad..72a719aa65 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java @@ -56,6 +56,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.InflaterCache; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.storage.pack.ObjectReuseAsIs; import org.eclipse.jgit.storage.pack.ObjectToPack; @@ -116,9 +117,13 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { return new LocalObjectToPack(obj); } - public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp) + public void selectObjectRepresentation(PackWriter packer, + ProgressMonitor monitor, Iterable objects) throws IOException, MissingObjectException { - db.selectObjectRepresentation(packer, otp, this); + for (ObjectToPack otp : objects) { + db.selectObjectRepresentation(packer, otp, this); + monitor.update(1); + } } public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java index a815e93542..4bfaa43ebc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java @@ -48,6 +48,7 @@ import java.io.IOException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.revwalk.RevObject; /** @@ -78,24 +79,32 @@ public interface ObjectReuseAsIs { /** * Select the best object representation for a packer. - *

+ * * Implementations should iterate through all available representations of * an object, and pass them in turn to the PackWriter though * {@link PackWriter#select(ObjectToPack, StoredObjectRepresentation)} so * the writer can select the most suitable representation to reuse into the * output stream. * + * The implementation may choose to consider multiple objects at once on + * concurrent threads, but must evaluate all representations of an object + * within the same thread. + * * @param packer * the packer that will write the object in the near future. - * @param otp - * the object to pack. + * @param monitor + * progress monitor, implementation should update the monitor + * once for each item in the iteration when selection is done. + * @param objects + * the objects that are being packed. * @throws MissingObjectException * there is no representation available for the object, as it is * no longer in the repository. Packing will abort. * @throws IOException * the repository cannot be accessed. Packing will abort. */ - public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp) + public void selectObjectRepresentation(PackWriter packer, + ProgressMonitor monitor, Iterable objects) throws IOException, MissingObjectException; /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index 1bb11c3a0d..5f6bf4bfcd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -388,7 +388,18 @@ public class PackWriter { * @return true if the object will appear in the output pack file. */ public boolean willInclude(final AnyObjectId id) { - return objectsMap.get(id) != null; + return get(id) != null; + } + + /** + * Lookup the ObjectToPack object for a given ObjectId. + * + * @param id + * the object to find in the pack. + * @return the object we are packing, or null. + */ + public ObjectToPack get(AnyObjectId id) { + return objectsMap.get(id); } /** @@ -477,7 +488,7 @@ public class PackWriter { writeMonitor = NullProgressMonitor.INSTANCE; if ((reuseDeltas || config.isReuseObjects()) && reuseSupport != null) - searchForReuse(); + searchForReuse(compressMonitor); if (config.isDeltaCompress()) searchForDeltas(compressMonitor); @@ -504,11 +515,11 @@ public class PackWriter { } } - private void searchForReuse() throws IOException { - for (List list : objectsLists) { - for (ObjectToPack otp : list) - reuseSupport.selectObjectRepresentation(this, otp); - } + private void searchForReuse(ProgressMonitor monitor) throws IOException { + monitor.beginTask(JGitText.get().searchForReuse, getObjectsNumber()); + for (List list : objectsLists) + reuseSupport.selectObjectRepresentation(this, monitor, list); + monitor.endTask(); } private void searchForDeltas(ProgressMonitor monitor) @@ -840,7 +851,8 @@ public class PackWriter { MissingObjectException { otp.clearDeltaBase(); otp.clearReuseAsIs(); - reuseSupport.selectObjectRepresentation(this, otp); + reuseSupport.selectObjectRepresentation(this, + NullProgressMonitor.INSTANCE, Collections.singleton(otp)); } private void writeWholeObjectDeflate(PackOutputStream out, -- 2.39.5