]> source.dussan.org Git - jgit.git/commitdiff
RefAdvertiser: Avoid object parsing 10/2410/1
authorShawn O. Pearce <spearce@spearce.org>
Wed, 2 Feb 2011 21:48:26 +0000 (13:48 -0800)
committerShawn O. Pearce <spearce@spearce.org>
Thu, 3 Feb 2011 01:16:32 +0000 (17:16 -0800)
It isn't strictly necessary to validate every reference's target
object is reachable in the repository before advertising it to a
client. This is an expensive operation when there are thousands of
references, and its very unlikely that a reference uses a missing
object, because garbage collection proceeds from the references and
walks down through the graph. So trying to hide a dangling reference
from clients is relatively pointless.

Even if we are trying to avoid giving a client a corrupt repository,
this simple check isn't sufficient.  It is possible for a reference to
point to a valid commit, but that commit to have a missing blob in its
root tree.  This can be caused by staging a file into the index,
waiting several weeks, then committing that file while also racing
against a prune.  The prune may delete the blob, since its
modification time is more than 2 weeks ago, but retain the commit,
since its modification time is right now.

Such graph corruption is already caught during PackWriter as it
enumerates the graph from the client's want list and digs back
to the roots or common base.  Leave the reference validation also
for that same phase, where we know we have to parse the object to
support the enumeration.

Change-Id: Iee70ead0d3ed2d2fcc980417d09d7a69b05f5c2f
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoRefsServlet.java
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileRepository.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

index 647919e0657ce6452b120754d12b3f184251320a..ffaa13153c9bbdefb317171d279e5f603d235fea 100644 (file)
@@ -56,8 +56,6 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevFlag;
-import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.transport.RefAdvertiser;
 import org.eclipse.jgit.util.HttpSupport;
 
@@ -73,36 +71,28 @@ class InfoRefsServlet extends HttpServlet {
                rsp.setCharacterEncoding(Constants.CHARACTER_ENCODING);
 
                final Repository db = getRepository(req);
-               final RevWalk walk = new RevWalk(db);
-               try {
-                       final RevFlag ADVERTISED = walk.newFlag("ADVERTISED");
+               final OutputStreamWriter out = new OutputStreamWriter(
+                               new SmartOutputStream(req, rsp), Constants.CHARSET);
+               final RefAdvertiser adv = new RefAdvertiser() {
+                       @Override
+                       protected void writeOne(final CharSequence line) throws IOException {
+                               // Whoever decided that info/refs should use a different
+                               // delimiter than the native git:// protocol shouldn't
+                               // be allowed to design this sort of stuff. :-(
+                               out.append(line.toString().replace(' ', '\t'));
+                       }
 
-                       final OutputStreamWriter out = new OutputStreamWriter(
-                                       new SmartOutputStream(req, rsp), Constants.CHARSET);
-                       final RefAdvertiser adv = new RefAdvertiser() {
-                               @Override
-                               protected void writeOne(final CharSequence line)
-                                               throws IOException {
-                                       // Whoever decided that info/refs should use a different
-                                       // delimiter than the native git:// protocol shouldn't
-                                       // be allowed to design this sort of stuff. :-(
-                                       out.append(line.toString().replace(' ', '\t'));
-                               }
+                       @Override
+                       protected void end() {
+                               // No end marker required for info/refs format.
+                       }
+               };
+               adv.init(db);
+               adv.setDerefTags(true);
 
-                               @Override
-                               protected void end() {
-                                       // No end marker required for info/refs format.
-                               }
-                       };
-                       adv.init(walk, ADVERTISED);
-                       adv.setDerefTags(true);
-
-                       Map<String, Ref> refs = db.getAllRefs();
-                       refs.remove(Constants.HEAD);
-                       adv.send(refs);
-                       out.close();
-               } finally {
-                       walk.release();
-               }
+               Map<String, Ref> refs = db.getAllRefs();
+               refs.remove(Constants.HEAD);
+               adv.send(refs);
+               out.close();
        }
 }
index cedb59472eb6ef770c7d808bf6f99b9b937a8739..5a54e86bfc1d1d87d5273204538080dfa52a3eb0 100644 (file)
@@ -343,8 +343,12 @@ public class FileRepository extends Repository {
                                Repository repo;
 
                                repo = ((AlternateRepository) d).repository;
-                               for (Ref ref : repo.getAllRefs().values())
-                                       r.add(ref.getObjectId());
+                               for (Ref ref : repo.getAllRefs().values()) {
+                                       if (ref.getObjectId() != null)
+                                               r.add(ref.getObjectId());
+                                       if (ref.getPeeledObjectId() != null)
+                                               r.add(ref.getPeeledObjectId());
+                               }
                                r.addAll(repo.getAdditionalHaves());
                        }
                }
index 59037c3bd77942eb4142cf5a0d426733fcc9da59..5ea8501890f9908d55500a64afc185c2f4b9a117 100644 (file)
@@ -694,8 +694,7 @@ public class ReceivePack {
                        return;
                }
 
-               final RevFlag advertised = walk.newFlag("ADVERTISED");
-               adv.init(walk, advertised);
+               adv.init(db);
                adv.advertiseCapability(CAPABILITY_SIDE_BAND_64K);
                adv.advertiseCapability(CAPABILITY_DELETE_REFS);
                adv.advertiseCapability(CAPABILITY_REPORT_STATUS);
index df0afe73faa8a4e3cab5de9f30fef749ab12b0f3..a77910bb9ef635976e3b984b207bb212ed23c7ed 100644 (file)
@@ -44,6 +44,7 @@
 package org.eclipse.jgit.transport;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
@@ -55,10 +56,6 @@ import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.RefComparator;
 import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevFlag;
-import org.eclipse.jgit.revwalk.RevObject;
-import org.eclipse.jgit.revwalk.RevTag;
-import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.util.RefMap;
 
 /** Support for the start of {@link UploadPack} and {@link ReceivePack}. */
@@ -88,33 +85,28 @@ public abstract class RefAdvertiser {
                }
        }
 
-       private RevWalk walk;
-
-       private RevFlag ADVERTISED;
-
        private final StringBuilder tmpLine = new StringBuilder(100);
 
        private final char[] tmpId = new char[Constants.OBJECT_ID_STRING_LENGTH];
 
        private final Set<String> capablities = new LinkedHashSet<String>();
 
+       private final Set<ObjectId> sent = new HashSet<ObjectId>();
+
+       private Repository repository;
+
        private boolean derefTags;
 
        private boolean first = true;
 
        /**
-        * Initialize a new advertisement formatter.
+        * Initialize this advertiser with a repository for peeling tags.
         *
-        * @param protoWalk
-        *            the RevWalk used to parse objects that are advertised.
-        * @param advertisedFlag
-        *            flag marked on any advertised objects parsed out of the
-        *            {@code protoWalk}'s object pool, permitting the caller to
-        *            later quickly determine if an object was advertised (or not).
+        * @param src
+        *            the repository to read from.
         */
-       public void init(final RevWalk protoWalk, final RevFlag advertisedFlag) {
-               walk = protoWalk;
-               ADVERTISED = advertisedFlag;
+       public void init(Repository src) {
+               repository = src;
        }
 
        /**
@@ -124,8 +116,6 @@ public abstract class RefAdvertiser {
         * This method must be invoked prior to any of the following:
         * <ul>
         * <li>{@link #send(Map)}
-        * <li>{@link #advertiseHave(AnyObjectId)}
-        * <li>{@link #includeAdditionalHaves(Repository)}
         * </ul>
         *
         * @param deref
@@ -163,19 +153,31 @@ public abstract class RefAdvertiser {
         *            zero or more refs to format for the client. The collection is
         *            sorted before display if necessary, and therefore may appear
         *            in any order.
+        * @return set of ObjectIds that were advertised to the client.
         * @throws IOException
         *             the underlying output stream failed to write out an
         *             advertisement record.
         */
-       public void send(final Map<String, Ref> refs) throws IOException {
-               for (final Ref r : getSortedRefs(refs)) {
-                       final RevObject obj = parseAnyOrNull(r.getObjectId());
-                       if (obj != null) {
-                               advertiseAny(obj, r.getName());
-                               if (derefTags && obj instanceof RevTag)
-                                       advertiseTag((RevTag) obj, r.getName() + "^{}");
+       public Set<ObjectId> send(Map<String, Ref> refs) throws IOException {
+               for (Ref ref : getSortedRefs(refs)) {
+                       if (ref.getObjectId() == null)
+                               continue;
+
+                       advertiseAny(ref.getObjectId(), ref.getName());
+
+                       if (!derefTags)
+                               continue;
+
+                       if (!ref.isPeeled()) {
+                               if (repository == null)
+                                       continue;
+                               ref = repository.peel(ref);
                        }
+
+                       if (ref.getPeeledObjectId() != null)
+                               advertiseAny(ref.getPeeledObjectId(), ref.getName() + "^{}");
                }
+               return sent;
        }
 
        private Iterable<Ref> getSortedRefs(Map<String, Ref> all) {
@@ -200,12 +202,7 @@ public abstract class RefAdvertiser {
         *             advertisement record.
         */
        public void advertiseHave(AnyObjectId id) throws IOException {
-               RevObject obj = parseAnyOrNull(id);
-               if (obj != null) {
-                       advertiseAnyOnce(obj, ".have");
-                       if (obj instanceof RevTag)
-                               advertiseAnyOnce(((RevTag) obj).getObject(), ".have");
-               }
+               advertiseAnyOnce(id, ".have");
        }
 
        /**
@@ -227,45 +224,18 @@ public abstract class RefAdvertiser {
                return first;
        }
 
-       private RevObject parseAnyOrNull(final AnyObjectId id) {
-               if (id == null)
-                       return null;
-               try {
-                       return walk.parseAny(id);
-               } catch (IOException e) {
-                       return null;
-               }
-       }
-
-       private void advertiseAnyOnce(final RevObject obj, final String refName)
+       private void advertiseAnyOnce(AnyObjectId obj, final String refName)
                        throws IOException {
-               if (!obj.has(ADVERTISED))
+               if (!sent.contains(obj))
                        advertiseAny(obj, refName);
        }
 
-       private void advertiseAny(final RevObject obj, final String refName)
+       private void advertiseAny(AnyObjectId obj, final String refName)
                        throws IOException {
-               obj.add(ADVERTISED);
+               sent.add(obj.toObjectId());
                advertiseId(obj, refName);
        }
 
-       private void advertiseTag(final RevTag tag, final String refName)
-                       throws IOException {
-               RevObject o = tag;
-               do {
-                       // Fully unwrap here so later on we have these already parsed.
-                       final RevObject target = ((RevTag) o).getObject();
-                       try {
-                               walk.parseHeaders(target);
-                       } catch (IOException err) {
-                               return;
-                       }
-                       target.add(ADVERTISED);
-                       o = target;
-               } while (o instanceof RevTag);
-               advertiseAny(tag.getObject(), refName);
-       }
-
        /**
         * Advertise one object under a specific name.
         * <p>
index 0d1c0355a735dc6377f6f7147951c567c19a2b46..1082cc06e18ff0a48de8591d8c49d50c929962d1 100644 (file)
@@ -151,8 +151,8 @@ public class UploadPack {
        /** null if {@link #commonBase} should be examined again. */
        private Boolean okToGiveUp;
 
-       /** Marked on objects we sent in our advertisement list. */
-       private final RevFlag ADVERTISED;
+       /** Objects we sent in our advertisement list, clients can ask for these. */
+       private Set<ObjectId> advertised;
 
        /** Marked on objects the client has asked us to give them. */
        private final RevFlag WANT;
@@ -181,7 +181,6 @@ public class UploadPack {
                walk = new RevWalk(db);
                walk.setRetainBody(false);
 
-               ADVERTISED = walk.newFlag("ADVERTISED");
                WANT = walk.newFlag("WANT");
                PEER_HAS = walk.newFlag("PEER_HAS");
                COMMON = walk.newFlag("COMMON");
@@ -189,7 +188,6 @@ public class UploadPack {
                walk.carry(PEER_HAS);
 
                SAVE = new RevFlagSet();
-               SAVE.add(ADVERTISED);
                SAVE.add(WANT);
                SAVE.add(PEER_HAS);
                refFilter = RefFilter.DEFAULT;
@@ -327,13 +325,11 @@ public class UploadPack {
                if (biDirectionalPipe)
                        sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut));
                else {
+                       advertised = new HashSet<ObjectId>();
                        refs = refFilter.filter(db.getAllRefs());
-                       for (Ref r : refs.values()) {
-                               try {
-                                       walk.parseAny(r.getObjectId()).add(ADVERTISED);
-                               } catch (IOException e) {
-                                       // Skip missing/corrupt objects
-                               }
+                       for (Ref ref : refs.values()) {
+                               if (ref.getObjectId() != null)
+                                       advertised.add(ref.getObjectId());
                        }
                }
 
@@ -361,7 +357,7 @@ public class UploadPack {
         *             the formatter failed to write an advertisement.
         */
        public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException {
-               adv.init(walk, ADVERTISED);
+               adv.init(db);
                adv.advertiseCapability(OPTION_INCLUDE_TAG);
                adv.advertiseCapability(OPTION_MULTI_ACK_DETAILED);
                adv.advertiseCapability(OPTION_MULTI_ACK);
@@ -372,7 +368,7 @@ public class UploadPack {
                adv.advertiseCapability(OPTION_NO_PROGRESS);
                adv.setDerefTags(true);
                refs = refFilter.filter(db.getAllRefs());
-               adv.send(refs);
+               advertised = adv.send(refs);
                adv.end();
        }
 
@@ -425,7 +421,7 @@ public class UploadPack {
                                if (o.has(WANT)) {
                                        // Already processed, the client repeated itself.
 
-                               } else if (o.has(ADVERTISED)) {
+                               } else if (advertised.contains(o)) {
                                        o.add(WANT);
                                        wantAll.add(o);