]> source.dussan.org Git - jgit.git/commitdiff
Teach UploadPack shallow fetch in protocol v2 56/123956/2
authorJonathan Tan <jonathantanmy@google.com>
Thu, 15 Mar 2018 22:56:50 +0000 (15:56 -0700)
committerJonathan Nieder <jrn@google.com>
Tue, 5 Jun 2018 05:08:18 +0000 (22:08 -0700)
Add support for the "shallow" and "deepen" parameters in the "fetch"
command in the fetch-pack/upload-pack protocol v2. Advertise support for
this in the capability advertisement.

TODO: implement deepen-relative, deepen-since, deepen-not

Change-Id: I7ffd80d6c38872f9d713ac7d6e0412106b3766d7
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Jonathan Nieder <jrn@google.com>
org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

index 133ecd01eb2e3c93f4dd9df591e6ddbda117e56a..45ea7fafd9d3c665aebd2e0b4455adc0493368c6 100644 (file)
@@ -377,7 +377,7 @@ public class UploadPackTest {
                        // allow additional commands to be added to the list,
                        // and additional capabilities to be added to existing
                        // commands without requiring test changes.
-                       hasItems("ls-refs", "fetch"));
+                       hasItems("ls-refs", "fetch=shallow"));
                assertTrue(pckIn.readString() == PacketLineIn.END);
                return recvStream;
        }
@@ -877,6 +877,102 @@ public class UploadPackTest {
                assertTrue(stats.getNumOfsDelta() != 0);
        }
 
+       @Test
+       public void testV2FetchShallow() throws Exception {
+               RevCommit commonParent = remote.commit().message("parent").create();
+               RevCommit fooChild = remote.commit().message("x").parent(commonParent).create();
+               RevCommit barChild = remote.commit().message("y").parent(commonParent).create();
+               remote.update("branch1", barChild);
+
+               // Without shallow, the server thinks that we have
+               // commonParent, so it doesn't send it.
+               ByteArrayInputStream recvStream = uploadPackV2(
+                       "command=fetch\n",
+                       PacketLineIn.DELIM,
+                       "want " + barChild.toObjectId().getName() + "\n",
+                       "have " + fooChild.toObjectId().getName() + "\n",
+                       "done\n",
+                       PacketLineIn.END);
+               PacketLineIn pckIn = new PacketLineIn(recvStream);
+               assertThat(pckIn.readString(), is("packfile"));
+               parsePack(recvStream);
+               assertTrue(client.hasObject(barChild.toObjectId()));
+               assertFalse(client.hasObject(commonParent.toObjectId()));
+
+               // With shallow, the server knows that we don't have
+               // commonParent, so it sends it.
+               recvStream = uploadPackV2(
+                       "command=fetch\n",
+                       PacketLineIn.DELIM,
+                       "want " + barChild.toObjectId().getName() + "\n",
+                       "have " + fooChild.toObjectId().getName() + "\n",
+                       "shallow " + fooChild.toObjectId().getName() + "\n",
+                       "done\n",
+                       PacketLineIn.END);
+               pckIn = new PacketLineIn(recvStream);
+               assertThat(pckIn.readString(), is("packfile"));
+               parsePack(recvStream);
+               assertTrue(client.hasObject(commonParent.toObjectId()));
+       }
+
+       @Test
+       public void testV2FetchDeepenAndDone() throws Exception {
+               RevCommit parent = remote.commit().message("parent").create();
+               RevCommit child = remote.commit().message("x").parent(parent).create();
+               remote.update("branch1", child);
+
+               // "deepen 1" sends only the child.
+               ByteArrayInputStream recvStream = uploadPackV2(
+                       "command=fetch\n",
+                       PacketLineIn.DELIM,
+                       "want " + child.toObjectId().getName() + "\n",
+                       "deepen 1\n",
+                       "done\n",
+                       PacketLineIn.END);
+               PacketLineIn pckIn = new PacketLineIn(recvStream);
+               assertThat(pckIn.readString(), is("shallow-info"));
+               assertThat(pckIn.readString(), is("shallow " + child.toObjectId().getName()));
+               assertThat(pckIn.readString(), theInstance(PacketLineIn.DELIM));
+               assertThat(pckIn.readString(), is("packfile"));
+               parsePack(recvStream);
+               assertTrue(client.hasObject(child.toObjectId()));
+               assertFalse(client.hasObject(parent.toObjectId()));
+
+               // Without that, the parent is sent too.
+               recvStream = uploadPackV2(
+                       "command=fetch\n",
+                       PacketLineIn.DELIM,
+                       "want " + child.toObjectId().getName() + "\n",
+                       "done\n",
+                       PacketLineIn.END);
+               pckIn = new PacketLineIn(recvStream);
+               assertThat(pckIn.readString(), is("packfile"));
+               parsePack(recvStream);
+               assertTrue(client.hasObject(parent.toObjectId()));
+       }
+
+       @Test
+       public void testV2FetchDeepenWithoutDone() throws Exception {
+               RevCommit parent = remote.commit().message("parent").create();
+               RevCommit child = remote.commit().message("x").parent(parent).create();
+               remote.update("branch1", child);
+
+               ByteArrayInputStream recvStream = uploadPackV2(
+                       "command=fetch\n",
+                       PacketLineIn.DELIM,
+                       "want " + child.toObjectId().getName() + "\n",
+                       "deepen 1\n",
+                       PacketLineIn.END);
+               PacketLineIn pckIn = new PacketLineIn(recvStream);
+
+               // Verify that only the correct section is sent. "shallow-info"
+               // is not sent because, according to the specification, it is
+               // sent only if a packfile is sent.
+               assertThat(pckIn.readString(), is("acknowledgments"));
+               assertThat(pckIn.readString(), is("NAK"));
+               assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
+       }
+
        private static class RejectAllRefFilter implements RefFilter {
                @Override
                public Map<String, Ref> filter(Map<String, Ref> refs) {
index f17391cfd825606a34b4d01b9059b4b7603e25e7..b608ca8533f85e1b9f4f3a73beb75a857b7ebc1a 100644 (file)
@@ -224,6 +224,8 @@ credentialPassword=Password
 credentialUsername=Username
 daemonAlreadyRunning=Daemon already running
 daysAgo={0} days ago
+deepenNotWithDeepen=Cannot combine deepen with deepen-not
+deepenSinceWithDeepen=Cannot combine deepen with deepen-since
 deleteBranchUnexpectedResult=Delete branch returned unexpected result {0}
 deleteFileFailed=Could not delete file {0}
 deleteRequiresZeroNewId=Delete requires new ID to be zero
@@ -395,6 +397,7 @@ invalidStageForPath=Invalid stage {0} for path {1}
 invalidSystemProperty=Invalid system property ''{0}'': ''{1}''; using default value {2}
 invalidTagOption=Invalid tag option: {0}
 invalidTimeout=Invalid timeout: {0}
+invalidTimestamp=Invalid timestamp in {0}
 invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2}
 invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3}
 invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name
index 12c264f51be1f9528e06855e813e3df64ae992b4..2ac75e1c2fb3262e680fe0c53205296cd48c8aec 100644 (file)
@@ -285,6 +285,8 @@ public class JGitText extends TranslationBundle {
        /***/ public String credentialUsername;
        /***/ public String daemonAlreadyRunning;
        /***/ public String daysAgo;
+       /***/ public String deepenNotWithDeepen;
+       /***/ public String deepenSinceWithDeepen;
        /***/ public String deleteBranchUnexpectedResult;
        /***/ public String deleteFileFailed;
        /***/ public String deleteRequiresZeroNewId;
@@ -455,6 +457,7 @@ public class JGitText extends TranslationBundle {
        /***/ public String invalidSystemProperty;
        /***/ public String invalidTagOption;
        /***/ public String invalidTimeout;
+       /***/ public String invalidTimestamp;
        /***/ public String invalidTimeUnitValue2;
        /***/ public String invalidTimeUnitValue3;
        /***/ public String invalidTreeZeroLengthName;
index 572549e97f5bdfbb70db2042dbeece013679bed0..10cd77530451699caadaa24992f336a4643519ee 100644 (file)
@@ -107,6 +107,14 @@ public class GitProtocolConstants {
         */
        public static final String OPTION_SHALLOW = "shallow"; //$NON-NLS-1$
 
+       /**
+        * The client wants the "deepen" command to be interpreted as relative to
+        * the client's shallow commits.
+        *
+        * @since 5.0
+        */
+       public static final String OPTION_DEEPEN_RELATIVE = "deepen-relative"; //$NON-NLS-1$
+
        /**
         * The client does not want progress messages and will ignore them.
         *
index df3e9bfe1a879b145cc23c616fd9bad73559d415..f38dfe485c73a93f25f315cabd60320cef966dce 100644 (file)
@@ -50,6 +50,7 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
+import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_DEEPEN_RELATIVE;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK;
@@ -123,7 +124,7 @@ public class UploadPack {
        private static final String[] v2CapabilityAdvertisement = {
                "version 2", //$NON-NLS-1$
                COMMAND_LS_REFS,
-               COMMAND_FETCH
+               COMMAND_FETCH + '=' + OPTION_SHALLOW
        };
 
        /** Policy the server uses to validate client requests */
@@ -302,6 +303,19 @@ public class UploadPack {
        /** Desired depth from the client on a shallow request. */
        private int depth;
 
+       /**
+        * Commit time of the newest objects the client has asked us using
+        * --shallow-since not to send. Cannot be nonzero if depth is nonzero.
+        */
+       private int shallowSince;
+
+       /**
+        * (Possibly short) ref names, ancestors of which the client has asked us
+        * not to send using --shallow-exclude. Cannot be non-null if depth is
+        * nonzero.
+        */
+       private @Nullable List<String> shallowExcludeRefs;
+
        /** Commit time of the oldest common commit, in seconds. */
        private int oldestTime;
 
@@ -813,7 +827,7 @@ public class UploadPack {
                        if (!clientShallowCommits.isEmpty())
                                verifyClientShallow();
                        if (depth != 0)
-                               processShallow(unshallowCommits);
+                               processShallow(null, unshallowCommits, true);
                        if (!clientShallowCommits.isEmpty())
                                walk.assumeShallow(clientShallowCommits);
                        sendPack = negotiate(accumulator);
@@ -968,12 +982,65 @@ public class UploadPack {
                                includeTag = true;
                        } else if (line.equals(OPTION_OFS_DELTA)) {
                                options.add(OPTION_OFS_DELTA);
+                       } else if (line.startsWith("shallow ")) { //$NON-NLS-1$
+                               clientShallowCommits.add(ObjectId.fromString(line.substring(8)));
+                       } else if (line.startsWith("deepen ")) { //$NON-NLS-1$
+                               depth = Integer.parseInt(line.substring(7));
+                               if (depth <= 0) {
+                                       throw new PackProtocolException(
+                                                       MessageFormat.format(JGitText.get().invalidDepth,
+                                                                       Integer.valueOf(depth)));
+                               }
+                               if (shallowSince != 0) {
+                                       throw new PackProtocolException(
+                                                       JGitText.get().deepenSinceWithDeepen);
+                               }
+                               if (shallowExcludeRefs != null) {
+                                       throw new PackProtocolException(
+                                                       JGitText.get().deepenNotWithDeepen);
+                               }
+                       } else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$
+                               List<String> exclude = shallowExcludeRefs;
+                               if (exclude == null) {
+                                       exclude = shallowExcludeRefs = new ArrayList<>();
+                               }
+                               exclude.add(line.substring(11));
+                               if (depth != 0) {
+                                       throw new PackProtocolException(
+                                                       JGitText.get().deepenNotWithDeepen);
+                               }
+                       } else if (line.equals(OPTION_DEEPEN_RELATIVE)) {
+                               options.add(OPTION_DEEPEN_RELATIVE);
+                       } else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$
+                               shallowSince = Integer.parseInt(line.substring(13));
+                               if (shallowSince <= 0) {
+                                       throw new PackProtocolException(
+                                                       MessageFormat.format(
+                                                                       JGitText.get().invalidTimestamp, line));
+                               }
+                               if (depth !=  0) {
+                                       throw new PackProtocolException(
+                                                       JGitText.get().deepenSinceWithDeepen);
+                               }
                        }
                        // else ignore it
                }
                rawOut.stopBuffering();
 
                boolean sectionSent = false;
+               @Nullable List<ObjectId> shallowCommits = null;
+               List<ObjectId> unshallowCommits = new ArrayList<>();
+
+               if (!clientShallowCommits.isEmpty()) {
+                       verifyClientShallow();
+               }
+               if (depth != 0 || shallowSince != 0 || shallowExcludeRefs != null) {
+                       shallowCommits = new ArrayList<ObjectId>();
+                       processShallow(shallowCommits, unshallowCommits, false);
+               }
+               if (!clientShallowCommits.isEmpty())
+                       walk.assumeShallow(clientShallowCommits);
+
                if (doneReceived) {
                        processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE));
                } else {
@@ -991,7 +1058,21 @@ public class UploadPack {
                        }
                        sectionSent = true;
                }
+
                if (doneReceived || okToGiveUp()) {
+                       if (shallowCommits != null) {
+                               if (sectionSent)
+                                       pckOut.writeDelim();
+                               pckOut.writeString("shallow-info\n"); //$NON-NLS-1$
+                               for (ObjectId o : shallowCommits) {
+                                       pckOut.writeString("shallow " + o.getName() + '\n'); //$NON-NLS-1$
+                               }
+                               for (ObjectId o : unshallowCommits) {
+                                       pckOut.writeString("unshallow " + o.getName() + '\n'); //$NON-NLS-1$
+                               }
+                               sectionSent = true;
+                       }
+
                        if (sectionSent)
                                pckOut.writeDelim();
                        pckOut.writeString("packfile\n"); //$NON-NLS-1$
@@ -1078,9 +1159,21 @@ public class UploadPack {
 
        /*
         * Determines what "shallow" and "unshallow" lines to send to the user.
-        * The information is written to pckOut and unshallowCommits.
+        * The information is written to shallowCommits (if not null) and
+        * unshallowCommits, and also written to #pckOut (if writeToPckOut is
+        * true).
         */
-       private void processShallow(List<ObjectId> unshallowCommits) throws IOException {
+       private void processShallow(@Nullable List<ObjectId> shallowCommits,
+                       List<ObjectId> unshallowCommits,
+                       boolean writeToPckOut) throws IOException {
+               if (options.contains(OPTION_DEEPEN_RELATIVE) ||
+                               shallowSince != 0 ||
+                               shallowExcludeRefs != null) {
+                       // TODO(jonathantanmy): Implement deepen-relative, deepen-since,
+                       // and deepen-not.
+                       throw new UnsupportedOperationException();
+               }
+
                int walkDepth = depth - 1;
                try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
                                walk.getObjectReader(), walkDepth)) {
@@ -1101,19 +1194,29 @@ public class UploadPack {
                                // Commits at the boundary which aren't already shallow in
                                // the client need to be marked as such
                                if (c.getDepth() == walkDepth
-                                               && !clientShallowCommits.contains(c))
-                                       pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
+                                               && !clientShallowCommits.contains(c)) {
+                                       if (shallowCommits != null) {
+                                               shallowCommits.add(c.copy());
+                                       }
+                                       if (writeToPckOut) {
+                                               pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
+                                       }
+                               }
 
                                // Commits not on the boundary which are shallow in the client
                                // need to become unshallowed
                                if (c.getDepth() < walkDepth
                                                && clientShallowCommits.remove(c)) {
                                        unshallowCommits.add(c.copy());
-                                       pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
+                                       if (writeToPckOut) {
+                                               pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
+                                       }
                                }
                        }
                }
-               pckOut.end();
+               if (writeToPckOut) {
+                       pckOut.end();
+               }
        }
 
        private void verifyClientShallow()