From 40e9c384052a9b86ed0b51dc435d83e58515e48d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 22 Aug 2018 14:45:37 -0700 Subject: [PATCH] Add fetch support to ProtocolV2Hook This makes it symmetrical with ls-refs operation and gives the instantiator of UploadPack the chance to run some code after parsing the protocol and before any actual work for the fetch starts. Request and Builder methods keep the naming in the original code to make this change just about request encapsulation and hook invocation. They are package-private for now to allow further improvements. Change-Id: I5ad585c914d3a5f23b11c8251803faa224beffb4 Signed-off-by: Ivan Frade --- .../jgit/transport/FetchV2Request.java | 335 ++++++++++++++++++ .../jgit/transport/ProtocolV2Hook.java | 9 + .../eclipse/jgit/transport/UploadPack.java | 100 ++++-- 3 files changed, 405 insertions(+), 39 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java new file mode 100644 index 0000000000..188c0e0570 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.lib.ObjectId; + +/** + * fetch protocol v2 request. + * + *

+ * This is used as an input to {@link ProtocolV2Hook}. + * + * @since 5.1 + */ +public final class FetchV2Request { + private final List peerHas; + + private final TreeMap wantedRefs; + + private final Set wantsIds; + + private final Set clientShallowCommits; + + private final int shallowSince; + + private final List shallowExcludeRefs; + + private final int depth; + + private final long filterBlobLimit; + + private final Set options; + + private FetchV2Request(List peerHas, + TreeMap wantedRefs, Set wantsIds, + Set clientShallowCommits, int shallowSince, + List shallowExcludeRefs, int depth, long filterBlobLimit, + Set options) { + this.peerHas = peerHas; + this.wantedRefs = wantedRefs; + this.wantsIds = wantsIds; + this.clientShallowCommits = clientShallowCommits; + this.shallowSince = shallowSince; + this.shallowExcludeRefs = shallowExcludeRefs; + this.depth = depth; + this.filterBlobLimit = filterBlobLimit; + this.options = options; + } + + /** + * @return object ids in the "have" lines of the request + */ + @NonNull + List getPeerHas() { + return this.peerHas; + } + + /** + * @return list of references in the "want-ref" lines of the request + */ + @NonNull + Map getWantedRefs() { + return this.wantedRefs; + } + + /** + * @return object ids in the "want" (and "want-ref") lines of the request + */ + @NonNull + Set getWantsIds() { + return wantsIds; + } + + /** + * Shallow commits the client already has. + * + * These are sent by the client in "shallow" request lines. + * + * @return set of commits the client has declared as shallow. + */ + @NonNull + Set getClientShallowCommits() { + return clientShallowCommits; + } + + /** + * The value in a "deepen-since" line in the request, indicating the + * timestamp where to stop fetching/cloning. + * + * @return timestamp where to stop the shallow fetch/clone. Defaults to 0 if + * not set in the request + */ + int getShallowSince() { + return shallowSince; + } + + /** + * @return the refs in "deepen-not" lines in the request. + */ + @NonNull + List getShallowExcludeRefs() { + return shallowExcludeRefs; + } + + /** + * @return the depth set in a "deepen" line. 0 by default. + */ + int getDepth() { + return depth; + } + + /** + * @return the blob limit set in a "filter" line (-1 if not set) + */ + long getFilterBlobLimit() { + return filterBlobLimit; + } + + /** + * Options that tune the expected response from the server, like + * "thin-pack", "no-progress" or "ofs-delta" + * + * These are options listed and well-defined in the git protocol + * specification + * + * @return options found in the request lines + */ + @NonNull + Set getOptions() { + return options; + } + + /** @return A builder of {@link FetchV2Request}. */ + static Builder builder() { + return new Builder(); + } + + + /** A builder for {@link FetchV2Request}. */ + static final class Builder { + List peerHas = new ArrayList<>(); + + TreeMap wantedRefs = new TreeMap<>(); + + Set wantsIds = new HashSet<>(); + + Set clientShallowCommits = new HashSet<>(); + + List shallowExcludeRefs = new ArrayList<>(); + + Set options = new HashSet<>(); + + int depth; + + int shallowSince; + + long filterBlobLimit = -1; + + private Builder() { + } + + /** + * @param objectId + * from a "have" line in a fetch request + * @return the builder + */ + Builder addPeerHas(ObjectId objectId) { + peerHas.add(objectId); + return this; + } + + /** + * From a "want-ref" line in a fetch request + * + * @param refName + * reference name + * @param oid + * object id + * @return the builder + */ + Builder addWantedRef(String refName, ObjectId oid) { + wantedRefs.put(refName, oid); + return this; + } + + /** + * @param option + * fetch request lines acting as options + * @return the builder + */ + Builder addOption(String option) { + options.add(option); + return this; + } + + /** + * @param objectId + * from a "want" line in a fetch request + * @return the builder + */ + Builder addWantsIds(ObjectId objectId) { + wantsIds.add(objectId); + return this; + } + + /** + * @param shallowOid + * from a "shallow" line in the fetch request + * @return the builder + */ + Builder addClientShallowCommit(ObjectId shallowOid) { + this.clientShallowCommits.add(shallowOid); + return this; + } + + /** + * @param d + * from a "deepen" line in the fetch request + * @return the builder + */ + Builder setDepth(int d) { + this.depth = d; + return this; + } + + /** + * @return depth set in the request (via a "deepen" line). Defaulting to + * 0 if not set. + */ + int getDepth() { + return this.depth; + } + + /** + * @return if there has been any "deepen not" line in the request + */ + boolean hasShallowExcludeRefs() { + return shallowExcludeRefs.size() > 0; + } + + /** + * @param shallowExcludeRef reference in a "deepen not" line + * @return the builder + */ + Builder addShallowExcludeRefs(String shallowExcludeRef) { + this.shallowExcludeRefs.add(shallowExcludeRef); + return this; + } + + /** + * @param value + * shallow since value received in a "deepen since" line + * @return the builder + */ + Builder setShallowSince(int value) { + this.shallowSince = value; + return this; + } + + /** + * @return shallow since value, sent before in a "deepen since" line. 0 + * by default. + */ + int getShallowSince() { + return this.shallowSince; + } + + /** + * @param filterBlobLimit + * set in a "filter" line + * @return the builder + */ + Builder setFilterBlobLimit(long filterBlobLimit) { + this.filterBlobLimit = filterBlobLimit; + return this; + } + + /** + * @return Initialized fetch request + */ + FetchV2Request build() { + return new FetchV2Request(peerHas, wantedRefs, wantsIds, + clientShallowCommits, shallowSince, shallowExcludeRefs, + depth, filterBlobLimit, options); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java index 02760fdde4..d67b90b6b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java @@ -79,4 +79,13 @@ public interface ProtocolV2Hook { throws ServiceMayNotContinueException { // Do nothing by default. } + + /** + * @param req the fetch request + * @throws ServiceMayNotContinueException abort; the message will be sent to the user + */ + default void onFetch(FetchV2Request req) + throws ServiceMayNotContinueException { + // Do nothing by default + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 3b66839545..60938facbd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -79,7 +79,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; @@ -945,11 +944,11 @@ public class UploadPack { } private void fetchV2() throws IOException { - options = new HashSet<>(); + FetchV2Request.Builder reqBuilder = FetchV2Request.builder(); // Packs are always sent multiplexed and using full 64K // lengths. - options.add(OPTION_SIDE_BAND_64K); + reqBuilder.addOption(OPTION_SIDE_BAND_64K); // Depending on the requestValidator, #processHaveLines may // require that advertised be set. Set it only in the required @@ -964,7 +963,6 @@ public class UploadPack { } String line; - List peerHas = new ArrayList<>(); boolean doneReceived = false; // Currently, we do not support any capabilities, so the next @@ -976,10 +974,9 @@ public class UploadPack { boolean includeTag = false; boolean filterReceived = false; - TreeMap wantedRefs = new TreeMap<>(); while ((line = pckIn.readString()) != PacketLineIn.END) { if (line.startsWith("want ")) { //$NON-NLS-1$ - wantIds.add(ObjectId.fromString(line.substring(5))); + reqBuilder.addWantsIds(ObjectId.fromString(line.substring(5))); } else if (transferConfig.isAllowRefInWant() && line.startsWith(OPTION_WANT_REF + " ")) { //$NON-NLS-1$ String refName = line.substring(OPTION_WANT_REF.length() + 1); @@ -995,64 +992,68 @@ public class UploadPack { MessageFormat.format(JGitText.get().invalidRefName, refName)); } - wantedRefs.put(refName, oid); - wantIds.add(oid); + reqBuilder.addWantedRef(refName, oid); + reqBuilder.addWantsIds(oid); } else if (line.startsWith("have ")) { //$NON-NLS-1$ - peerHas.add(ObjectId.fromString(line.substring(5))); + reqBuilder.addPeerHas(ObjectId.fromString(line.substring(5))); } else if (line.equals("done")) { //$NON-NLS-1$ doneReceived = true; } else if (line.equals(OPTION_THIN_PACK)) { - options.add(OPTION_THIN_PACK); + reqBuilder.addOption(OPTION_THIN_PACK); } else if (line.equals(OPTION_NO_PROGRESS)) { - options.add(OPTION_NO_PROGRESS); + reqBuilder.addOption(OPTION_NO_PROGRESS); } else if (line.equals(OPTION_INCLUDE_TAG)) { - options.add(OPTION_INCLUDE_TAG); + reqBuilder.addOption(OPTION_INCLUDE_TAG); includeTag = true; } else if (line.equals(OPTION_OFS_DELTA)) { - options.add(OPTION_OFS_DELTA); + reqBuilder.addOption(OPTION_OFS_DELTA); } else if (line.startsWith("shallow ")) { //$NON-NLS-1$ - clientShallowCommits.add(ObjectId.fromString(line.substring(8))); + reqBuilder.addClientShallowCommit( + ObjectId.fromString(line.substring(8))); } else if (line.startsWith("deepen ")) { //$NON-NLS-1$ - depth = Integer.parseInt(line.substring(7)); - if (depth <= 0) { + int parsedDepth = Integer.parseInt(line.substring(7)); + if (parsedDepth <= 0) { throw new PackProtocolException( MessageFormat.format(JGitText.get().invalidDepth, Integer.valueOf(depth))); } - if (shallowSince != 0) { + if (reqBuilder.getShallowSince() != 0) { throw new PackProtocolException( JGitText.get().deepenSinceWithDeepen); } - if (!shallowExcludeRefs.isEmpty()) { + if (reqBuilder.hasShallowExcludeRefs()) { throw new PackProtocolException( JGitText.get().deepenNotWithDeepen); } + reqBuilder.setDepth(parsedDepth); } else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$ - shallowExcludeRefs.add(line.substring(11)); - if (depth != 0) { + reqBuilder.addShallowExcludeRefs(line.substring(11)); + if (reqBuilder.getDepth() != 0) { throw new PackProtocolException( JGitText.get().deepenNotWithDeepen); } } else if (line.equals(OPTION_DEEPEN_RELATIVE)) { - options.add(OPTION_DEEPEN_RELATIVE); + reqBuilder.addOption(OPTION_DEEPEN_RELATIVE); } else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$ - shallowSince = Integer.parseInt(line.substring(13)); - if (shallowSince <= 0) { + int parsedShallowSince = Integer.parseInt(line.substring(13)); + if (parsedShallowSince <= 0) { throw new PackProtocolException( MessageFormat.format( JGitText.get().invalidTimestamp, line)); } - if (depth != 0) { + if (reqBuilder.getDepth() != 0) { throw new PackProtocolException( JGitText.get().deepenSinceWithDeepen); } + reqBuilder.setShallowSince(parsedShallowSince); } else if (transferConfig.isAllowFilter() && line.startsWith(OPTION_FILTER + ' ')) { if (filterReceived) { throw new PackProtocolException(JGitText.get().tooManyFilters); } filterReceived = true; - parseFilter(line.substring(OPTION_FILTER.length() + 1)); + reqBuilder.setFilterBlobLimit(parseFilter( + line.substring(OPTION_FILTER.length() + 1))); } else { throw new PackProtocolException(MessageFormat .format(JGitText.get().unexpectedPacketLine, line)); @@ -1060,30 +1061,46 @@ public class UploadPack { } rawOut.stopBuffering(); + FetchV2Request req = reqBuilder.build(); + protocolV2Hook.onFetch(req); + + // TODO(ifrade): Refactor to pass around the Request object, instead of + // copying data back to class fields + options = req.getOptions(); + wantIds.addAll(req.getWantsIds()); + clientShallowCommits.addAll(req.getClientShallowCommits()); + depth = req.getDepth(); + shallowSince = req.getShallowSince(); + filterBlobLimit = req.getFilterBlobLimit(); + shallowExcludeRefs = req.getShallowExcludeRefs(); + boolean sectionSent = false; @Nullable List shallowCommits = null; List unshallowCommits = new ArrayList<>(); - if (!clientShallowCommits.isEmpty()) { + if (!req.getClientShallowCommits().isEmpty()) { verifyClientShallow(); } - if (depth != 0 || shallowSince != 0 || !shallowExcludeRefs.isEmpty()) { + if (req.getDepth() != 0 || req.getShallowSince() != 0 + || !req.getShallowExcludeRefs().isEmpty()) { shallowCommits = new ArrayList<>(); processShallow(shallowCommits, unshallowCommits, false); } - if (!clientShallowCommits.isEmpty()) - walk.assumeShallow(clientShallowCommits); + if (!req.getClientShallowCommits().isEmpty()) + walk.assumeShallow(req.getClientShallowCommits()); if (doneReceived) { - processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE)); + processHaveLines(req.getPeerHas(), ObjectId.zeroId(), + new PacketLineOut(NullOutputStream.INSTANCE)); } else { pckOut.writeString("acknowledgments\n"); //$NON-NLS-1$ - for (ObjectId id : peerHas) { + for (ObjectId id : req.getPeerHas()) { if (walk.getObjectReader().has(id)) { pckOut.writeString("ACK " + id.getName() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ } } - processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE)); + processHaveLines(req.getPeerHas(), ObjectId.zeroId(), + new PacketLineOut(NullOutputStream.INSTANCE)); if (okToGiveUp()) { pckOut.writeString("ready\n"); //$NON-NLS-1$ } else if (commonBase.isEmpty()) { @@ -1106,12 +1123,13 @@ public class UploadPack { sectionSent = true; } - if (!wantedRefs.isEmpty()) { + if (!req.getWantedRefs().isEmpty()) { if (sectionSent) { pckOut.writeDelim(); } pckOut.writeString("wanted-refs\n"); //$NON-NLS-1$ - for (Map.Entry entry : wantedRefs.entrySet()) { + for (Map.Entry entry : req.getWantedRefs() + .entrySet()) { pckOut.writeString(entry.getValue().getName() + ' ' + entry.getKey() + '\n'); } @@ -1437,12 +1455,14 @@ public class UploadPack { return msgOut; } - private void parseFilter(String arg) throws PackProtocolException { + private long parseFilter(String arg) throws PackProtocolException { + long blobLimit = -1; + if (arg.equals("blob:none")) { //$NON-NLS-1$ - filterBlobLimit = 0; + blobLimit = 0; } else if (arg.startsWith("blob:limit=")) { //$NON-NLS-1$ try { - filterBlobLimit = Long.parseLong( + blobLimit = Long.parseLong( arg.substring("blob:limit=".length())); //$NON-NLS-1$ } catch (NumberFormatException e) { throw new PackProtocolException( @@ -1457,11 +1477,13 @@ public class UploadPack { * latter, then it must be nonnegative. Throw * if (1) or (2) is not met. */ - if (filterBlobLimit < 0) { + if (blobLimit < 0) { throw new PackProtocolException( MessageFormat.format(JGitText.get().invalidFilter, arg)); } + + return blobLimit; } private void recvWants() throws IOException { @@ -1504,7 +1526,7 @@ public class UploadPack { } filterReceived = true; - parseFilter(arg); + filterBlobLimit = parseFilter(arg); continue; } -- 2.39.5