Teach UploadPack to support filtering by blob size

Teach UploadPack to advertise the filter capability and support a
"filter" line in the request, accepting blob sizes only, if the
configuration variable "uploadpack.allowfilter" is true. This feature is
currently in the "master" branch of Git, and as of the time of writing,
this feature is to be released in Git 2.17.

This is incomplete in that the filter-by-sparse-specification feature
also supported by Git is not included in this patch.

If a JGit server were to be patched with this commit, and a repository
on that server configured with RequestPolicy.ANY or
RequestPolicy.REACHABLE_COMMIT_TIP, a Git client built from the "master"
branch would be able to perform a partial clone.

Change-Id: If72b4b422c06ab432137e9e5272d353b14b73259
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>

7 files changed, 140 insertions, 4 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 753a7f9a8e..3c02302d22 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -368,6 +368,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String fileModeNotSetForPath;
 	/***/ public String filterExecutionFailed;
 	/***/ public String filterExecutionFailedRc;
+	/***/ public String filterRequiresCapability;
 	/***/ public String findingGarbage;
 	/***/ public String flagIsDisposed;
 	/***/ public String flagNotFromThis;
@@ -419,6 +420,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String invalidDepth;
 	/***/ public String invalidEncryption;
 	/***/ public String invalidExpandWildcard;
+	/***/ public String invalidFilter;
 	/***/ public String invalidGitdirRef;
 	/***/ public String invalidGitType;
 	/***/ public String invalidId;
@@ -726,6 +728,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String timeIsUncertain;
 	/***/ public String timerAlreadyTerminated;
 	/***/ public String tooManyCommands;
+	/***/ public String tooManyFilters;
 	/***/ public String tooManyIncludeRecursions;
 	/***/ public String topologicalSortRequired;
 	/***/ public String transportExceptionBadRef;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 84e8edee8f..8b16a2ecd7 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -303,6 +303,8 @@ public class PackWriter implements AutoCloseable {
 
 	private ObjectCountCallback callback;
 
+	private long filterBlobLimit = -1;
+
 	/**
 	 * Create writer for specified repository.
 	 * <p>
@@ -639,6 +641,14 @@ public class PackWriter implements AutoCloseable {
 	}
 
 	/**
+	 * @param bytes exclude blobs of size greater than this
+	 * @since 5.0
+	 */
+	public void setFilterBlobLimit(long bytes) {
+		filterBlobLimit = bytes;
+	}
+
+	/**
 	 * Returns objects number in a pack file that was created by this writer.
 	 *
 	 * @return number of objects in pack.
@@ -1960,7 +1970,7 @@ public class PackWriter implements AutoCloseable {
 				byte[] pathBuf = walker.getPathBuffer();
 				int pathLen = walker.getPathLength();
 				bases.addBase(o.getType(), pathBuf, pathLen, pathHash);
-				addObject(o, pathHash);
+				filterAndAddObject(o, o.getType(), pathHash);
 				countingMonitor.update(1);
 			}
 		} else {
@@ -1970,7 +1980,7 @@ public class PackWriter implements AutoCloseable {
 					continue;
 				if (exclude(o))
 					continue;
-				addObject(o, walker.getPathHashCode());
+				filterAndAddObject(o, o.getType(), walker.getPathHashCode());
 				countingMonitor.update(1);
 			}
 		}
@@ -2003,7 +2013,7 @@ public class PackWriter implements AutoCloseable {
 				needBitmap.remove(objectId);
 				continue;
 			}
-			addObject(objectId, obj.getType(), 0);
+			filterAndAddObject(objectId, obj.getType(), 0);
 		}
 
 		if (thin)
@@ -2062,6 +2072,24 @@ public class PackWriter implements AutoCloseable {
 		objectsMap.add(otp);
 	}
 
+	/**
+	 * Adds the given object as an object to be packed, first performing
+	 * filtering on blobs at or exceeding a given size.
+	 *
+	 * @see #setFilterBlobLimit
+	 */
+	private void filterAndAddObject(@NonNull AnyObjectId src, int type,
+			int pathHashCode) throws IOException {
+		// Check if this object needs to be rejected, doing the cheaper
+		// checks first.
+		boolean reject = filterBlobLimit >= 0 &&
+			type == OBJ_BLOB &&
+			reader.getObjectSize(src, OBJ_BLOB) > filterBlobLimit;
+		if (!reject) {
+			addObject(src, type, pathHashCode);
+		}
+	}
+
 	private boolean exclude(AnyObjectId objectId) {
 		if (excludeInPacks == null)
 			return false;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java
index 1383045031..0de9dfd7e2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java
@@ -200,6 +200,13 @@ public abstract class BasePackFetchConnection extends BasePackConnection
 	 */
 	public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
 
+	/**
+	 * The client specified a filter expression.
+	 *
+	 * @since 5.0
+	 */
+	public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;
+
 	private final RevWalk walk;
 
 	/** All commits that are immediately reachable by a local ref. */
@@ -242,6 +249,9 @@ public abstract class BasePackFetchConnection extends BasePackConnection
 
 	private PacketLineOut pckState;
 
+	/** If not -1, the maximum blob size to be sent to the server. */
+	private final long filterBlobLimit;
+
 	/**
 	 * Create a new connection to fetch using the native git transport.
 	 *
@@ -262,6 +272,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection
 		}
 		includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
 		thinPack = transport.isFetchThin();
+		filterBlobLimit = transport.getFilterBlobLimit();
 
 		if (local != null) {
 			walk = new RevWalk(local);
@@ -524,6 +535,11 @@ public abstract class BasePackFetchConnection extends BasePackConnection
 		if (first) {
 			return false;
 		}
+		if (filterBlobLimit == 0) {
+			p.writeString(OPTION_FILTER + " blob:none");
+		} else if (filterBlobLimit > 0) {
+			p.writeString(OPTION_FILTER + " blob:limit=" + filterBlobLimit);
+		}
 		p.end();
 		outNeedsEnd = false;
 		return true;
@@ -564,6 +580,11 @@ public abstract class BasePackFetchConnection extends BasePackConnection
 					OPTION_MULTI_ACK_DETAILED));
 		}
 
+		if (filterBlobLimit >= 0 && !wantCapability(line, OPTION_FILTER)) {
+			throw new PackProtocolException(uri,
+					JGitText.get().filterRequiresCapability);
+		}
+
 		addUserAgentCapability(line);
 		return line.toString();
 	}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java
index 2031147820..6d39dcd8a7 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java
@@ -152,6 +152,13 @@ public class GitProtocolConstants {
 	public static final String OPTION_PUSH_CERT = "push-cert"; //$NON-NLS-1$
 
 	/**
+	 * The client specified a filter expression.
+	 *
+	 * @since 5.0
+	 */
+	public static final String OPTION_FILTER = "filter"; //$NON-NLS-1$
+
+	/**
 	 * The client supports atomic pushes. If this option is used, the server
 	 * will update all refs within one atomic transaction.
 	 *
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java
index 4c70725e42..3b4181d6de 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java
@@ -101,6 +101,7 @@ public class TransferConfig {
 	private final boolean safeForMacOS;
 	private final boolean allowTipSha1InWant;
 	private final boolean allowReachableSha1InWant;
+	private final boolean allowFilter;
 	final String[] hideRefs;
 
 	TransferConfig(final Repository db) {
@@ -153,6 +154,8 @@ public class TransferConfig {
 				"uploadpack", "allowtipsha1inwant", false); //$NON-NLS-1$ //$NON-NLS-2$
 		allowReachableSha1InWant = rc.getBoolean(
 				"uploadpack", "allowreachablesha1inwant", false); //$NON-NLS-1$ //$NON-NLS-2$
+		allowFilter = rc.getBoolean(
+				"uploadpack", "allowfilter", false); //$NON-NLS-1$ //$NON-NLS-2$
 		hideRefs = rc.getStringList("uploadpack", null, "hiderefs"); //$NON-NLS-1$ //$NON-NLS-2$
 	}
 
@@ -220,6 +223,14 @@ public class TransferConfig {
 	}
 
 	/**
+	 * @return true if clients are allowed to specify a "filter" line
+	 * @since 5.0
+	 */
+	public boolean isAllowFilter() {
+		return allowFilter;
+	}
+
+	/**
 	 * Get {@link org.eclipse.jgit.transport.RefFilter} respecting configured
 	 * hidden refs.
 	 *
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java
index 0a8091faaa..afefbff5d8 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java
@@ -791,6 +791,8 @@ public abstract class Transport implements AutoCloseable {
 	/** Should refs no longer on the source be pruned from the destination? */
 	private boolean removeDeletedRefs;
 
+	private long filterBlobLimit = -1;
+
 	/** Timeout in seconds to wait before aborting an IO read or write. */
 	private int timeout;
 
@@ -1066,6 +1068,23 @@ public abstract class Transport implements AutoCloseable {
 	}
 
 	/**
+	 * @return the last value passed to {@link #setFilterBlobLimit}, or -1 if
+	 *         it was never invoked.
+	 * @since 5.0
+	 */
+	public long getFilterBlobLimit() {
+		return filterBlobLimit;
+	}
+
+	/**
+	 * @param bytes exclude blobs of size greater than this
+	 * @since 5.0
+	 */
+	public void setFilterBlobLimit(final long bytes) {
+		filterBlobLimit = bytes;
+	}
+
+	/**
 	 * Apply provided remote configuration on this transport.
 	 *
 	 * @param cfg
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
index 86c6a148fa..7f36ab98d2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
@@ -47,6 +47,7 @@ import static org.eclipse.jgit.lib.RefDatabase.ALL;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
+import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
@@ -317,6 +318,8 @@ public class UploadPack {
 
 	private PackStatistics statistics;
 
+	private long filterBlobLimit = -1;
+
 	/**
 	 * Create a new pack upload for an open repository.
 	 *
@@ -942,6 +945,9 @@ public class UploadPack {
 				|| policy == null)
 			adv.advertiseCapability(OPTION_ALLOW_REACHABLE_SHA1_IN_WANT);
 		adv.advertiseCapability(OPTION_AGENT, UserAgent.get());
+		if (transferConfig.isAllowFilter()) {
+			adv.advertiseCapability(OPTION_FILTER);
+		}
 		adv.setDerefTags(true);
 		Map<String, Ref> advertisedOrDefaultRefs = getAdvertisedOrDefaultRefs();
 		findSymrefs(adv, advertisedOrDefaultRefs);
@@ -982,6 +988,7 @@ public class UploadPack {
 
 	private void recvWants() throws IOException {
 		boolean isFirst = true;
+		boolean filterReceived = false;
 		for (;;) {
 			String line;
 			try {
@@ -1010,6 +1017,41 @@ public class UploadPack {
 				continue;
 			}
 
+			if (transferConfig.isAllowFilter()
+					&& line.startsWith(OPTION_FILTER + " ")) { //$NON-NLS-1$
+				String arg = line.substring(OPTION_FILTER.length() + 1);
+
+				if (filterReceived) {
+					throw new PackProtocolException(JGitText.get().tooManyFilters);
+				}
+				filterReceived = true;
+
+				if (arg.equals("blob:none")) {
+					filterBlobLimit = 0;
+				} else if (arg.startsWith("blob:limit=")) {
+					try {
+						filterBlobLimit = Long.parseLong(arg.substring("blob:limit=".length()));
+					} catch (NumberFormatException e) {
+						throw new PackProtocolException(
+								MessageFormat.format(JGitText.get().invalidFilter,
+										arg));
+					}
+				}
+				/*
+				 * We must have (1) either "blob:none" or
+				 * "blob:limit=" set (because we only support
+				 * blob size limits for now), and (2) if the
+				 * latter, then it must be nonnegative. Throw
+				 * if (1) or (2) is not met.
+				 */
+				if (filterBlobLimit < 0) {
+					throw new PackProtocolException(
+							MessageFormat.format(JGitText.get().invalidFilter,
+									arg));
+				}
+				continue;
+			}
+
 			if (!line.startsWith("want ") || line.length() < 45) //$NON-NLS-1$
 				throw new PackProtocolException(MessageFormat.format(JGitText.get().expectedGot, "want", line)); //$NON-NLS-1$
 
@@ -1547,7 +1589,12 @@ public class UploadPack {
 				accumulator);
 		try {
 			pw.setIndexDisabled(true);
-			pw.setUseCachedPacks(true);
+			if (filterBlobLimit >= 0) {
+				pw.setFilterBlobLimit(filterBlobLimit);
+				pw.setUseCachedPacks(false);
+			} else {
+				pw.setUseCachedPacks(true);
+			}
 			pw.setUseBitmaps(depth == 0 && clientShallowCommits.isEmpty());
 			pw.setClientShallowCommits(clientShallowCommits);
 			pw.setReuseDeltaCommits(true);