LFS: Dramatically improve checkout speed with SSH authentication

SSH Authentication is quite expensive (~120ms on localhost against
Gerrit with LFS plugin). The SSH authentication typically also sends a
validity time of the returned token, which allows to re-use it for a
certain time, avoiding the expensive authentication on every download
request. This improves checkout times by large factors depending on the
LFS object amount/sizes.

Also make sure that all instances of Gson used by LFS are configured in
the same way.

Change-Id: I422c94c37021b4322789b3829fa0185e25d683f2
Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com>

4 files changed, 108 insertions, 15 deletions

diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java
index ffc1ee39a7..c522572ee3 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java
@@ -43,8 +43,8 @@
 package org.eclipse.jgit.lfs;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
 import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD;
+import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
 import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK;
 import static org.eclipse.jgit.util.HttpSupport.METHOD_POST;
 import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT;
@@ -123,6 +123,7 @@ public class LfsPrePushHook extends PrePushHook {
 		Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush);
 		uploadContents(api, oid2ptr);
 		return EMPTY;
+
 	}
 
 	private Set<LfsPointer> findObjectsToPush() throws IOException,
@@ -201,7 +202,7 @@ public class LfsPrePushHook extends PrePushHook {
 		for (LfsPointer p : res) {
 			oidStr2ptr.put(p.getOid().name(), p);
 		}
-		Gson gson = new Gson();
+		Gson gson = Protocol.gson();
 		api.getOutputStream().write(
 				gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8));
 		int responseCode = api.getResponseCode();
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
index 81b1810208..d88742ed9e 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
@@ -46,6 +46,10 @@ package org.eclipse.jgit.lfs;
 import java.util.List;
 import java.util.Map;
 
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
 /**
  * This interface describes the network protocol used between lfs client and lfs
  * server
@@ -97,6 +101,24 @@ public interface Protocol {
 		public Map<String, String> header;
 	}
 
+	/**
+	 * An action with an additional expiration timestamp
+	 *
+	 * @since 4.11
+	 */
+	class ExpiringAction extends Action {
+		/**
+		 * Absolute date/time in format "yyyy-MM-dd'T'HH:mm:ss.SSSX"
+		 */
+		public String expiresAt;
+
+		/**
+		 * Validity time in milliseconds (preferred over expiresAt as specified:
+		 * https://github.com/git-lfs/git-lfs/blob/master/docs/api/authentication.md)
+		 */
+		public String expiresIn;
+	}
+
 	/** Describes an error to be returned by the LFS batch API */
 	class Error {
 		public int code;
@@ -138,4 +160,17 @@ public interface Protocol {
 	 * Path to the LFS objects servlet.
 	 */
 	String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$
+
+	/**
+	 * @return a {@link Gson} instance suitable for handling this
+	 *         {@link Protocol}
+	 *
+	 * @since 4.11
+	 */
+	public static Gson gson() {
+		return new GsonBuilder()
+				.setFieldNamingPolicy(
+						FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+				.disableHtmlEscaping().create();
+	}
 }
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
index 1b1b8c1e4a..ae7fab83af 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
@@ -158,7 +158,7 @@ public class SmudgeFilter extends FilterCommand {
 		}
 		HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db,
 				HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD);
-		Gson gson = new Gson();
+		Gson gson = Protocol.gson();
 		lfsServerConn.getOutputStream()
 				.write(gson
 						.toJson(LfsConnectionFactory
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java
index 254bd5ae05..3196e0730e 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java
@@ -52,6 +52,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.ProxySelector;
 import java.net.URL;
+import java.text.SimpleDateFormat;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.TreeMap;
@@ -75,8 +76,6 @@ import org.eclipse.jgit.util.HttpSupport;
 import org.eclipse.jgit.util.io.MessageWriter;
 import org.eclipse.jgit.util.io.StreamCopyThread;
 
-import com.google.gson.Gson;
-
 /**
  * Provides means to get a valid LFS connection for a given repository.
  */
@@ -84,6 +83,7 @@ public class LfsConnectionFactory {
 
 	private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
 	private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
+	private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
 
 	/**
 	 * Determine URL of LFS server by looking into config parameters lfs.url,
@@ -166,17 +166,9 @@ public class LfsConnectionFactory {
 			Map<String, String> additionalHeaders, String remoteUrl) {
 		try {
 			URIish u = new URIish(remoteUrl);
-
 			if (SCHEME_SSH.equals(u.getScheme())) {
-				// discover and authenticate; git-lfs does "ssh -p
-				// <port> -- <host> git-lfs-authenticate <project>
-				// <upload/download>"
-				String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
-						"git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
-								+ " " + purpose); //$NON-NLS-1$
-
-				Protocol.Action action = new Gson().fromJson(json,
-						Protocol.Action.class);
+				Protocol.ExpiringAction action = getSshAuthentication(
+						db, purpose, remoteUrl, u);
 				additionalHeaders.putAll(action.header);
 				return action.href;
 			} else {
@@ -187,6 +179,34 @@ public class LfsConnectionFactory {
 		}
 	}
 
+	private static Protocol.ExpiringAction getSshAuthentication(
+			Repository db, String purpose, String remoteUrl, URIish u)
+			throws IOException {
+		AuthCache cached = sshAuthCache.get(remoteUrl);
+		Protocol.ExpiringAction action = null;
+		if (cached != null && cached.validUntil > System.currentTimeMillis()) {
+			action = cached.cachedAction;
+		}
+
+		if (action == null) {
+			// discover and authenticate; git-lfs does "ssh
+			// -p <port> -- <host> git-lfs-authenticate
+			// <project> <upload/download>"
+			String json = runSshCommand(u.setPath(""), //$NON-NLS-1$
+					db.getFS(),
+					"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
+							+ purpose);
+
+			action = Protocol.gson().fromJson(json,
+					Protocol.ExpiringAction.class);
+
+			// cache the result as long as possible.
+			AuthCache c = new AuthCache(action);
+			sshAuthCache.put(remoteUrl, c);
+		}
+		return action;
+	}
+
 	/**
 	 * Create a connection for the specified
 	 * {@link org.eclipse.jgit.lfs.Protocol.Action}.
@@ -291,4 +311,41 @@ public class LfsConnectionFactory {
 		return req;
 	}
 
+	private static final class AuthCache {
+		private static final long AUTH_CACHE_EAGER_TIMEOUT = 100;
+
+		private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
+				"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
+
+		/**
+		 * Creates a cache entry for an authentication response.
+		 * <p>
+		 * The timeout of the cache token is extracted from the given action. If
+		 * no timeout can be determined, the token will be used only once.
+		 *
+		 * @param action
+		 */
+		public AuthCache(Protocol.ExpiringAction action) {
+			this.cachedAction = action;
+			try {
+				if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
+					this.validUntil = System.currentTimeMillis()
+							+ Long.parseLong(action.expiresIn);
+				} else if (action.expiresAt != null
+						&& !action.expiresAt.isEmpty()) {
+					this.validUntil = ISO_FORMAT.parse(action.expiresAt)
+							.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
+				} else {
+					this.validUntil = System.currentTimeMillis();
+				}
+			} catch (Exception e) {
+				this.validUntil = System.currentTimeMillis();
+			}
+		}
+
+		long validUntil;
+
+		Protocol.ExpiringAction cachedAction;
+	}
+
 }