sshd: support the ProxyJump ssh config

This is useful to access git repositories behind a bastion server
(jump host).

Add a constant for the config; rewrite the whole connection initiation
to parse the value and (recursively) set up the chain of hops. Add
tests for a single hop and two different ways to configure a two-hop
chain.

The connection timeout applies to each hop in the chain individually.

Change-Id: Idd25af95aa2ec5367404587e4e530b0663c03665
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

6 files changed, 232 insertions, 34 deletions

diff --git a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF
index e6ccbec284..c5c64fcd9a 100644
--- a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF
@@ -45,6 +45,7 @@ Import-Package: net.i2p.crypto.eddsa;version="[0.3.0,0.4.0)",
  org.apache.sshd.client.future;version="[2.4.0,2.5.0)",
  org.apache.sshd.client.keyverifier;version="[2.4.0,2.5.0)",
  org.apache.sshd.client.session;version="[2.4.0,2.5.0)",
+ org.apache.sshd.client.session.forward;version="[2.4.0,2.5.0)",
  org.apache.sshd.client.subsystem.sftp;version="[2.4.0,2.5.0)",
  org.apache.sshd.common;version="[2.4.0,2.5.0)",
  org.apache.sshd.common.auth;version="[2.4.0,2.5.0)",
diff --git a/org.eclipse.jgit.ssh.apache/resources/org/eclipse/jgit/internal/transport/sshd/SshdText.properties b/org.eclipse.jgit.ssh.apache/resources/org/eclipse/jgit/internal/transport/sshd/SshdText.properties
index b89bc606a7..504e6001cc 100644
--- a/org.eclipse.jgit.ssh.apache/resources/org/eclipse/jgit/internal/transport/sshd/SshdText.properties
+++ b/org.eclipse.jgit.ssh.apache/resources/org/eclipse/jgit/internal/transport/sshd/SshdText.properties
@@ -4,8 +4,11 @@ closeListenerFailed=Ssh session close listener failed
 configInvalidPath=Invalid path in ssh config key {0}: {1}
 configInvalidPattern=Invalid pattern in ssh config key {0}: {1}
 configInvalidPositive=Ssh config entry {0} must be a strictly positive number but is ''{1}''
+configInvalidProxyJump=Ssh config, host ''{0}'': Cannot parse ProxyJump ''{1}''
 configNoKnownHostKeyAlgorithms=No implementations for any of the algorithms ''{0}'' given in HostKeyAlgorithms in the ssh config; using the default.
 configNoRemainingHostKeyAlgorithms=Ssh config removed all host key algorithms: HostKeyAlgorithms ''{0}''
+configProxyJumpNotSsh=Non-ssh URI in ProxyJump ssh config
+configProxyJumpWithPath=ProxyJump ssh config: jump host specification must not have a path
 ftpCloseFailed=Closing the SFTP channel failed
 gssapiFailure=GSS-API error for mechanism OID {0}
 gssapiInitFailure=GSS-API initialization failure for mechanism {0}
@@ -46,12 +49,14 @@ knownHostsUnknownKeyPrompt=Accept and store this key, and continue connecting?
 knownHostsUnknownKeyType=Cannot read server key from known hosts file {0}; line {1}
 knownHostsUserAskCreationMsg=File {0} does not exist.
 knownHostsUserAskCreationPrompt=Create file {0} ?
+loginDenied=Log-in denied at {0}:{1}
 passwordPrompt=Password
 proxyCannotAuthenticate=Cannot authenticate to proxy {0}
 proxyHttpFailure=HTTP Proxy connection to {0} failed with code {1}: {2}
 proxyHttpInvalidUserName=HTTP proxy connection {0} with invalid user name; must not contain colons: {1}
 proxyHttpUnexpectedReply=Unexpected HTTP proxy response from {0}: {1}
 proxyHttpUnspecifiedFailureReason=unspecified reason
+proxyJumpAbort=ProxyJump chain too long at {0}
 proxyPasswordPrompt=Proxy password
 proxySocksAuthenticationFailed=Authentication to SOCKS5 proxy {0} failed
 proxySocksFailureForbidden=SOCKS5 proxy {0}: connection to {1} not allowed by ruleset
@@ -80,4 +85,5 @@ sessionWithoutUsername=SSH session created without user name; cannot authenticat
 sshClosingDown=Apache MINA sshd session factory is closing down; cannot create new ssh sessions on this factory
 sshCommandTimeout={0} timed out after {1} seconds while opening the channel
 sshProcessStillRunning={0} is not yet completed, cannot get exit code
+sshProxySessionCloseFailed=Error while closing proxy session {0}
 unknownProxyProtocol=Ignoring unknown proxy protocol {0}
\ No newline at end of file
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java
index 1825fb37b2..beaaecaac9 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java
@@ -49,6 +49,7 @@ import org.apache.sshd.common.keyprovider.KeyIdentityProvider;
 import org.apache.sshd.common.session.SessionContext;
 import org.apache.sshd.common.session.helpers.AbstractSession;
 import org.apache.sshd.common.util.ValidateUtils;
+import org.apache.sshd.common.util.net.SshdSocketAddress;
 import org.eclipse.jgit.internal.transport.sshd.JGitClientSession.ChainingAttributes;
 import org.eclipse.jgit.internal.transport.sshd.JGitClientSession.SessionAttributes;
 import org.eclipse.jgit.internal.transport.sshd.proxy.HttpClientConnector;
@@ -82,6 +83,16 @@ public class JGitSshClient extends SshClient {
 	 */
 	public static final AttributeKey<String> PREFERRED_AUTHENTICATIONS = new AttributeKey<>();
 
+	/**
+	 * An attribute key for storing an alternate local address to connect to if
+	 * a local forward from a ProxyJump ssh config is present. If set,
+	 * {@link #connect(HostConfigEntry, AttributeRepository, SocketAddress)}
+	 * will not connect to the address obtained from the {@link HostConfigEntry}
+	 * but to the address stored in this key (which is assumed to forward the
+	 * {@code HostConfigEntry} address).
+	 */
+	public static final AttributeKey<SshdSocketAddress> LOCAL_FORWARD_ADDRESS = new AttributeKey<>();
+
 	private KeyCache keyCache;
 
 	private CredentialsProvider credentialsProvider;
@@ -102,25 +113,37 @@ public class JGitSshClient extends SshClient {
 			throw new IllegalStateException("SshClient not started."); //$NON-NLS-1$
 		}
 		Objects.requireNonNull(hostConfig, "No host configuration"); //$NON-NLS-1$
-		String host = ValidateUtils.checkNotNullAndNotEmpty(
+		String originalHost = ValidateUtils.checkNotNullAndNotEmpty(
 				hostConfig.getHostName(), "No target host"); //$NON-NLS-1$
-		int port = hostConfig.getPort();
-		ValidateUtils.checkTrue(port > 0, "Invalid port: %d", port); //$NON-NLS-1$
+		int originalPort = hostConfig.getPort();
+		ValidateUtils.checkTrue(originalPort > 0, "Invalid port: %d", //$NON-NLS-1$
+				originalPort);
+		InetSocketAddress originalAddress = new InetSocketAddress(originalHost,
+				originalPort);
+		InetSocketAddress targetAddress = originalAddress;
 		String userName = hostConfig.getUsername();
+		String id = userName + '@' + originalAddress;
 		AttributeRepository attributes = chain(context, this);
-		InetSocketAddress address = new InetSocketAddress(host, port);
-		ConnectFuture connectFuture = new DefaultConnectFuture(
-				userName + '@' + address, null);
+		SshdSocketAddress localForward = attributes
+				.resolveAttribute(LOCAL_FORWARD_ADDRESS);
+		if (localForward != null) {
+			targetAddress = new InetSocketAddress(localForward.getHostName(),
+					localForward.getPort());
+			id += '/' + targetAddress.toString();
+		}
+		ConnectFuture connectFuture = new DefaultConnectFuture(id, null);
 		SshFutureListener<IoConnectFuture> listener = createConnectCompletionListener(
-				connectFuture, userName, address, hostConfig);
-		attributes = sessionAttributes(attributes, hostConfig, address);
+				connectFuture, userName, originalAddress, hostConfig);
+		attributes = sessionAttributes(attributes, hostConfig, originalAddress);
 		// Proxy support
-		ProxyData proxy = getProxyData(address);
-		if (proxy != null) {
-			address = configureProxy(proxy, address);
-			proxy.clearPassword();
+		if (localForward == null) {
+			ProxyData proxy = getProxyData(targetAddress);
+			if (proxy != null) {
+				targetAddress = configureProxy(proxy, targetAddress);
+				proxy.clearPassword();
+			}
 		}
-		connector.connect(address, attributes, localAddress)
+		connector.connect(targetAddress, attributes, localAddress)
 				.addListener(listener);
 		return connectFuture;
 	}
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/SshdText.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/SshdText.java
index 22966f956e..13bb3ebe75 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/SshdText.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/SshdText.java
@@ -24,8 +24,11 @@ public final class SshdText extends TranslationBundle {
 	/***/ public String configInvalidPath;
 	/***/ public String configInvalidPattern;
 	/***/ public String configInvalidPositive;
+	/***/ public String configInvalidProxyJump;
 	/***/ public String configNoKnownHostKeyAlgorithms;
 	/***/ public String configNoRemainingHostKeyAlgorithms;
+	/***/ public String configProxyJumpNotSsh;
+	/***/ public String configProxyJumpWithPath;
 	/***/ public String ftpCloseFailed;
 	/***/ public String gssapiFailure;
 	/***/ public String gssapiInitFailure;
@@ -58,12 +61,14 @@ public final class SshdText extends TranslationBundle {
 	/***/ public String knownHostsUnknownKeyType;
 	/***/ public String knownHostsUserAskCreationMsg;
 	/***/ public String knownHostsUserAskCreationPrompt;
+	/***/ public String loginDenied;
 	/***/ public String passwordPrompt;
 	/***/ public String proxyCannotAuthenticate;
 	/***/ public String proxyHttpFailure;
 	/***/ public String proxyHttpInvalidUserName;
 	/***/ public String proxyHttpUnexpectedReply;
 	/***/ public String proxyHttpUnspecifiedFailureReason;
+	/***/ public String proxyJumpAbort;
 	/***/ public String proxyPasswordPrompt;
 	/***/ public String proxySocksAuthenticationFailed;
 	/***/ public String proxySocksFailureForbidden;
@@ -92,6 +97,7 @@ public final class SshdText extends TranslationBundle {
 	/***/ public String sshClosingDown;
 	/***/ public String sshCommandTimeout;
 	/***/ public String sshProcessStillRunning;
+	/***/ public String sshProxySessionCloseFailed;
 	/***/ public String unknownProxyProtocol;
 
 }
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java
index dfd7cca1b4..0fb0610b99 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018, Thomas Wolf <thomas.wolf@paranor.ch> and others
+ * Copyright (C) 2018, 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -10,36 +10,53 @@
 package org.eclipse.jgit.transport.sshd;
 
 import static java.text.MessageFormat.format;
+import static org.apache.sshd.common.SshConstants.SSH2_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.net.URISyntaxException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 
 import org.apache.sshd.client.SshClient;
 import org.apache.sshd.client.channel.ChannelExec;
 import org.apache.sshd.client.channel.ClientChannelEvent;
+import org.apache.sshd.client.config.hosts.HostConfigEntry;
+import org.apache.sshd.client.future.ConnectFuture;
 import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.client.session.forward.PortForwardingTracker;
 import org.apache.sshd.client.subsystem.sftp.SftpClient;
 import org.apache.sshd.client.subsystem.sftp.SftpClient.CloseableHandle;
 import org.apache.sshd.client.subsystem.sftp.SftpClient.CopyMode;
 import org.apache.sshd.client.subsystem.sftp.SftpClientFactory;
-import org.apache.sshd.common.session.Session;
-import org.apache.sshd.common.session.SessionListener;
+import org.apache.sshd.common.AttributeRepository;
+import org.apache.sshd.common.SshException;
+import org.apache.sshd.common.future.CloseFuture;
+import org.apache.sshd.common.future.SshFutureListener;
 import org.apache.sshd.common.subsystem.sftp.SftpException;
+import org.apache.sshd.common.util.io.IoUtils;
+import org.apache.sshd.common.util.net.SshdSocketAddress;
 import org.eclipse.jgit.annotations.NonNull;
+import org.eclipse.jgit.errors.TransportException;
+import org.eclipse.jgit.internal.transport.sshd.JGitSshClient;
 import org.eclipse.jgit.internal.transport.sshd.SshdText;
 import org.eclipse.jgit.transport.FtpChannel;
 import org.eclipse.jgit.transport.RemoteSession;
+import org.eclipse.jgit.transport.SshConstants;
 import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,6 +70,11 @@ public class SshdSession implements RemoteSession {
 	private static final Logger LOG = LoggerFactory
 			.getLogger(SshdSession.class);
 
+	private static final Pattern SHORT_SSH_FORMAT = Pattern
+			.compile("[-\\w.]+(?:@[-\\w.]+)?(?::\\d+)?"); //$NON-NLS-1$
+
+	private static final int MAX_DEPTH = 10;
+
 	private final CopyOnWriteArrayList<SessionCloseListener> listeners = new CopyOnWriteArrayList<>();
 
 	private final URIish uri;
@@ -71,32 +93,169 @@ public class SshdSession implements RemoteSession {
 			client.start();
 		}
 		try {
-			String username = uri.getUser();
-			String host = uri.getHost();
-			int port = uri.getPort();
-			long t = timeout.toMillis();
-			if (t <= 0) {
-				session = client.connect(username, host, port).verify()
-						.getSession();
-			} else {
-				session = client.connect(username, host, port)
-						.verify(timeout.toMillis()).getSession();
-			}
-			session.addSessionListener(new SessionListener() {
+			session = connect(uri, Collections.emptyList(),
+					future -> notifyCloseListeners(), timeout, MAX_DEPTH);
+		} catch (IOException e) {
+			disconnect(e);
+			throw e;
+		}
+	}
 
-				@Override
-				public void sessionClosed(Session s) {
-					notifyCloseListeners();
+	private ClientSession connect(URIish target, List<URIish> jumps,
+			SshFutureListener<CloseFuture> listener, Duration timeout,
+			int depth) throws IOException {
+		if (--depth < 0) {
+			throw new IOException(
+					format(SshdText.get().proxyJumpAbort, target));
+		}
+		HostConfigEntry hostConfig = getHostConfig(target.getUser(),
+				target.getHost(), target.getPort());
+		String host = hostConfig.getHostName();
+		int port = hostConfig.getPort();
+		List<URIish> hops = determineHops(jumps, hostConfig, target.getHost());
+		ClientSession resultSession = null;
+		ClientSession proxySession = null;
+		PortForwardingTracker portForward = null;
+		try {
+			if (!hops.isEmpty()) {
+				URIish hop = hops.remove(0);
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Connecting to jump host {}", hop); //$NON-NLS-1$
 				}
-			});
+				proxySession = connect(hop, hops, null, timeout, depth);
+			}
+			AttributeRepository context = null;
+			if (proxySession != null) {
+				SshdSocketAddress remoteAddress = new SshdSocketAddress(host,
+						port);
+				portForward = proxySession.createLocalPortForwardingTracker(
+						SshdSocketAddress.LOCALHOST_ADDRESS, remoteAddress);
+				// We must connect to the locally bound address, not the one
+				// from the host config.
+				context = AttributeRepository.ofKeyValuePair(
+						JGitSshClient.LOCAL_FORWARD_ADDRESS,
+						portForward.getBoundAddress());
+			}
+			resultSession = connect(hostConfig, context, timeout);
+			if (proxySession != null) {
+				final PortForwardingTracker tracker = portForward;
+				final ClientSession pSession = proxySession;
+				resultSession.addCloseFutureListener(future -> {
+					IoUtils.closeQuietly(tracker);
+					String sessionName = pSession.toString();
+					try {
+						pSession.close();
+					} catch (IOException e) {
+						LOG.error(format(
+								SshdText.get().sshProxySessionCloseFailed,
+								sessionName), e);
+					}
+				});
+				portForward = null;
+				proxySession = null;
+			}
+			if (listener != null) {
+				resultSession.addCloseFutureListener(listener);
+			}
 			// Authentication timeout is by default 2 minutes.
-			session.auth().verify(session.getAuthTimeout());
+			resultSession.auth().verify(resultSession.getAuthTimeout());
+			return resultSession;
 		} catch (IOException e) {
-			disconnect(e);
+			close(portForward, e);
+			close(proxySession, e);
+			close(resultSession, e);
+			if (e instanceof SshException && ((SshException) e)
+					.getDisconnectCode() == SSH2_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE) {
+				// Ensure the user gets to know on which URI the authentication
+				// was denied.
+				throw new TransportException(target,
+						format(SshdText.get().loginDenied, host,
+								Integer.toString(port)),
+						e);
+			}
 			throw e;
 		}
 	}
 
+	private ClientSession connect(HostConfigEntry config,
+			AttributeRepository context, Duration timeout)
+			throws IOException {
+		ConnectFuture connected = client.connect(config, context, null);
+		long timeoutMillis = timeout.toMillis();
+		if (timeoutMillis <= 0) {
+			connected = connected.verify();
+		} else {
+			connected = connected.verify(timeoutMillis);
+		}
+		return connected.getSession();
+	}
+
+	private void close(Closeable toClose, Throwable error) {
+		if (toClose != null) {
+			try {
+				toClose.close();
+			} catch (IOException e) {
+				error.addSuppressed(e);
+			}
+		}
+	}
+
+	private HostConfigEntry getHostConfig(String username, String host,
+			int port) throws IOException {
+		HostConfigEntry entry = client.getHostConfigEntryResolver()
+				.resolveEffectiveHost(host, port, null, username, null);
+		if (entry == null) {
+			if (SshdSocketAddress.isIPv6Address(host)) {
+				return new HostConfigEntry("", host, port, username); //$NON-NLS-1$
+			}
+			return new HostConfigEntry(host, host, port, username);
+		}
+		return entry;
+	}
+
+	private List<URIish> determineHops(List<URIish> currentHops,
+			HostConfigEntry hostConfig, String host) throws IOException {
+		if (currentHops.isEmpty()) {
+			String jumpHosts = hostConfig.getProperty(SshConstants.PROXY_JUMP);
+			if (!StringUtils.isEmptyOrNull(jumpHosts)) {
+				try {
+					return parseProxyJump(jumpHosts);
+				} catch (URISyntaxException e) {
+					throw new IOException(
+							format(SshdText.get().configInvalidProxyJump, host,
+									jumpHosts),
+							e);
+				}
+			}
+		}
+		return currentHops;
+	}
+
+	private List<URIish> parseProxyJump(String proxyJump)
+			throws URISyntaxException {
+		String[] hops = proxyJump.split(","); //$NON-NLS-1$
+		List<URIish> result = new LinkedList<>();
+		for (String hop : hops) {
+			// There shouldn't be any whitespace, but let's be lenient
+			hop = hop.trim();
+			if (SHORT_SSH_FORMAT.matcher(hop).matches()) {
+				// URIish doesn't understand the short SSH format
+				// user@host:port, only user@host:path
+				hop = SshConstants.SSH_SCHEME + "://" + hop; //$NON-NLS-1$
+			}
+			URIish to = new URIish(hop);
+			if (!SshConstants.SSH_SCHEME.equalsIgnoreCase(to.getScheme())) {
+				throw new URISyntaxException(hop,
+						SshdText.get().configProxyJumpNotSsh);
+			} else if (!StringUtils.isEmptyOrNull(to.getPath())) {
+				throw new URISyntaxException(hop,
+						SshdText.get().configProxyJumpWithPath);
+			}
+			result.add(to);
+		}
+		return result;
+	}
+
 	/**
 	 * Adds a {@link SessionCloseListener} to this session. Has no effect if the
 	 * given {@code listener} is already registered with this session.
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java
index 0f7ab849f5..4ad3c4a4ba 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java
@@ -230,6 +230,9 @@ public class SshdSessionFactory extends SshSessionFactory implements Closeable {
 			return session;
 		} catch (Exception e) {
 			unregister(session);
+			if (e instanceof TransportException) {
+				throw (TransportException) e;
+			}
 			throw new TransportException(uri, e.getMessage(), e);
 		}
 	}