Merge branch 'stable-5.5'

* stable-5.5:
  Prepare 5.4.4-SNAPSHOT builds
  JGit v5.4.3.201909031940-r
  Prepare 5.3.6-SNAPSHOT builds
  JGit v5.3.5.201909031855-r
  Prepare 5.1.12-SNAPSHOT builds
  JGit v5.1.11.201909031202-r
  Prepare 4.11.10-SNAPSHOT builds
  JGit v4.11.9.201909030838-r
  Bazel: Update bazlets to the latest master revision
  Bazel: Remove FileTreeIteratorWithTimeControl from BUILD file
  BatchRefUpdate: repro racy atomic update, and fix it
  Delete unused FileTreeIteratorWithTimeControl
  Fix RacyGitTests#testRacyGitDetection
  Change RacyGitTests to create a racy git situation in a stable way
  Silence API warnings
  sshd: fix proxy connections with the DefaultProxyDataFactory
  sshd: support the HashKnownHosts configuration
  sshd: configurable server key verification
  sshd: allow setting a null ssh config
  sshd: simplify OpenSshServerKeyVerifier
  sshd: simplify ServerKeyLookup interface
  Use https in update site URLs

Change-Id: Icd21a8fcccffd56bfedbd037e48028308db6d13b
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

3 files changed, 282 insertions, 2 deletions

diff --git a/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF
index b863bf3fe9..4bae3d9b6e 100644
--- a/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF
@@ -7,7 +7,15 @@ Bundle-Version: 5.6.0.qualifier
 Bundle-Vendor: %Bundle-Vendor
 Bundle-Localization: plugin
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
-Import-Package: org.eclipse.jgit.api.errors;version="[5.6.0,5.7.0)",
+Import-Package: org.apache.sshd.client.config.hosts;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.auth;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.config.keys;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.keyprovider;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.session;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.util.net;version="[2.2.0,2.3.0)",
+ org.apache.sshd.common.util.security;version="[2.2.0,2.3.0)",
+ org.eclipse.jgit.api.errors;version="[5.6.0,5.7.0)",
  org.eclipse.jgit.internal.transport.sshd.proxy;version="[5.6.0,5.7.0)",
  org.eclipse.jgit.junit;version="[5.6.0,5.7.0)",
  org.eclipse.jgit.junit.ssh;version="[5.6.0,5.7.0)",
diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java
index df0b832a0f..b9b7353d3e 100644
--- a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java
+++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java
@@ -42,16 +42,22 @@
  */
 package org.eclipse.jgit.transport.sshd;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.sshd.client.config.hosts.KnownHostEntry;
 import org.eclipse.jgit.api.errors.TransportException;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.transport.SshSessionFactory;
 import org.eclipse.jgit.transport.ssh.SshTestBase;
-import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
 import org.eclipse.jgit.util.FS;
 import org.junit.Test;
 import org.junit.experimental.theories.Theories;
@@ -102,6 +108,51 @@ public class ApacheSshTest extends SshTestBase {
 	}
 
 	@Test
+	public void testHashedKnownHosts() throws Exception {
+		assertTrue("Failed to delete known_hosts", knownHosts.delete());
+		// The provider will answer "yes" to all questions, so we should be able
+		// to connect and end up with a new known_hosts file with the host key.
+		TestCredentialsProvider provider = new TestCredentialsProvider();
+		cloneWith("ssh://localhost/doesntmatter", defaultCloneDir, provider, //
+				"HashKnownHosts yes", //
+				"Host localhost", //
+				"HostName localhost", //
+				"Port " + testPort, //
+				"User " + TEST_USER, //
+				"IdentityFile " + privateKey1.getAbsolutePath());
+		List<LogEntry> messages = provider.getLog();
+		assertFalse("Expected user interaction", messages.isEmpty());
+		assertEquals(
+				"Expected to be asked about the key, and the file creation", 2,
+				messages.size());
+		assertTrue("~/.ssh/known_hosts should exist now", knownHosts.exists());
+		// Let's clone again without provider. If it works, the server host key
+		// was written correctly.
+		File clonedAgain = new File(getTemporaryDirectory(), "cloned2");
+		cloneWith("ssh://localhost/doesntmatter", clonedAgain, null, //
+				"Host localhost", //
+				"HostName localhost", //
+				"Port " + testPort, //
+				"User " + TEST_USER, //
+				"IdentityFile " + privateKey1.getAbsolutePath());
+		// Check that the first line contains neither "localhost" nor
+		// "127.0.0.1", but does contain the expected hash.
+		List<String> lines = Files.readAllLines(knownHosts.toPath()).stream()
+				.filter(s -> s != null && s.length() >= 1 && s.charAt(0) != '#'
+						&& !s.trim().isEmpty())
+				.collect(Collectors.toList());
+		assertEquals("Unexpected number of known_hosts lines", 1, lines.size());
+		String line = lines.get(0);
+		assertFalse("Found host in line", line.contains("localhost"));
+		assertFalse("Found IP in line", line.contains("127.0.0.1"));
+		assertTrue("Hash not found", line.contains("|"));
+		KnownHostEntry entry = KnownHostEntry.parseKnownHostEntry(line);
+		assertTrue("Hash doesn't match localhost",
+				entry.isHostMatch("localhost", testPort)
+						|| entry.isHostMatch("127.0.0.1", testPort));
+	}
+
+	@Test
 	public void testPreamble() throws Exception {
 		// Test that the client can deal with strange lines being sent before
 		// the server identification string.
diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/NoFilesSshTest.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/NoFilesSshTest.java
new file mode 100644
index 0000000000..c0dc2cf20e
--- /dev/null
+++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/NoFilesSshTest.java
@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) 2019 Thomas Wolf <thomas.wolf@paranor.ch>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.transport.sshd;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyPair;
+import java.security.PublicKey;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.sshd.common.NamedResource;
+import org.apache.sshd.common.config.keys.KeyUtils;
+import org.apache.sshd.common.keyprovider.KeyIdentityProvider;
+import org.apache.sshd.common.session.SessionContext;
+import org.apache.sshd.common.util.net.SshdSocketAddress;
+import org.apache.sshd.common.util.security.SecurityUtils;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.SshSessionFactory;
+import org.eclipse.jgit.transport.ssh.SshTestHarness;
+import org.eclipse.jgit.util.FS;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Test for using the SshdSessionFactory without files in ~/.ssh but with an
+ * in-memory setup.
+ */
+public class NoFilesSshTest extends SshTestHarness {
+
+
+	private PublicKey testServerKey;
+
+	private KeyPair testUserKey;
+
+	@Override
+	protected SshSessionFactory createSessionFactory() {
+		SshdSessionFactory result = new SshdSessionFactory(new JGitKeyCache(),
+				null) {
+
+			@Override
+			protected File getSshConfig(File dir) {
+				return null;
+			}
+
+			@Override
+			protected ServerKeyDatabase getServerKeyDatabase(File homeDir,
+					File dir) {
+				return new ServerKeyDatabase() {
+
+					@Override
+					public List<PublicKey> lookup(String connectAddress,
+							InetSocketAddress remoteAddress,
+							Configuration config) {
+						return Collections.singletonList(testServerKey);
+					}
+
+					@Override
+					public boolean accept(String connectAddress,
+							InetSocketAddress remoteAddress,
+							PublicKey serverKey, Configuration config,
+							CredentialsProvider provider) {
+						return KeyUtils.compareKeys(serverKey, testServerKey);
+					}
+
+				};
+			}
+
+			@Override
+			protected Iterable<KeyPair> getDefaultKeys(File dir) {
+				// This would work for this simple test case:
+				// return Collections.singletonList(testUserKey);
+				// But let's see if we can check the host and username that's used.
+				// For that, we need access to the sshd SessionContext:
+				return new KeyAuthenticator();
+			}
+
+			@Override
+			protected String getDefaultPreferredAuthentications() {
+				return "publickey";
+			}
+		};
+
+		// The home directory is mocked at this point!
+		result.setHomeDirectory(FS.DETECTED.userHome());
+		result.setSshDirectory(sshDir);
+		return result;
+	}
+
+	private class KeyAuthenticator implements KeyIdentityProvider, Iterable<KeyPair> {
+
+		@Override
+		public Iterator<KeyPair> iterator() {
+			// Should not be called. The use of the Iterable interface in
+			// SshdSessionFactory.getDefaultKeys() made sense in sshd 2.0.0,
+			// but sshd 2.2.0 added the SessionContext, which although good
+			// (without it we couldn't check here) breaks the Iterable analogy.
+			// But we're stuck now with that interface for getDefaultKeys, and
+			// so this override throwing an exception is unfortunately needed.
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public Iterable<KeyPair> loadKeys(SessionContext session)
+				throws IOException, GeneralSecurityException {
+			if (!TEST_USER.equals(session.getUsername())) {
+				return Collections.emptyList();
+			}
+			SshdSocketAddress remoteAddress = SshdSocketAddress
+					.toSshdSocketAddress(session.getRemoteAddress());
+			switch (remoteAddress.getHostName()) {
+			case "localhost":
+			case "127.0.0.1":
+				return Collections.singletonList(testUserKey);
+			default:
+				return Collections.emptyList();
+			}
+		}
+	}
+
+	@After
+	public void cleanUp() {
+		testServerKey = null;
+		testUserKey = null;
+	}
+
+	@Override
+	protected void installConfig(String... config) {
+		File configFile = new File(sshDir, Constants.CONFIG);
+		if (config != null) {
+			try {
+				Files.write(configFile.toPath(), Arrays.asList(config));
+			} catch (IOException e) {
+				throw new UncheckedIOException(e);
+			}
+		}
+	}
+
+	private KeyPair load(Path path) throws Exception {
+		try (InputStream in = Files.newInputStream(path)) {
+			return SecurityUtils
+					.loadKeyPairIdentities(null,
+							NamedResource.ofName(path.toString()), in, null)
+					.iterator().next();
+		}
+	}
+
+	@Test
+	public void testCloneWithBuiltInKeys() throws Exception {
+		// This test should fail unless our in-memory setup is taken: no
+		// known_hosts file, and a config that specifies a non-existing key.
+		File newHostKey = new File(getTemporaryDirectory(), "newhostkey");
+		copyTestResource("id_ed25519", newHostKey);
+		server.addHostKey(newHostKey.toPath(), true);
+		testServerKey = load(newHostKey.toPath()).getPublic();
+		assertTrue(newHostKey.delete());
+		testUserKey = load(privateKey1.getAbsoluteFile().toPath());
+		assertNotNull(testServerKey);
+		assertNotNull(testUserKey);
+		cloneWith(
+				"ssh://" + TEST_USER + "@localhost:" + testPort
+						+ "/doesntmatter",
+				new File(getTemporaryDirectory(), "cloned"), null, //
+				"Host localhost", //
+				"IdentityFile "
+						+ new File(sshDir, "does_not_exist").getAbsolutePath());
+	}
+
+}