Client-side protocol V2 support for fetching

Make all transports request protocol V2 when fetching. Depending on
the transport, set the GIT_PROTOCOL environment variable (file and
ssh), pass the Git-Protocol header (http), or set the hidden
"\0version=2\0" (git anon). We'll fall back to V0 if the server
doesn't reply with a version 2 answer.

A user can control which protocol the client requests via the git
config protocol.version; if not set, JGit requests protocol V2 for
fetching. Pushing always uses protocol V0 still.

In the API, there is only a new Transport.openFetch() version that
takes a collection of RefSpecs plus additional patterns to construct
the Ref prefixes for the "ls-refs" command in protocol V2. If none
are given, the server will still advertise all refs, even in protocol
V2.

BasePackConnection.readAdvertisedRefs() handles falling back to
protocol V0. It newly returns true if V0 was used and the advertised
refs were read, and false if V2 is used and an explicit "ls-refs" is
needed. (This can't be done transparently inside readAdvertisedRefs()
because a "stateless RPC" transport like TransportHttp may need to
open a new connection for writing.)

BasePackFetchConnection implements the changes needed for the protocol
V2 "fetch" command (stateless protocol, simplified ACK handling,
delimiters, section headers).

In TransportHttp, change readSmartHeaders() to also recognize the
"version 2" packet line as a valid smart server indication.

Adapt tests, and run all the HTTP tests not only with both HTTP
connection factories (JDK and Apache HttpClient) but also with both
protocol V0 and V2. The SSH tests are much slower and much more
focused on the SSH protocol and SSH key handling. Factor out two
very simple cloning and pulling tests and make those run with
protocol V2.

Bug: 553083
Change-Id: I357c7f5daa7efb2872f1c64ee6f6d54229031ae1
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

3 files changed, 150 insertions, 14 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java
index 64b16f659a..7d438c1dd8 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java
@@ -16,11 +16,11 @@ import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectIdRef;
 import org.eclipse.jgit.lib.Ref;
@@ -30,18 +30,6 @@ import org.junit.Test;
 public class BasePackConnectionTest {
 
 	@Test
-	public void testExtractSymRefsFromCapabilities() {
-		final Map<String, String> symRefs = BasePackConnection
-				.extractSymRefsFromCapabilities(
-						Arrays.asList("symref=HEAD:refs/heads/main",
-								"symref=refs/heads/sym:refs/heads/other"));
-
-		assertEquals(2, symRefs.size());
-		assertEquals("refs/heads/main", symRefs.get("HEAD"));
-		assertEquals("refs/heads/other", symRefs.get("refs/heads/sym"));
-	}
-
-	@Test
 	public void testUpdateWithSymRefsAdds() {
 		final Ref mainRef = new ObjectIdRef.Unpeeled(Ref.Storage.LOOSE,
 				"refs/heads/main", ObjectId.fromString(
@@ -230,4 +218,30 @@ public class BasePackConnectionTest {
 		assertThat(refMap, not(hasKey("refs/heads/sym1")));
 		assertThat(refMap, not(hasKey("refs/heads/sym2")));
 	}
+
+	@Test
+	public void testUpdateWithSymRefsFillInHead() {
+		final String oidName = "0000000000000000000000000000000000000001";
+		final Ref advertised = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK,
+				Constants.HEAD, ObjectId.fromString(oidName));
+
+		final Map<String, Ref> refMap = new HashMap<>();
+		refMap.put(advertised.getName(), advertised);
+
+		final Map<String, String> symRefs = new HashMap<>();
+		symRefs.put("HEAD", "refs/heads/main");
+
+		BasePackConnection.updateWithSymRefs(refMap, symRefs);
+
+		assertThat(refMap, hasKey("HEAD"));
+		assertThat(refMap, hasKey("refs/heads/main"));
+		final Ref headRef = refMap.get("HEAD");
+		final Ref mainRef = refMap.get("refs/heads/main");
+		assertThat(headRef, instanceOf(SymbolicRef.class));
+		final SymbolicRef headSymRef = (SymbolicRef) headRef;
+		assertEquals(Constants.HEAD, headSymRef.getName());
+		assertSame(mainRef, headSymRef.getTarget());
+		assertEquals(oidName, headRef.getObjectId().name());
+		assertEquals(oidName, mainRef.getObjectId().name());
+	}
 }
\ No newline at end of file
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java
index 7f03357e9c..505e0088c4 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009, Google Inc. and others
+ * Copyright (C) 2009, 2020 Google Inc. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -13,6 +13,7 @@ package org.eclipse.jgit.transport;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -297,6 +298,58 @@ public class PacketLineInTest {
 		}
 	}
 
+	// parseACKv2
+
+	@Test
+	public void testParseAckV2_NAK() throws IOException {
+		final ObjectId expid = ObjectId
+				.fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e");
+		final MutableObjectId actid = new MutableObjectId();
+		actid.fromString(expid.name());
+
+		assertSame(PacketLineIn.AckNackResult.NAK,
+				PacketLineIn.parseACKv2("NAK", actid));
+		assertEquals(expid, actid);
+	}
+
+	@Test
+	public void testParseAckV2_ACK() throws IOException {
+		final ObjectId expid = ObjectId
+				.fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e");
+		final MutableObjectId actid = new MutableObjectId();
+
+		assertSame(PacketLineIn.AckNackResult.ACK_COMMON,
+				PacketLineIn.parseACKv2(
+						"ACK fcfcfb1fd94829c1a1704f894fc111d14770d34e", actid));
+		assertEquals(expid, actid);
+	}
+
+	@Test
+	public void testParseAckV2_Ready() throws IOException {
+		final ObjectId expid = ObjectId
+				.fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e");
+		final MutableObjectId actid = new MutableObjectId();
+		actid.fromString(expid.name());
+
+		assertSame(PacketLineIn.AckNackResult.ACK_READY,
+				PacketLineIn.parseACKv2("ready", actid));
+		assertEquals(expid, actid);
+	}
+
+	@Test
+	public void testParseAckV2_ERR() {
+		IOException e = assertThrows(IOException.class, () -> PacketLineIn
+				.parseACKv2("ERR want is not valid", new MutableObjectId()));
+		assertTrue(e.getMessage().contains("want is not valid"));
+	}
+
+	@Test
+	public void testParseAckV2_Invalid() {
+		IOException e = assertThrows(IOException.class,
+				() -> PacketLineIn.parseACKv2("HELO", new MutableObjectId()));
+		assertTrue(e.getMessage().contains("xpected ACK/NAK"));
+	}
+
 	// test support
 
 	private void init(String msg) {
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java
new file mode 100644
index 0000000000..d9b85fb003
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.transport;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.eclipse.jgit.lib.Config;
+import org.junit.Test;
+
+/**
+ * Tests for {@link TransferConfig} parsing.
+ */
+public class TransferConfigTest {
+
+	@Test
+	public void testParseProtocolV0() {
+		Config rc = new Config();
+		rc.setInt("protocol", null, "version", 0);
+		TransferConfig tc = new TransferConfig(rc);
+		assertEquals(TransferConfig.ProtocolVersion.V0, tc.protocolVersion);
+	}
+
+	@Test
+	public void testParseProtocolV1() {
+		Config rc = new Config();
+		rc.setInt("protocol", null, "version", 1);
+		TransferConfig tc = new TransferConfig(rc);
+		assertEquals(TransferConfig.ProtocolVersion.V0, tc.protocolVersion);
+	}
+
+	@Test
+	public void testParseProtocolV2() {
+		Config rc = new Config();
+		rc.setInt("protocol", null, "version", 2);
+		TransferConfig tc = new TransferConfig(rc);
+		assertEquals(TransferConfig.ProtocolVersion.V2, tc.protocolVersion);
+	}
+
+	@Test
+	public void testParseProtocolNotSet() {
+		Config rc = new Config();
+		TransferConfig tc = new TransferConfig(rc);
+		assertNull(tc.protocolVersion);
+	}
+
+	@Test
+	public void testParseProtocolUnknown() {
+		Config rc = new Config();
+		rc.setInt("protocol", null, "version", 3);
+		TransferConfig tc = new TransferConfig(rc);
+		assertNull(tc.protocolVersion);
+	}
+
+	@Test
+	public void testParseProtocolInvalid() {
+		Config rc = new Config();
+		rc.setString("protocol", null, "version", "foo");
+		TransferConfig tc = new TransferConfig(rc);
+		assertNull(tc.protocolVersion);
+	}
+}