Fetch: add support for shallow

This adds support for shallow cloning. The CloneCommand and the
FetchCommand now have the new methods setDepth, setShallowSince and
addShallowExclude to tell the server that the client doesn't want to
download the complete history.

Bug: 475615
Change-Id: Ic80fb6efb5474543ae59be590ebe385bec21cc0d

3 files changed, 275 insertions, 5 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java
index c928d2ad22..6053c8c568 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2011, 2013 Chris Aniszczyk <caniszczyk@gmail.com> and others
+ * Copyright (C) 2011, 2022 Chris Aniszczyk <caniszczyk@gmail.com> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -19,10 +19,14 @@ import static org.junit.Assert.fail;
 import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 import org.eclipse.jgit.api.ListBranchCommand.ListMode;
 import org.eclipse.jgit.api.errors.GitAPIException;
@@ -40,6 +44,7 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevBlob;
 import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.submodule.SubmoduleStatus;
 import org.eclipse.jgit.submodule.SubmoduleStatusType;
 import org.eclipse.jgit.submodule.SubmoduleWalk;
@@ -895,6 +900,234 @@ public class CloneCommandTest extends RepositoryTestCase {
 		assertEquals("refs/heads/test-copy", git2.getRepository().getFullBranch());
 	}
 
+    @Test
+    public void testCloneRepositoryWithDepth() throws IOException, JGitInternalException, GitAPIException {
+		File directory = createTempDirectory("testCloneRepositoryWithDepth");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+        command.setDepth(1);
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(1, log.size());
+		RevCommit commit = log.get(0);
+		assertEquals(Set.of(commit.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals("Second commit", commit.getFullMessage());
+		assertEquals(0, commit.getParentCount());
+	}
+
+	@Test
+	public void testCloneRepositoryWithDepthAndAllBranches() throws IOException, JGitInternalException, GitAPIException {
+		File directory = createTempDirectory("testCloneRepositoryWithDepthAndAllBranches");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.setDepth(1);
+		command.setCloneAllBranches(true);
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(log.stream().map(RevCommit::getId).collect(Collectors.toSet()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals(List.of("Second commit", "Initial commit"),
+				log.stream().map(RevCommit::getFullMessage).collect(Collectors.toList()));
+		for (RevCommit commit : log) {
+			assertEquals(0, commit.getParentCount());
+		}
+	}
+
+	@Test
+	public void testCloneRepositoryWithDepth2() throws Exception {
+		RevCommit parent = tr.git().log().call().iterator().next();
+		RevCommit commit = tr.commit()
+				.parent(parent)
+				.message("Third commit")
+				.add("test.txt", "Hello world")
+				.create();
+		tr.update("refs/heads/test", commit);
+
+		File directory = createTempDirectory("testCloneRepositoryWithDepth2");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.setDepth(2);
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(Set.of(parent.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals(List.of("Third commit", "Second commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+		assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)),
+				log.stream().map(RevCommit::getParentCount)
+						.collect(Collectors.toList()));
+	}
+
+	@Test
+	public void testCloneRepositoryWithDepthAndFetch() throws Exception {
+		File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetch");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.setDepth(1);
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		RevCommit parent = tr.git().log().call().iterator().next();
+		RevCommit commit = tr.commit()
+				.parent(parent)
+				.message("Third commit")
+				.add("test.txt", "Hello world")
+				.create();
+		tr.update("refs/heads/test", commit);
+
+		git2.fetch().call();
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(Set.of(parent.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals(List.of("Third commit", "Second commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+		assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)),
+				log.stream().map(RevCommit::getParentCount)
+						.collect(Collectors.toList()));
+	}
+
+	@Test
+	public void testCloneRepositoryWithDepthAndFetchWithDepth() throws Exception {
+		File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetchWithDepth");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.setDepth(1);
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		RevCommit parent = tr.git().log().call().iterator().next();
+		RevCommit commit = tr.commit()
+				.parent(parent)
+				.message("Third commit")
+				.add("test.txt", "Hello world")
+				.create();
+		tr.update("refs/heads/test", commit);
+
+		git2.fetch().setDepth(1).call();
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(
+				log.stream().map(RevObject::getId).collect(Collectors.toSet()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals(List.of("Third commit", "Second commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+		assertEquals(List.of(Integer.valueOf(0), Integer.valueOf(0)),
+				log.stream().map(RevCommit::getParentCount)
+						.collect(Collectors.toList()));
+	}
+
+	@Test
+	public void testCloneRepositoryWithDepthAndFetchUnshallow() throws Exception {
+		File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetchUnshallow");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.setDepth(1);
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		git2.fetch().setUnshallow(true).call();
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(Set.of(),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals(List.of("Second commit", "Initial commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+		assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)),
+				log.stream().map(RevCommit::getParentCount)
+						.collect(Collectors.toList()));
+	}
+
+    @Test
+	public void testCloneRepositoryWithShallowSince() throws Exception {
+		RevCommit commit = tr.commit()
+				.parent(tr.git().log().call().iterator().next())
+				.message("Third commit").add("test.txt", "Hello world")
+				.create();
+		tr.update("refs/heads/test", commit);
+
+        File directory = createTempDirectory("testCloneRepositoryWithShallowSince");
+        CloneCommand command = Git.cloneRepository();
+        command.setDirectory(directory);
+        command.setURI(fileUri());
+        command.setShallowSince(Instant.ofEpochSecond(commit.getCommitTime()));
+        command.setBranchesToClone(Set.of("refs/heads/test"));
+        Git git2 = command.call();
+        addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(1, log.size());
+		assertEquals(Set.of(commit.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals("Third commit", log.get(0).getFullMessage());
+		assertEquals(0, log.get(0).getParentCount());
+    }
+
+	@Test
+	public void testCloneRepositoryWithShallowExclude() throws Exception {
+		RevCommit parent = tr.git().log().call().iterator().next();
+		tr.update("refs/heads/test",
+				tr.commit()
+					.parent(parent)
+					.message("Third commit")
+					.add("test.txt", "Hello world")
+					.create());
+
+		File directory = createTempDirectory("testCloneRepositoryWithShallowExclude");
+		CloneCommand command = Git.cloneRepository();
+		command.setDirectory(directory);
+		command.setURI(fileUri());
+		command.addShallowExclude(parent.getId());
+		command.setBranchesToClone(Set.of("refs/heads/test"));
+		Git git2 = command.call();
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(1, log.size());
+		RevCommit commit = log.get(0);
+		assertEquals(Set.of(commit.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals("Third commit", commit.getFullMessage());
+		assertEquals(0, commit.getParentCount());
+	}
+
 	private void assertTagOption(Repository repo, TagOpt expectedTagOption)
 			throws URISyntaxException {
 		RemoteConfig remoteConfig = new RemoteConfig(
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java
index c0db83a820..b2a4af30ab 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018, Google LLC. and others
+ * Copyright (C) 2018, 2022 Google LLC. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -15,6 +15,7 @@ import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
 import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
 import static org.eclipse.jgit.transport.ObjectIdMatcher.hasOnlyObjectIds;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -133,6 +134,42 @@ public class ProtocolV0ParserTest {
 	}
 
 	@Test
+	public void testRecvWantsDeepenSince()
+			throws PackProtocolException, IOException {
+		PacketLineIn pckIn = formatAsPacketLine(
+				"want 4624442d68ee402a94364191085b77137618633e\n",
+				"want f900c8326a43303685c46b279b9f70411bff1a4b\n",
+				"deepen-since 1652773020\n",
+				PacketLineIn.end());
+		ProtocolV0Parser parser = new ProtocolV0Parser(defaultConfig());
+		FetchV0Request request = parser.recvWants(pckIn);
+		assertTrue(request.getClientCapabilities().isEmpty());
+		assertEquals(1652773020, request.getDeepenSince());
+		assertThat(request.getWantIds(),
+				   hasOnlyObjectIds("4624442d68ee402a94364191085b77137618633e",
+									"f900c8326a43303685c46b279b9f70411bff1a4b"));
+	}
+
+	@Test
+	public void testRecvWantsDeepenNots()
+			throws PackProtocolException, IOException {
+		PacketLineIn pckIn = formatAsPacketLine(
+				"want 4624442d68ee402a94364191085b77137618633e\n",
+				"want f900c8326a43303685c46b279b9f70411bff1a4b\n",
+				"deepen-not 856d5138d7269a483efe276d4a6b5c25b4fbb1a4\n",
+				"deepen-not heads/refs/test\n",
+				PacketLineIn.end());
+		ProtocolV0Parser parser = new ProtocolV0Parser(defaultConfig());
+		FetchV0Request request = parser.recvWants(pckIn);
+		assertTrue(request.getClientCapabilities().isEmpty());
+		assertThat(request.getDeepenNots(), contains("856d5138d7269a483efe276d4a6b5c25b4fbb1a4",
+													 "heads/refs/test"));
+		assertThat(request.getWantIds(),
+				   hasOnlyObjectIds("4624442d68ee402a94364191085b77137618633e",
+									"f900c8326a43303685c46b279b9f70411bff1a4b"));
+	}
+
+	@Test
 	public void testRecvWantsShallow()
 			throws PackProtocolException, IOException {
 		PacketLineIn pckIn = formatAsPacketLine(
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java
index 837bdce919..167b5b72c6 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018, Google LLC. and others
+ * Copyright (C) 2018, 2022 Google LLC. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -152,7 +152,7 @@ public class ProtocolV2ParserTest {
 		assertThat(request.getClientShallowCommits(),
 				hasOnlyObjectIds("28274d02c489f4c7e68153056e9061a46f62d7a0",
 						"145e683b229dcab9d0e2ccb01b386f9ecc17d29d"));
-		assertTrue(request.getDeepenNotRefs().isEmpty());
+		assertTrue(request.getDeepenNots().isEmpty());
 		assertEquals(15, request.getDepth());
 		assertTrue(request.getClientCapabilities()
 				.contains(GitProtocolConstants.OPTION_DEEPEN_RELATIVE));
@@ -171,7 +171,7 @@ public class ProtocolV2ParserTest {
 		assertThat(request.getClientShallowCommits(),
 				hasOnlyObjectIds("28274d02c489f4c7e68153056e9061a46f62d7a0",
 						"145e683b229dcab9d0e2ccb01b386f9ecc17d29d"));
-		assertThat(request.getDeepenNotRefs(),
+		assertThat(request.getDeepenNots(),
 				hasItems("a08595f76159b09d57553e37a5123f1091bb13e7"));
 	}