GC: Write commit-graph files when gc

If 'core.commitGraph' and 'gc.writeCommitGraph' are both true, then gc
will rewrite the commit-graph file when 'git gc' is run. Defaults to
false while the commit-graph feature matures.

Bug: 574368
Change-Id: Ic94cd69034c524285c938414610f2e152198e06e
Signed-off-by: kylezhao <kylezhao@tencent.com>

4 files changed, 268 insertions, 0 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java
new file mode 100644
index 0000000000..0a0d85c8aa
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2022, Tencent.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Collections;
+
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.util.IO;
+import org.junit.Test;
+
+public class GcCommitGraphTest extends GcTestCase {
+
+	@Test
+	public void testCommitGraphConfig() {
+		StoredConfig config = repo.getConfig();
+		assertFalse(gc.shouldWriteCommitGraphWhenGc());
+
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
+		assertTrue(gc.shouldWriteCommitGraphWhenGc());
+
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
+		assertFalse(gc.shouldWriteCommitGraphWhenGc());
+	}
+
+	@Test
+	public void testWriteEmptyRepo() throws Exception {
+		StoredConfig config = repo.getConfig();
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, true);
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
+
+		assertTrue(gc.shouldWriteCommitGraphWhenGc());
+		gc.writeCommitGraph(Collections.emptySet());
+		File graphFile = new File(repo.getObjectsDirectory(),
+				Constants.INFO_COMMIT_GRAPH);
+		assertFalse(graphFile.exists());
+	}
+
+	@Test
+	public void testWriteWhenGc() throws Exception {
+		StoredConfig config = repo.getConfig();
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, true);
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
+
+		RevCommit tip = commitChain(10);
+		TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
+		bb.update(tip);
+
+		assertTrue(gc.shouldWriteCommitGraphWhenGc());
+		gc.gc();
+		File graphFile = new File(repo.getObjectsDirectory(),
+				Constants.INFO_COMMIT_GRAPH);
+		assertGraphFile(graphFile);
+	}
+
+	@Test
+	public void testDefaultWriteWhenGc() throws Exception {
+		RevCommit tip = commitChain(10);
+		TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
+		bb.update(tip);
+
+		assertFalse(gc.shouldWriteCommitGraphWhenGc());
+		gc.gc();
+		File graphFile = new File(repo.getObjectsDirectory(),
+				Constants.INFO_COMMIT_GRAPH);
+		assertFalse(graphFile.exists());
+	}
+
+	@Test
+	public void testDisableWriteWhenGc() throws Exception {
+		RevCommit tip = commitChain(10);
+		TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
+		bb.update(tip);
+		File graphFile = new File(repo.getObjectsDirectory(),
+				Constants.INFO_COMMIT_GRAPH);
+
+		StoredConfig config = repo.getConfig();
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, false);
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
+
+		gc.gc();
+		assertFalse(graphFile.exists());
+
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, true);
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
+		gc.gc();
+		assertFalse(graphFile.exists());
+
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, false);
+		config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
+		gc.gc();
+		assertFalse(graphFile.exists());
+	}
+
+	@Test
+	public void testWriteCommitGraphOnly() throws Exception {
+		RevCommit tip = commitChain(10);
+		TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
+		bb.update(tip);
+
+		StoredConfig config = repo.getConfig();
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, false);
+		gc.writeCommitGraph(Collections.singleton(tip));
+
+		File graphFile = new File(repo.getObjectsDirectory(),
+				Constants.INFO_COMMIT_GRAPH);
+		assertFalse(graphFile.exists());
+
+		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_COMMIT_GRAPH, true);
+		gc.writeCommitGraph(Collections.singleton(tip));
+		assertGraphFile(graphFile);
+	}
+
+	private void assertGraphFile(File graphFile) throws Exception {
+		assertTrue(graphFile.exists());
+		try (InputStream os = new FileInputStream(graphFile)) {
+			byte[] magic = new byte[4];
+			IO.readFully(os, magic, 0, 4);
+			assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H' }, magic);
+		}
+	}
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index d81f8ee49c..c9ecebe80a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -63,10 +63,13 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.NoWorkTreeException;
 import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter;
+import org.eclipse.jgit.internal.storage.commitgraph.GraphCommits;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.internal.storage.pack.PackWriter;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.CoreConfig;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.ObjectId;
@@ -121,6 +124,8 @@ public class GC {
 
 	private static final int DEFAULT_AUTOLIMIT = 6700;
 
+	private static final boolean DEFAULT_WRITE_COMMIT_GRAPH = false;
+
 	private static volatile ExecutorService executor;
 
 	/**
@@ -266,6 +271,9 @@ public class GC {
 		Collection<Pack> newPacks = repack();
 		prune(Collections.emptySet());
 		// TODO: implement rerere_gc(pm);
+		if (shouldWriteCommitGraphWhenGc()) {
+			writeCommitGraph(refsToObjectIds(getAllRefs()));
+		}
 		return newPacks;
 	}
 
@@ -875,6 +883,102 @@ public class GC {
 		return ret;
 	}
 
+	private Set<ObjectId> refsToObjectIds(Collection<Ref> refs)
+			throws IOException {
+		Set<ObjectId> objectIds = new HashSet<>();
+		for (Ref ref : refs) {
+			checkCancelled();
+			if (ref.getPeeledObjectId() != null) {
+				objectIds.add(ref.getPeeledObjectId());
+				continue;
+			}
+
+			if (ref.getObjectId() != null) {
+				objectIds.add(ref.getObjectId());
+			}
+		}
+		return objectIds;
+	}
+
+	/**
+	 * Generate a new commit-graph file when 'core.commitGraph' is true.
+	 *
+	 * @param wants
+	 *            the list of wanted objects, writer walks commits starting at
+	 *            these. Must not be {@code null}.
+	 * @throws IOException
+	 */
+	void writeCommitGraph(@NonNull Set<? extends ObjectId> wants)
+			throws IOException {
+		if (!repo.getConfig().get(CoreConfig.KEY).enableCommitGraph()) {
+			return;
+		}
+		checkCancelled();
+		if (wants.isEmpty()) {
+			return;
+		}
+		File tmpFile = null;
+		try (RevWalk walk = new RevWalk(repo)) {
+			CommitGraphWriter writer = new CommitGraphWriter(
+					GraphCommits.fromWalk(pm, wants, walk));
+			tmpFile = File.createTempFile("commit_", ".graph_tmp", //$NON-NLS-1$//$NON-NLS-2$
+					repo.getObjectDatabase().getInfoDirectory());
+			// write the commit-graph file
+			try (FileOutputStream fos = new FileOutputStream(tmpFile);
+					FileChannel channel = fos.getChannel();
+					OutputStream channelStream = Channels
+							.newOutputStream(channel)) {
+				writer.write(pm, channelStream);
+				channel.force(true);
+			}
+
+			// rename the temporary file to real file
+			File realFile = new File(repo.getObjectsDirectory(),
+					Constants.INFO_COMMIT_GRAPH);
+			FileUtils.rename(tmpFile, realFile, StandardCopyOption.ATOMIC_MOVE);
+		} finally {
+			if (tmpFile != null && tmpFile.exists()) {
+				tmpFile.delete();
+			}
+		}
+		deleteTempCommitGraph();
+	}
+
+	private void deleteTempCommitGraph() {
+		Path objectsDir = repo.getObjectDatabase().getInfoDirectory().toPath();
+		Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
+		if (!Files.exists(objectsDir)) {
+			return;
+		}
+		try (DirectoryStream<Path> stream = Files.newDirectoryStream(objectsDir,
+				"commit_*_tmp")) { //$NON-NLS-1$
+			stream.forEach(t -> {
+				try {
+					Instant lastModified = Files.getLastModifiedTime(t)
+							.toInstant();
+					if (lastModified.isBefore(threshold)) {
+						Files.deleteIfExists(t);
+					}
+				} catch (IOException e) {
+					LOG.error(e.getMessage(), e);
+				}
+			});
+		} catch (IOException e) {
+			LOG.error(e.getMessage(), e);
+		}
+	}
+
+	/**
+	 * If {@code true}, will rewrite the commit-graph file when gc is run.
+	 *
+	 * @return true if commit-graph should be writen. Default is {@code false}.
+	 */
+	boolean shouldWriteCommitGraphWhenGc() {
+		return repo.getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION,
+				ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH,
+				DEFAULT_WRITE_COMMIT_GRAPH);
+	}
+
 	private static boolean isHead(Ref ref) {
 		return ref.getName().startsWith(Constants.R_HEADS);
 	}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
index ff7ef93278..f7ccceca47 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
@@ -805,4 +805,8 @@ public class ObjectDirectory extends FileObjectDatabase {
 	AlternateHandle.Id getAlternateId() {
 		return handle.getId();
 	}
+
+	File getInfoDirectory() {
+		return infoDirectory;
+	}
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
index c166abe128..0b8bf8c6c5 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
@@ -284,6 +284,12 @@ public final class Constants {
 	 */
 	public static final String INFO_HTTP_ALTERNATES = "info/http-alternates";
 
+	/**
+	 * info commit-graph file (goes under OBJECTS)
+	 * @since 6.5
+	 */
+	public static final String INFO_COMMIT_GRAPH = "info/commit-graph";
+
 	/** Packed refs file */
 	public static final String PACKED_REFS = "packed-refs";