Teach JGit to write a commit-graph formatted file by walking commit graph from specified commit objects. See: https://git-scm.com/docs/commit-graph-format/2.19.0 Bug: 574368 Change-Id: I34f9f28f8729080c275f86215ebf30b2d05af41d Signed-off-by: kylezhao <kylezhao@tencent.com>changes/32/182832/5
@@ -35,6 +35,7 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", | |||
org.eclipse.jgit.internal;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.fsck;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.revwalk;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.storage.commitgraph;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.storage.dfs;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.storage.file;version="[5.13.0,5.14.0)", | |||
org.eclipse.jgit.internal.storage.io;version="[5.13.0,5.14.0)", |
@@ -0,0 +1,113 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.junit.Assert.assertEquals; | |||
import static org.junit.Assert.assertFalse; | |||
import static org.junit.Assert.assertTrue; | |||
import java.io.ByteArrayOutputStream; | |||
import java.util.Collections; | |||
import java.util.Set; | |||
import org.eclipse.jgit.internal.storage.file.FileRepository; | |||
import org.eclipse.jgit.junit.RepositoryTestCase; | |||
import org.eclipse.jgit.junit.TestRepository; | |||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.revwalk.RevCommit; | |||
import org.eclipse.jgit.revwalk.RevWalk; | |||
import org.junit.Before; | |||
import org.junit.Test; | |||
public class CommitGraphWriterTest extends RepositoryTestCase { | |||
private TestRepository<FileRepository> tr; | |||
private CommitGraphConfig config; | |||
private ByteArrayOutputStream os; | |||
private CommitGraphWriter writer; | |||
@Override | |||
@Before | |||
public void setUp() throws Exception { | |||
super.setUp(); | |||
os = new ByteArrayOutputStream(); | |||
config = new CommitGraphConfig(db); | |||
tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); | |||
} | |||
@Test | |||
public void testConstructor() { | |||
writer = new CommitGraphWriter(config, db.newObjectReader()); | |||
assertTrue(config.isComputeGeneration()); | |||
assertTrue(writer.isComputeGeneration()); | |||
assertEquals(0, writer.getCommitCnt()); | |||
} | |||
@Test | |||
public void testModifySettings() { | |||
config.setComputeGeneration(false); | |||
assertFalse(config.isComputeGeneration()); | |||
writer = new CommitGraphWriter(config, db.newObjectReader()); | |||
assertFalse(writer.isComputeGeneration()); | |||
writer.setComputeGeneration(true); | |||
assertTrue(writer.isComputeGeneration()); | |||
} | |||
@Test | |||
public void testWriterWithExtraEdgeList() throws Exception { | |||
RevCommit root = commit(); | |||
RevCommit a = commit(root); | |||
RevCommit b = commit(root); | |||
RevCommit c = commit(root); | |||
RevCommit tip = commit(a, b, c); | |||
Set<ObjectId> wants = Collections.singleton(tip); | |||
NullProgressMonitor m = NullProgressMonitor.INSTANCE; | |||
writer = new CommitGraphWriter(config, db.newObjectReader()); | |||
writer.prepareCommitGraph(m, m, wants); | |||
assertTrue(writer.willWriteExtraEdgeList()); | |||
assertEquals(5, writer.getCommitCnt()); | |||
writer.writeCommitGraph(m, os); | |||
byte[] data = os.toByteArray(); | |||
assertTrue(data.length > 0); | |||
} | |||
@Test | |||
public void testWriterWithoutExtraEdgeList() throws Exception { | |||
RevCommit root = commit(); | |||
RevCommit a = commit(root); | |||
RevCommit b = commit(root); | |||
RevCommit tip = commit(a, b); | |||
Set<ObjectId> wants = Collections.singleton(tip); | |||
NullProgressMonitor m = NullProgressMonitor.INSTANCE; | |||
writer = new CommitGraphWriter(config, db.newObjectReader()); | |||
writer.prepareCommitGraph(m, m, wants); | |||
assertFalse(writer.willWriteExtraEdgeList()); | |||
assertEquals(4, writer.getCommitCnt()); | |||
writer.writeCommitGraph(m, os); | |||
byte[] data = os.toByteArray(); | |||
assertTrue(data.length > 0); | |||
} | |||
RevCommit commit(RevCommit... parents) throws Exception { | |||
return tr.commit(parents); | |||
} | |||
} |
@@ -74,6 +74,8 @@ Export-Package: org.eclipse.jgit.annotations;version="5.13.0", | |||
x-friends:="org.eclipse.jgit.test", | |||
org.eclipse.jgit.internal.revwalk;version="5.13.0"; | |||
x-friends:="org.eclipse.jgit.test", | |||
org.eclipse.jgit.internal.storage.commitgraph;version="5.13.0"; | |||
x-friends:="org.eclipse.jgit.test", | |||
org.eclipse.jgit.internal.storage.dfs;version="5.13.0"; | |||
x-friends:="org.eclipse.jgit.test, | |||
org.eclipse.jgit.http.server, |
@@ -141,11 +141,13 @@ collisionOn=Collision on {0} | |||
commandClosedStderrButDidntExit=Command {0} closed stderr stream but didn''t exit within timeout {1} seconds | |||
commandRejectedByHook=Rejected by "{0}" hook.\n{1} | |||
commandWasCalledInTheWrongState=Command {0} was called in the wrong state | |||
commitGraphGeneratingCancelledDuringWriting=commit-graph generating cancelled during writing | |||
commitMessageNotSpecified=commit message not specified | |||
commitOnRepoWithoutHEADCurrentlyNotSupported=Commit on repo without HEAD currently not supported | |||
commitAmendOnInitialNotPossible=Amending is not possible on initial commit. | |||
commitsHaveAlreadyBeenMarkedAsStart=Commits have already been marked as walk starts. | |||
compressingObjects=Compressing objects | |||
computingCommitGeneration=Computing commit graph generation numbers | |||
configSubsectionContainsNewline=config subsection name contains newline | |||
configSubsectionContainsNullByte=config subsection name contains byte 0x00 | |||
configValueContainsNullByte=config value contains byte 0x00 | |||
@@ -318,6 +320,7 @@ fileModeNotSetForPath=FileMode not set for path {0} | |||
filterExecutionFailed=Execution of filter command ''{0}'' on file ''{1}'' failed | |||
filterExecutionFailedRc=Execution of filter command ''{0}'' on file ''{1}'' failed with return code ''{2}'', message on stderr: ''{3}'' | |||
filterRequiresCapability=filter requires server to advertise that capability | |||
findingCommitsForCommitGraph=Finding commits for commit graph | |||
findingGarbage=Finding garbage | |||
flagIsDisposed={0} is disposed. | |||
flagNotFromThis={0} not from this. | |||
@@ -808,6 +811,7 @@ writerAlreadyInitialized=Writer already initialized | |||
writeTimedOut=Write timed out after {0} ms | |||
writingNotPermitted=Writing not permitted | |||
writingNotSupported=Writing {0} not supported. | |||
writingOutCommitGraph=Writing out commit graph in {0} passes | |||
writingObjects=Writing objects | |||
wrongDecompressedLength=wrong decompressed length | |||
wrongRepositoryState=Wrong Repository State: {0} |
@@ -169,11 +169,13 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String commandClosedStderrButDidntExit; | |||
/***/ public String commandRejectedByHook; | |||
/***/ public String commandWasCalledInTheWrongState; | |||
/***/ public String commitGraphGeneratingCancelledDuringWriting; | |||
/***/ public String commitMessageNotSpecified; | |||
/***/ public String commitOnRepoWithoutHEADCurrentlyNotSupported; | |||
/***/ public String commitAmendOnInitialNotPossible; | |||
/***/ public String commitsHaveAlreadyBeenMarkedAsStart; | |||
/***/ public String compressingObjects; | |||
/***/ public String computingCommitGeneration; | |||
/***/ public String configSubsectionContainsNewline; | |||
/***/ public String configSubsectionContainsNullByte; | |||
/***/ public String configValueContainsNullByte; | |||
@@ -346,6 +348,7 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String filterExecutionFailed; | |||
/***/ public String filterExecutionFailedRc; | |||
/***/ public String filterRequiresCapability; | |||
/***/ public String findingCommitsForCommitGraph; | |||
/***/ public String findingGarbage; | |||
/***/ public String flagIsDisposed; | |||
/***/ public String flagNotFromThis; | |||
@@ -836,6 +839,7 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String writeTimedOut; | |||
/***/ public String writingNotPermitted; | |||
/***/ public String writingNotSupported; | |||
/***/ public String writingOutCommitGraph; | |||
/***/ public String writingObjects; | |||
/***/ public String wrongDecompressedLength; | |||
/***/ public String wrongRepositoryState; |
@@ -0,0 +1,99 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_COMMIT_GRAPH_SECTION; | |||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_COMPUTE_GENERATION; | |||
import org.eclipse.jgit.lib.Config; | |||
import org.eclipse.jgit.lib.Repository; | |||
/** | |||
* Configuration used by a commit-graph writer when constructing the stream. | |||
*/ | |||
public class CommitGraphConfig { | |||
/** | |||
* Default value of compute generation option: {@value} | |||
* | |||
* @see #setComputeGeneration(boolean) | |||
*/ | |||
public static final boolean DEFAULT_COMPUTE_GENERATION = true; | |||
private boolean computeGeneration = DEFAULT_COMPUTE_GENERATION; | |||
/** | |||
* Create a default configuration. | |||
*/ | |||
public CommitGraphConfig() { | |||
} | |||
/** | |||
* Create a configuration honoring the repository's settings. | |||
* | |||
* @param db | |||
* the repository to read settings from. The repository is not | |||
* retained by the new configuration, instead its settings are | |||
* copied during the constructor. | |||
*/ | |||
public CommitGraphConfig(Repository db) { | |||
fromConfig(db.getConfig()); | |||
} | |||
/** | |||
* Create a configuration honoring settings in a | |||
* {@link org.eclipse.jgit.lib.Config}. | |||
* | |||
* @param cfg | |||
* the source to read settings from. The source is not retained | |||
* by the new configuration, instead its settings are copied | |||
* during the constructor. | |||
*/ | |||
public CommitGraphConfig(Config cfg) { | |||
fromConfig(cfg); | |||
} | |||
/** | |||
* Checks whether to compute generation numbers. | |||
* | |||
* @return {@code true} if the writer should compute generation numbers. | |||
*/ | |||
public boolean isComputeGeneration() { | |||
return computeGeneration; | |||
} | |||
/** | |||
* Whether the writer should compute generation numbers. | |||
* | |||
* Default setting: {@value #DEFAULT_COMPUTE_GENERATION} | |||
* | |||
* @param computeGeneration | |||
* if {@code true} the commit-graph will include the computed | |||
* generation numbers. | |||
*/ | |||
public void setComputeGeneration(boolean computeGeneration) { | |||
this.computeGeneration = computeGeneration; | |||
} | |||
/** | |||
* Update properties by setting fields from the configuration. | |||
* | |||
* If a property's corresponding variable is not defined in the supplied | |||
* configuration, then it is left unmodified. | |||
* | |||
* @param rc | |||
* configuration to read properties from. | |||
*/ | |||
public void fromConfig(Config rc) { | |||
computeGeneration = rc.getBoolean(CONFIG_COMMIT_GRAPH_SECTION, | |||
CONFIG_KEY_COMPUTE_GENERATION, computeGeneration); | |||
} | |||
} |
@@ -0,0 +1,46 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
/** | |||
* Constants relating to commit-graph. | |||
*/ | |||
class CommitGraphConstants { | |||
static final int COMMIT_GRAPH_MAGIC = 0x43475048; /* "CGPH" */ | |||
static final int CHUNK_ID_OID_FANOUT = 0x4f494446; /* "OIDF" */ | |||
static final int CHUNK_ID_OID_LOOKUP = 0x4f49444c; /* "OIDL" */ | |||
static final int CHUNK_ID_COMMIT_DATA = 0x43444154; /* "CDAT" */ | |||
static final int CHUNK_ID_EXTRA_EDGE_LIST = 0x45444745; /* "EDGE" */ | |||
static final int GRAPH_CHUNK_LOOKUP_WIDTH = 12; | |||
static final int COMMIT_DATA_EXTRA_LENGTH = 16; | |||
/** Mask to make the last edgeValue into position */ | |||
static final int GRAPH_EDGE_LAST_MASK = 0x7fffffff; | |||
/** EdgeValue & GRAPH_LAST_EDGE != 0 means it is the last edgeValue */ | |||
static final int GRAPH_LAST_EDGE = 0x80000000; | |||
/** EdgeValue == GRAPH_NO_PARENT means it has no parents */ | |||
static final int GRAPH_NO_PARENT = 0x70000000; | |||
/** | |||
* EdgeValue & GRAPH_EXTRA_EDGES_NEEDED != 0 means it's other parents are in | |||
* Chunk Extra Edge List | |||
*/ | |||
static final int GRAPH_EXTRA_EDGES_NEEDED = 0x80000000; | |||
} |
@@ -0,0 +1,113 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_GRAPH_MAGIC; | |||
import java.io.IOException; | |||
import java.io.OutputStream; | |||
import java.security.MessageDigest; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.lib.Constants; | |||
import org.eclipse.jgit.lib.ProgressMonitor; | |||
import org.eclipse.jgit.util.NB; | |||
/** | |||
* Custom output stream to support | |||
* {@link org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter}. | |||
*/ | |||
public class CommitGraphOutPutStream extends OutputStream { | |||
private static final int BYTES_TO_WRITE_BEFORE_CANCEL_CHECK = 128 * 1024; | |||
private final ProgressMonitor writeMonitor; | |||
private final OutputStream out; | |||
private final MessageDigest md = Constants.newMessageDigest(); | |||
private long count; | |||
private long checkCancelAt; | |||
private final byte[] headerBuffer = new byte[16]; | |||
/** | |||
* Initialize a commit-graph output stream. | |||
* | |||
* @param writeMonitor | |||
* monitor to update on output progress. | |||
* @param out | |||
* target stream to receive all contents. | |||
*/ | |||
public CommitGraphOutPutStream(ProgressMonitor writeMonitor, | |||
OutputStream out) { | |||
this.writeMonitor = writeMonitor; | |||
this.out = out; | |||
this.checkCancelAt = BYTES_TO_WRITE_BEFORE_CANCEL_CHECK; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public final void write(int b) throws IOException { | |||
count++; | |||
out.write(b); | |||
md.update((byte) b); | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public final void write(byte[] b, int off, int len) throws IOException { | |||
while (0 < len) { | |||
int n = Math.min(len, BYTES_TO_WRITE_BEFORE_CANCEL_CHECK); | |||
count += n; | |||
if (checkCancelAt <= count) { | |||
if (writeMonitor.isCancelled()) { | |||
throw new IOException(JGitText | |||
.get().commitGraphGeneratingCancelledDuringWriting); | |||
} | |||
checkCancelAt = count + BYTES_TO_WRITE_BEFORE_CANCEL_CHECK; | |||
} | |||
out.write(b, off, n); | |||
md.update(b, off, n); | |||
off += n; | |||
len -= n; | |||
} | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public void flush() throws IOException { | |||
out.flush(); | |||
} | |||
void writeFileHeader(int version, int hashVersion, int chunksNumber) | |||
throws IOException { | |||
NB.encodeInt32(headerBuffer, 0, COMMIT_GRAPH_MAGIC); | |||
byte[] buff = { (byte) version, (byte) hashVersion, (byte) chunksNumber, | |||
(byte) 0 }; | |||
System.arraycopy(buff, 0, headerBuffer, 4, 4); | |||
write(headerBuffer, 0, 8); | |||
} | |||
/** @return obtain the current SHA-1 digest. */ | |||
byte[] getDigest() { | |||
return md.digest(); | |||
} | |||
void updateMonitor() { | |||
writeMonitor.update(1); | |||
} | |||
} |
@@ -0,0 +1,517 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_DATA_EXTRA_LENGTH; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_CHUNK_LOOKUP_WIDTH; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EXTRA_EDGES_NEEDED; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_LAST_EDGE; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_NO_PARENT; | |||
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; | |||
import java.io.IOException; | |||
import java.io.OutputStream; | |||
import java.nio.ByteBuffer; | |||
import java.text.MessageFormat; | |||
import java.util.Collections; | |||
import java.util.List; | |||
import java.util.Set; | |||
import java.util.Stack; | |||
import org.eclipse.jgit.annotations.NonNull; | |||
import org.eclipse.jgit.errors.MissingObjectException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.Constants; | |||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.lib.ObjectIdOwnerMap; | |||
import org.eclipse.jgit.lib.ObjectReader; | |||
import org.eclipse.jgit.lib.ProgressMonitor; | |||
import org.eclipse.jgit.lib.Repository; | |||
import org.eclipse.jgit.revwalk.ObjectWalk; | |||
import org.eclipse.jgit.revwalk.RevCommit; | |||
import org.eclipse.jgit.revwalk.RevObject; | |||
import org.eclipse.jgit.revwalk.RevSort; | |||
import org.eclipse.jgit.util.BlockList; | |||
import org.eclipse.jgit.util.NB; | |||
/** | |||
* Writes a commit-graph formatted file. | |||
*/ | |||
public class CommitGraphWriter { | |||
private static final int COMMIT_GRAPH_VERSION_GENERATED = 1; | |||
private static final int OID_HASH_VERSION = 1; | |||
private static final int GENERATION_NUMBER_MAX = 0x3FFFFFFF; | |||
private static final int MAX_NUM_CHUNKS = 5; | |||
private static final int GRAPH_FANOUT_SIZE = 4 * 256; | |||
private final ObjectWalk walk; | |||
private List<ObjectToCommitData> commitDataList = new BlockList<>(); | |||
private List<ObjectToCommitData> commitDataSortedByName; | |||
private ObjectIdOwnerMap<ObjectToCommitData> commitDataMap = new ObjectIdOwnerMap<>(); | |||
private int numExtraEdges; | |||
private boolean computeGeneration; | |||
/** | |||
* Create writer for specified repository. | |||
* | |||
* @param repo | |||
* repository where objects are stored. | |||
*/ | |||
public CommitGraphWriter(Repository repo) { | |||
this(repo, repo.newObjectReader()); | |||
} | |||
/** | |||
* Create writer for specified repository. | |||
* | |||
* @param repo | |||
* repository where objects are stored. | |||
* @param reader | |||
* reader to read from the repository with. | |||
*/ | |||
public CommitGraphWriter(Repository repo, ObjectReader reader) { | |||
this(new CommitGraphConfig(repo), reader); | |||
} | |||
/** | |||
* Create writer with a specified configuration. | |||
* | |||
* @param cfg | |||
* configuration for the commit-graph writer. | |||
* @param reader | |||
* reader to read from the repository with. | |||
*/ | |||
public CommitGraphWriter(CommitGraphConfig cfg, ObjectReader reader) { | |||
this.walk = new ObjectWalk(reader); | |||
this.computeGeneration = cfg.isComputeGeneration(); | |||
} | |||
/** | |||
* Prepare the list of commits to be written to the commit-graph stream. | |||
* | |||
* @param findingMonitor | |||
* progress monitor to report the number of commits found. | |||
* @param computeGenerationMonitor | |||
* progress monitor to report generation computation work. | |||
* @param wants | |||
* the list of wanted objects, writer walks commits starting at | |||
* these. Must not be {@code null}. | |||
* @throws IOException | |||
*/ | |||
public void prepareCommitGraph(ProgressMonitor findingMonitor, | |||
ProgressMonitor computeGenerationMonitor, | |||
@NonNull Set<? extends ObjectId> wants) throws IOException { | |||
BlockList<RevCommit> commits = findCommits(findingMonitor, wants); | |||
if (computeGeneration) { | |||
computeGenerationNumbers(computeGenerationMonitor, commits); | |||
} | |||
} | |||
/** | |||
* Write the prepared commits to the supplied stream. | |||
* | |||
* @param writeMonitor | |||
* progress monitor to report the number of items written. | |||
* @param commitGraphStream | |||
* output stream of commit-graph data. The stream should be | |||
* buffered by the caller. The caller is responsible for closing | |||
* the stream. | |||
* @throws IOException | |||
*/ | |||
public void writeCommitGraph(ProgressMonitor writeMonitor, | |||
OutputStream commitGraphStream) throws IOException { | |||
if (writeMonitor == null) { | |||
writeMonitor = NullProgressMonitor.INSTANCE; | |||
} | |||
ChunkInfo[] chunks = new ChunkInfo[MAX_NUM_CHUNKS]; | |||
for (int i = 0; i < chunks.length; i++) { | |||
chunks[i] = new ChunkInfo(); | |||
} | |||
int numChunks = 3; | |||
int hashsz = OBJECT_ID_LENGTH; | |||
long writeCount = 0; | |||
long chunkOffset; | |||
CommitGraphOutPutStream out = new CommitGraphOutPutStream(writeMonitor, | |||
commitGraphStream); | |||
chunks[0].id = CHUNK_ID_OID_FANOUT; | |||
chunks[0].size = GRAPH_FANOUT_SIZE; | |||
writeCount += 256; | |||
chunks[1].id = CHUNK_ID_OID_LOOKUP; | |||
chunks[1].size = hashsz * commitDataList.size(); | |||
writeCount += commitDataList.size(); | |||
chunks[2].id = CHUNK_ID_COMMIT_DATA; | |||
chunks[2].size = (hashsz + 16) * commitDataList.size(); | |||
writeCount += commitDataList.size(); | |||
if (numExtraEdges > 0) { | |||
chunks[numChunks].id = CHUNK_ID_EXTRA_EDGE_LIST; | |||
chunks[numChunks].size = numExtraEdges * 4; | |||
writeCount += numExtraEdges; | |||
numChunks++; | |||
} | |||
chunks[numChunks].id = 0; | |||
chunks[numChunks].size = 0L; | |||
beginPhase(MessageFormat.format(JGitText.get().writingOutCommitGraph, | |||
Integer.valueOf(numChunks)), writeMonitor, writeCount); | |||
try { | |||
// write header | |||
out.writeFileHeader(getVersion(), OID_HASH_VERSION, numChunks); | |||
out.flush(); | |||
// write chunk lookup | |||
chunkOffset = 8 + (numChunks + 1) * GRAPH_CHUNK_LOOKUP_WIDTH; | |||
for (int i = 0; i <= numChunks; i++) { | |||
ChunkInfo chunk = chunks[i]; | |||
ByteBuffer buffer = ByteBuffer | |||
.allocate(GRAPH_CHUNK_LOOKUP_WIDTH); | |||
buffer.putInt(chunk.id); | |||
buffer.putLong(chunkOffset); | |||
out.write(buffer.array()); | |||
chunkOffset += chunk.size; | |||
} | |||
// write chunks | |||
for (int i = 0; i < numChunks; i++) { | |||
int chunkId = chunks[i].id; | |||
switch (chunkId) { | |||
case CHUNK_ID_OID_FANOUT: | |||
writeFanoutTable(out); | |||
break; | |||
case CHUNK_ID_OID_LOOKUP: | |||
writeOidLookUp(out, hashsz); | |||
break; | |||
case CHUNK_ID_COMMIT_DATA: | |||
writeCommitData(out, hashsz); | |||
break; | |||
case CHUNK_ID_EXTRA_EDGE_LIST: | |||
writeExtraEdges(out); | |||
break; | |||
} | |||
} | |||
// write check sum | |||
out.write(out.getDigest()); | |||
out.flush(); | |||
} finally { | |||
endPhase(writeMonitor); | |||
} | |||
} | |||
/** | |||
* Returns commits number that was created by this writer. | |||
* | |||
* @return number of commits. | |||
*/ | |||
public long getCommitCnt() { | |||
return commitDataList.size(); | |||
} | |||
/** | |||
* Whether to compute generation numbers. | |||
* | |||
* Default setting: {@value CommitGraphConfig#DEFAULT_COMPUTE_GENERATION} | |||
* | |||
* @return {@code true} if the writer should compute generation numbers. | |||
*/ | |||
public boolean isComputeGeneration() { | |||
return computeGeneration; | |||
} | |||
/** | |||
* Whether the writer should compute generation numbers. | |||
* | |||
* Default setting: {@value CommitGraphConfig#DEFAULT_COMPUTE_GENERATION} | |||
* | |||
* @param computeGeneration | |||
* if {@code true} the commits in commit-graph will have the | |||
* computed generation number. | |||
*/ | |||
public void setComputeGeneration(boolean computeGeneration) { | |||
this.computeGeneration = computeGeneration; | |||
} | |||
/** | |||
* Whether to write the extra edge list. | |||
* <p> | |||
* This list of 4-byte values store the second through nth parents for all | |||
* octopus merges. | |||
* | |||
* @return {@code true} if the writer will write the extra edge list. | |||
*/ | |||
public boolean willWriteExtraEdgeList() { | |||
return numExtraEdges > 0; | |||
} | |||
private void writeFanoutTable(CommitGraphOutPutStream out) | |||
throws IOException { | |||
byte[] tmp = new byte[4]; | |||
int[] fanout = new int[256]; | |||
for (ObjectToCommitData oc : commitDataList) { | |||
fanout[oc.getFirstByte() & 0xff]++; | |||
} | |||
for (int i = 1; i < fanout.length; i++) { | |||
fanout[i] += fanout[i - 1]; | |||
} | |||
for (int n : fanout) { | |||
NB.encodeInt32(tmp, 0, n); | |||
out.write(tmp, 0, 4); | |||
out.updateMonitor(); | |||
} | |||
} | |||
private void writeOidLookUp(CommitGraphOutPutStream out, int hashsz) | |||
throws IOException { | |||
byte[] tmp = new byte[4 + hashsz]; | |||
List<ObjectToCommitData> sortedByName = commitDataSortByName(); | |||
for (int i = 0; i < sortedByName.size(); i++) { | |||
ObjectToCommitData commitData = sortedByName.get(i); | |||
commitData.setOidPosition(i); | |||
commitData.copyRawTo(tmp, 0); | |||
out.write(tmp, 0, hashsz); | |||
out.updateMonitor(); | |||
} | |||
commitDataList = sortedByName; | |||
} | |||
private void writeCommitData(CommitGraphOutPutStream out, int hashsz) | |||
throws IOException { | |||
int num = 0; | |||
byte[] tmp = new byte[hashsz + COMMIT_DATA_EXTRA_LENGTH]; | |||
for (ObjectToCommitData oc : commitDataList) { | |||
int edgeValue; | |||
int[] packedDate = new int[2]; | |||
RevCommit commit = walk.parseCommit(oc); | |||
ObjectId treeId = commit.getTree(); | |||
treeId.copyRawTo(tmp, 0); | |||
RevCommit[] parents = commit.getParents(); | |||
if (parents.length == 0) { | |||
edgeValue = GRAPH_NO_PARENT; | |||
} else { | |||
RevCommit parent = parents[0]; | |||
edgeValue = getCommitOidPosition(parent); | |||
} | |||
NB.encodeInt32(tmp, hashsz, edgeValue); | |||
if (parents.length == 1) { | |||
edgeValue = GRAPH_NO_PARENT; | |||
} else if (parents.length == 2) { | |||
RevCommit parent = parents[1]; | |||
edgeValue = getCommitOidPosition(parent); | |||
} else if (parents.length > 2) { | |||
edgeValue = GRAPH_EXTRA_EDGES_NEEDED | num; | |||
num += parents.length - 1; | |||
} | |||
NB.encodeInt32(tmp, hashsz + 4, edgeValue); | |||
packedDate[0] = 0; // commitTime is an int in JGit now | |||
packedDate[0] |= oc.getGeneration() << 2; | |||
packedDate[1] = commit.getCommitTime(); | |||
NB.encodeInt32(tmp, hashsz + 8, packedDate[0]); | |||
NB.encodeInt32(tmp, hashsz + 12, packedDate[1]); | |||
out.write(tmp); | |||
out.updateMonitor(); | |||
} | |||
} | |||
private void writeExtraEdges(CommitGraphOutPutStream out) | |||
throws IOException { | |||
byte[] tmp = new byte[4]; | |||
for (ObjectToCommitData oc : commitDataList) { | |||
RevCommit commit = walk.parseCommit(oc); | |||
RevCommit[] parents = commit.getParents(); | |||
if (parents.length > 2) { | |||
int edgeValue; | |||
for (int n = 1; n < parents.length; n++) { | |||
RevCommit parent = parents[n]; | |||
edgeValue = getCommitOidPosition(parent); | |||
if (n == parents.length - 1) { | |||
edgeValue |= GRAPH_LAST_EDGE; | |||
} | |||
NB.encodeInt32(tmp, 0, edgeValue); | |||
out.write(tmp); | |||
out.updateMonitor(); | |||
} | |||
} | |||
} | |||
} | |||
private BlockList<RevCommit> findCommits(ProgressMonitor findingMonitor, | |||
Set<? extends ObjectId> wants) throws IOException { | |||
if (findingMonitor == null) { | |||
findingMonitor = NullProgressMonitor.INSTANCE; | |||
} | |||
for (ObjectId id : wants) { | |||
RevObject o = walk.parseAny(id); | |||
if (o instanceof RevCommit) { | |||
walk.markStart((RevCommit) o); | |||
} | |||
} | |||
walk.sort(RevSort.COMMIT_TIME_DESC); | |||
BlockList<RevCommit> commits = new BlockList<>(); | |||
RevCommit c; | |||
beginPhase(JGitText.get().findingCommitsForCommitGraph, findingMonitor, | |||
ProgressMonitor.UNKNOWN); | |||
while ((c = walk.next()) != null) { | |||
findingMonitor.update(1); | |||
commits.add(c); | |||
addCommitData(c); | |||
if (c.getParentCount() > 2) { | |||
numExtraEdges += c.getParentCount() - 1; | |||
} | |||
} | |||
endPhase(findingMonitor); | |||
return commits; | |||
} | |||
private void computeGenerationNumbers( | |||
ProgressMonitor computeGenerationMonitor, List<RevCommit> commits) | |||
throws MissingObjectException { | |||
if (computeGenerationMonitor == null) { | |||
computeGenerationMonitor = NullProgressMonitor.INSTANCE; | |||
} | |||
beginPhase(JGitText.get().computingCommitGeneration, | |||
computeGenerationMonitor, commits.size()); | |||
for (RevCommit cmit : commits) { | |||
computeGenerationMonitor.update(1); | |||
int generation = getCommitGeneration(cmit); | |||
if (generation != CommitGraph.GENERATION_NUMBER_ZERO | |||
&& generation != CommitGraph.GENERATION_NUMBER_INFINITY) { | |||
continue; | |||
} | |||
Stack<RevCommit> commitStack = new Stack<>(); | |||
commitStack.push(cmit); | |||
while (!commitStack.empty()) { | |||
int maxGeneration = 0; | |||
boolean allParentComputed = true; | |||
RevCommit current = commitStack.peek(); | |||
RevCommit parent; | |||
for (int i = 0; i < current.getParentCount(); i++) { | |||
parent = current.getParent(i); | |||
generation = getCommitGeneration(parent); | |||
if (generation == CommitGraph.GENERATION_NUMBER_ZERO | |||
|| generation == CommitGraph.GENERATION_NUMBER_INFINITY) { | |||
allParentComputed = false; | |||
commitStack.push(parent); | |||
break; | |||
} else if (generation > maxGeneration) { | |||
maxGeneration = generation; | |||
} | |||
} | |||
if (allParentComputed) { | |||
RevCommit commit = commitStack.pop(); | |||
generation = maxGeneration + 1; | |||
if (generation > GENERATION_NUMBER_MAX) { | |||
generation = GENERATION_NUMBER_MAX; | |||
} | |||
setCommitGeneration(commit, generation); | |||
} | |||
} | |||
} | |||
endPhase(computeGenerationMonitor); | |||
} | |||
private int getVersion() { | |||
return COMMIT_GRAPH_VERSION_GENERATED; | |||
} | |||
private static class ChunkInfo { | |||
int id; | |||
long size; | |||
} | |||
private int getCommitGeneration(RevCommit commit) | |||
throws MissingObjectException { | |||
ObjectToCommitData oc = commitDataMap.get(commit); | |||
if (oc == null) { | |||
throw new MissingObjectException(commit, Constants.OBJ_COMMIT); | |||
} | |||
return oc.getGeneration(); | |||
} | |||
private void setCommitGeneration(RevCommit commit, int generation) | |||
throws MissingObjectException { | |||
ObjectToCommitData oc = commitDataMap.get(commit); | |||
if (oc == null) { | |||
throw new MissingObjectException(commit, Constants.OBJ_COMMIT); | |||
} | |||
oc.setGeneration(generation); | |||
} | |||
private int getCommitOidPosition(RevCommit commit) | |||
throws MissingObjectException { | |||
ObjectToCommitData oc = commitDataMap.get(commit); | |||
if (oc == null) { | |||
throw new MissingObjectException(commit, Constants.OBJ_COMMIT); | |||
} | |||
return oc.getOidPosition(); | |||
} | |||
private void addCommitData(RevCommit commit) { | |||
ObjectToCommitData otc = new ObjectToCommitData(commit); | |||
commitDataList.add(otc); | |||
commitDataMap.add(otc); | |||
} | |||
private List<ObjectToCommitData> commitDataSortByName() { | |||
if (commitDataSortedByName == null) { | |||
commitDataSortedByName = new BlockList<>(commitDataList.size()); | |||
commitDataSortedByName.addAll(commitDataList); | |||
Collections.sort(commitDataSortedByName); | |||
} | |||
return commitDataSortedByName; | |||
} | |||
private void beginPhase(String task, ProgressMonitor monitor, long cnt) { | |||
monitor.beginTask(task, (int) cnt); | |||
} | |||
private void endPhase(ProgressMonitor monitor) { | |||
monitor.endTask(); | |||
} | |||
} |
@@ -0,0 +1,52 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import org.eclipse.jgit.lib.AnyObjectId; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.ObjectIdOwnerMap; | |||
/** | |||
* Per-object state used by | |||
* {@link org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter} | |||
*/ | |||
class ObjectToCommitData extends ObjectIdOwnerMap.Entry { | |||
private int generation = CommitGraph.GENERATION_NUMBER_ZERO; | |||
private int oidPosition = -1; | |||
/** | |||
* Initialize this entry with a specific ObjectId. | |||
* | |||
* @param id | |||
* the id the entry represents. | |||
*/ | |||
ObjectToCommitData(AnyObjectId id) { | |||
super(id); | |||
} | |||
int getGeneration() { | |||
return generation; | |||
} | |||
void setGeneration(int generation) { | |||
this.generation = generation; | |||
} | |||
int getOidPosition() { | |||
return oidPosition; | |||
} | |||
void setOidPosition(int oidPosition) { | |||
this.oidPosition = oidPosition; | |||
} | |||
} |
@@ -0,0 +1,132 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.lib; | |||
/** | |||
* The CommitGraph is a supplemental data structure that accelerates commit | |||
* graph walks. | |||
* <p> | |||
* If a user downgrades or disables the <code>core.commitGraph</code> config | |||
* setting, then the existing object database is sufficient. | |||
* </p> | |||
* <p> | |||
* It stores the commit graph structure along with some extra metadata to speed | |||
* up graph walks. By listing commit OIDs in lexicographic order, we can | |||
* identify an integer position for each commit and refer to the parents of a | |||
* commit using those integer positions. We use binary search to find initial | |||
* commits and then use the integer positions for fast lookups during the walk. | |||
* </p> | |||
* | |||
* @since 5.13 | |||
*/ | |||
public interface CommitGraph { | |||
/** | |||
* We use GENERATION_NUMBER_INFINITY(-1) to mark commits not in the | |||
* commit-graph file. | |||
*/ | |||
int GENERATION_NUMBER_INFINITY = -1; | |||
/** | |||
* If a commit-graph file was written by a version of Git that did not | |||
* compute generation numbers, then those commits will have generation | |||
* number represented by GENERATION_NUMBER_ZERO(0). | |||
*/ | |||
int GENERATION_NUMBER_ZERO = 0; | |||
/** | |||
* Get the metadata of a commit. | |||
* | |||
* @param commit | |||
* the commit object id to inspect. | |||
* @return the metadata of a commit or null if it's not found. | |||
*/ | |||
CommitData getCommitData(AnyObjectId commit); | |||
/** | |||
* Get the metadata of a commit。 | |||
* | |||
* @param graphPos | |||
* the position in the commit-graph of the object. | |||
* @return the metadata of a commit or null if it's not found. | |||
*/ | |||
CommitData getCommitData(int graphPos); | |||
/** | |||
* Get the object at the commit-graph position. | |||
* | |||
* @param graphPos | |||
* the position in the commit-graph of the object. | |||
* @return the ObjectId or null if it's not found. | |||
*/ | |||
ObjectId getObjectId(int graphPos); | |||
/** | |||
* Obtain the total number of commits described by this commit-graph. | |||
* | |||
* @return number of commits in this commit-graph | |||
*/ | |||
long getCommitCnt(); | |||
/** | |||
* Metadata of a commit in commit data chunk. | |||
*/ | |||
interface CommitData { | |||
/** | |||
* Get a reference to this commit's tree. | |||
* | |||
* @return tree of this commit. | |||
*/ | |||
ObjectId getTree(); | |||
/** | |||
* Obtain an array of all parents. | |||
* <p> | |||
* The method only provides the positions of parents in commit-graph, | |||
* call {@link CommitGraph#getObjectId(int)} to get the real objectId. | |||
* | |||
* @return the array of parents. | |||
*/ | |||
int[] getParents(); | |||
/** | |||
* Time from the "committer" line. | |||
* | |||
* @return commit time | |||
*/ | |||
long getCommitTime(); | |||
/** | |||
* Get the generation number of the commit. | |||
* <p> | |||
* If A and B are commits with generation numbers N and M, respectively, | |||
* and N <= M, then A cannot reach B. That is, we know without searching | |||
* that B is not an ancestor of A because it is further from a root | |||
* commit than A. | |||
* <p> | |||
* Conversely, when checking if A is an ancestor of B, then we only need | |||
* to walk commits until all commits on the walk boundary have | |||
* generation number at most N. If we walk commits using a priority | |||
* queue seeded by generation numbers, then we always expand the | |||
* boundary commit with highest generation number and can easily detect | |||
* the stopping condition. | |||
* <p> | |||
* We use {@value #GENERATION_NUMBER_INFINITY} to mark commits not in | |||
* the commit-graph file. If a commit-graph file was written without | |||
* computing generation numbers, then those commits will have generation | |||
* number represented by {@value #GENERATION_NUMBER_ZERO}. | |||
* | |||
* @return the generation number | |||
*/ | |||
int getGeneration(); | |||
} | |||
} |
@@ -1,7 +1,7 @@ | |||
/* | |||
* Copyright (C) 2010, Mathias Kinzler <mathias.kinzler@sap.com> | |||
* Copyright (C) 2010, Chris Aniszczyk <caniszczyk@gmail.com> | |||
* Copyright (C) 2012, 2020, Robin Rosenberg and others | |||
* Copyright (C) 2012, 2021, Robin Rosenberg and others | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
@@ -744,4 +744,18 @@ public final class ConfigConstants { | |||
*/ | |||
public static final String CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT = "searchforreusetimeout"; | |||
/** | |||
* The "commitGraph" section | |||
* | |||
* @since 5.13 | |||
*/ | |||
public static final String CONFIG_COMMIT_GRAPH_SECTION = "commitGraph"; | |||
/** | |||
* The "computeGeneration" key | |||
* | |||
* @since 5.13 | |||
*/ | |||
public static final String CONFIG_KEY_COMPUTE_GENERATION = "computeGeneration"; | |||
} |