This change teaches JGit to use a single commit-graph file to accelerate commit graph walks. When new commits are added to the repo, the commit-graph gets further and further behind, and then we write the whole commit-graph file again to include new commits. Multiple commit-graph files are not supported in the current implementation. With this feature, git will write a commit-graph after every git fetch command that downloads a pack-file from a remote. JGit can read a commit-graph file from a buffered stream, which means that we can provide this feature for both FileRepository and DfsRepository. Bug: 574368 Change-Id: Ib5c0d6678cb242870a0f5841bd413ad3885e95f6 Signed-off-by: kylezhao <kylezhao@tencent.com>changes/92/182892/2
@@ -0,0 +1,266 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.junit.Assert.assertArrayEquals; | |||
import static org.junit.Assert.assertEquals; | |||
import java.io.ByteArrayInputStream; | |||
import java.io.ByteArrayOutputStream; | |||
import java.io.InputStream; | |||
import java.util.Collections; | |||
import java.util.HashSet; | |||
import java.util.Set; | |||
import org.eclipse.jgit.internal.storage.file.FileRepository; | |||
import org.eclipse.jgit.junit.RepositoryTestCase; | |||
import org.eclipse.jgit.junit.TestRepository; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.ConfigConstants; | |||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.lib.StoredConfig; | |||
import org.eclipse.jgit.revwalk.RevCommit; | |||
import org.eclipse.jgit.revwalk.RevWalk; | |||
import org.junit.Before; | |||
import org.junit.Test; | |||
public class CommitGraphTest extends RepositoryTestCase { | |||
private TestRepository<FileRepository> tr; | |||
private CommitGraph commitGraph; | |||
@Override | |||
@Before | |||
public void setUp() throws Exception { | |||
super.setUp(); | |||
tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); | |||
} | |||
@Test | |||
public void testGraphWithManyParents() throws Exception { | |||
int parentsNum = 40; | |||
RevCommit root = commit(); | |||
RevCommit[] parents = new RevCommit[parentsNum]; | |||
for (int i = 0; i < parents.length; i++) { | |||
parents[i] = commit(root); | |||
} | |||
RevCommit tip = commit(parents); | |||
Set<ObjectId> wants = Collections.singleton(tip); | |||
writeCommitGraph(wants); | |||
assertEquals(parentsNum + 2, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
assertEquals(1, getGenerationNumber(root)); | |||
for (RevCommit parent : parents) { | |||
assertEquals(2, getGenerationNumber(parent)); | |||
} | |||
assertEquals(3, getGenerationNumber(tip)); | |||
} | |||
@Test | |||
public void testGraphWithoutMerges() throws Exception { | |||
int commitNum = 20; | |||
RevCommit[] commits = new RevCommit[commitNum]; | |||
for (int i = 0; i < commitNum; i++) { | |||
if (i == 0) { | |||
commits[i] = commit(); | |||
} else { | |||
commits[i] = commit(commits[i - 1]); | |||
} | |||
} | |||
Set<ObjectId> wants = Collections.singleton(commits[commitNum - 1]); | |||
writeCommitGraph(wants); | |||
assertEquals(commitNum, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
for (int i = 0; i < commitNum; i++) { | |||
assertEquals(i + 1, getGenerationNumber(commits[i])); | |||
} | |||
} | |||
@Test | |||
public void testGraphWithoutGeneration() throws Exception { | |||
StoredConfig storedConfig = db.getConfig(); | |||
storedConfig.setBoolean(ConfigConstants.CONFIG_COMMIT_GRAPH_SECTION, | |||
null, ConfigConstants.CONFIG_KEY_COMPUTE_GENERATION, false); | |||
storedConfig.save(); | |||
int commitNum = 10; | |||
RevCommit[] commits = new RevCommit[commitNum]; | |||
for (int i = 0; i < commitNum; i++) { | |||
if (i == 0) { | |||
commits[i] = commit(); | |||
} else { | |||
commits[i] = commit(commits[i - 1]); | |||
} | |||
} | |||
Set<ObjectId> wants = Collections.singleton(commits[commitNum - 1]); | |||
writeCommitGraph(wants); | |||
assertEquals(commitNum, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
for (int i = 0; i < commitNum; i++) { | |||
assertEquals(CommitGraph.GENERATION_NUMBER_ZERO, | |||
getGenerationNumber(commits[i])); | |||
} | |||
} | |||
@Test | |||
public void testGraphWithMerges() throws Exception { | |||
RevCommit c1 = commit(); | |||
RevCommit c2 = commit(c1); | |||
RevCommit c3 = commit(c2); | |||
RevCommit c4 = commit(c1); | |||
RevCommit c5 = commit(c4); | |||
RevCommit c6 = commit(c1); | |||
RevCommit c7 = commit(c6); | |||
RevCommit m1 = commit(c2, c4); | |||
RevCommit m2 = commit(c4, c6); | |||
RevCommit m3 = commit(c3, c5, c7); | |||
Set<ObjectId> wants = new HashSet<>(); | |||
/* | |||
* <pre> | |||
* current graph structure: | |||
* M1 | |||
* / \ | |||
* 2 4 | |||
* |___/ | |||
* 1 | |||
* </pre> | |||
*/ | |||
wants.add(m1); | |||
writeCommitGraph(wants); | |||
assertEquals(4, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
/* | |||
* <pre> | |||
* current graph structure: | |||
* M1 M2 | |||
* / \ / \ | |||
* 2 4 6 | |||
* |___/____/ | |||
* 1 | |||
* </pre> | |||
*/ | |||
wants.add(m2); | |||
writeCommitGraph(wants); | |||
assertEquals(6, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
/* | |||
* <pre> | |||
* current graph structure: | |||
* | |||
* __M3___ | |||
* / | \ | |||
* 3 M1 5 M2 7 | |||
* |/ \|/ \| | |||
* 2 4 6 | |||
* |___/____/ | |||
* 1 | |||
* </pre> | |||
*/ | |||
wants.add(m3); | |||
writeCommitGraph(wants); | |||
assertEquals(10, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
/* | |||
* <pre> | |||
* current graph structure: | |||
* 8 | |||
* | | |||
* __M3___ | |||
* / | \ | |||
* 3 M1 5 M2 7 | |||
* |/ \|/ \| | |||
* 2 4 6 | |||
* |___/____/ | |||
* 1 | |||
* </pre> | |||
*/ | |||
RevCommit c8 = commit(m3); | |||
wants.add(c8); | |||
writeCommitGraph(wants); | |||
assertEquals(11, commitGraph.getCommitCnt()); | |||
verifyCommitGraph(); | |||
assertEquals(getGenerationNumber(c1), 1); | |||
assertEquals(getGenerationNumber(c2), 2); | |||
assertEquals(getGenerationNumber(c4), 2); | |||
assertEquals(getGenerationNumber(c6), 2); | |||
assertEquals(getGenerationNumber(c3), 3); | |||
assertEquals(getGenerationNumber(c5), 3); | |||
assertEquals(getGenerationNumber(c7), 3); | |||
assertEquals(getGenerationNumber(m1), 3); | |||
assertEquals(getGenerationNumber(m2), 3); | |||
assertEquals(getGenerationNumber(m3), 4); | |||
assertEquals(getGenerationNumber(c8), 5); | |||
} | |||
void writeCommitGraph(Set<ObjectId> wants) throws Exception { | |||
NullProgressMonitor m = NullProgressMonitor.INSTANCE; | |||
CommitGraphWriter writer = new CommitGraphWriter(db); | |||
ByteArrayOutputStream os = new ByteArrayOutputStream(); | |||
writer.prepareCommitGraph(m, m, wants); | |||
writer.writeCommitGraph(m, os); | |||
InputStream inputStream = new ByteArrayInputStream(os.toByteArray()); | |||
CommitGraphData graphData = CommitGraphData.read(inputStream); | |||
commitGraph = new CommitGraphSingleImpl(graphData); | |||
} | |||
void verifyCommitGraph() throws Exception { | |||
try (RevWalk walk = new RevWalk(db)) { | |||
for (int i = 0; i < commitGraph.getCommitCnt(); i++) { | |||
ObjectId objId = commitGraph.getObjectId(i); | |||
CommitGraph.CommitData commit = commitGraph.getCommitData(i); | |||
int[] pList = commit.getParents(); | |||
RevCommit expect = walk.parseCommit(objId); | |||
assertEquals(expect.getCommitTime(), commit.getCommitTime()); | |||
assertEquals(expect.getTree(), commit.getTree()); | |||
assertEquals(expect.getParentCount(), pList.length); | |||
if (pList.length > 0) { | |||
ObjectId[] parents = new ObjectId[pList.length]; | |||
for (int j = 0; j < parents.length; j++) { | |||
parents[j] = commitGraph.getObjectId(pList[j]); | |||
} | |||
assertArrayEquals(expect.getParents(), parents); | |||
} | |||
} | |||
} | |||
} | |||
int getGenerationNumber(ObjectId id) { | |||
CommitGraph.CommitData commitData = commitGraph.getCommitData(id); | |||
if (commitData != null) { | |||
return commitData.getGeneration(); | |||
} | |||
return CommitGraph.GENERATION_NUMBER_INFINITY; | |||
} | |||
RevCommit commit(RevCommit... parents) throws Exception { | |||
return tr.commit(parents); | |||
} | |||
} |
@@ -141,7 +141,12 @@ collisionOn=Collision on {0} | |||
commandClosedStderrButDidntExit=Command {0} closed stderr stream but didn''t exit within timeout {1} seconds | |||
commandRejectedByHook=Rejected by "{0}" hook.\n{1} | |||
commandWasCalledInTheWrongState=Command {0} was called in the wrong state | |||
commitGraphChunkImproperOffset=commit-graph improper chunk 0x{0} offset {1} | |||
commitGraphChunkRepeated=commit-graph chunk id 0x{0} appears multiple times | |||
commitGraphChunkUnknown=unknown commit-graph chunk: 0x{0} | |||
commitGraphFileIsTooLargeForJgit=commit-graph file is too large for jgit | |||
commitGraphGeneratingCancelledDuringWriting=commit-graph generating cancelled during writing | |||
commitGraphOidFanoutNeeded=commit-graph oid fanout chunk has not been loaded | |||
commitMessageNotSpecified=commit message not specified | |||
commitOnRepoWithoutHEADCurrentlyNotSupported=Commit on repo without HEAD currently not supported | |||
commitAmendOnInitialNotPossible=Amending is not possible on initial commit. | |||
@@ -495,6 +500,7 @@ noSuchRefKnown=no such ref: {0} | |||
noSuchSubmodule=no such submodule {0} | |||
notABoolean=Not a boolean: {0} | |||
notABundle=not a bundle | |||
notACommitGraph=not a commit-graph | |||
notADIRCFile=Not a DIRC file. | |||
notAGitDirectory=not a git directory | |||
notAPACKFile=Not a PACK file. | |||
@@ -767,6 +773,7 @@ unlockLockFileFailed=Unlocking LockFile ''{0}'' failed | |||
unmergedPath=Unmerged path: {0} | |||
unmergedPaths=Repository contains unmerged paths | |||
unpackException=Exception while parsing pack stream | |||
unreadableCommitGraph=Unreadable commit graph: {0} | |||
unreadablePackIndex=Unreadable pack index: {0} | |||
unrecognizedPackExtension=Unrecognized pack extension: {0} | |||
unrecognizedRef=Unrecognized ref: {0} | |||
@@ -774,6 +781,7 @@ unsetMark=Mark not set | |||
unsupportedAlternates=Alternates not supported | |||
unsupportedArchiveFormat=Unknown archive format ''{0}'' | |||
unsupportedCommand0=unsupported command 0 | |||
unsupportedCommitGraphVersion=Unsupported commit graph version: {0} | |||
unsupportedEncryptionAlgorithm=Unsupported encryption algorithm: {0} | |||
unsupportedEncryptionVersion=Unsupported encryption version: {0} | |||
unsupportedGC=Unsupported garbage collector for repository type: {0} |
@@ -0,0 +1,33 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.errors; | |||
import java.io.IOException; | |||
/** | |||
* Thrown when a commit-graph file's format is different than we expected | |||
* | |||
* @since 5.13 | |||
*/ | |||
public class CommitGraphFormatException extends IOException { | |||
private static final long serialVersionUID = 1L; | |||
/** | |||
* Construct an exception. | |||
* | |||
* @param why | |||
* description of the type of error. | |||
*/ | |||
public CommitGraphFormatException(String why) { | |||
super(why); | |||
} | |||
} |
@@ -169,7 +169,12 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String commandClosedStderrButDidntExit; | |||
/***/ public String commandRejectedByHook; | |||
/***/ public String commandWasCalledInTheWrongState; | |||
/***/ public String commitGraphChunkImproperOffset; | |||
/***/ public String commitGraphChunkRepeated; | |||
/***/ public String commitGraphChunkUnknown; | |||
/***/ public String commitGraphFileIsTooLargeForJgit; | |||
/***/ public String commitGraphGeneratingCancelledDuringWriting; | |||
/***/ public String commitGraphOidFanoutNeeded; | |||
/***/ public String commitMessageNotSpecified; | |||
/***/ public String commitOnRepoWithoutHEADCurrentlyNotSupported; | |||
/***/ public String commitAmendOnInitialNotPossible; | |||
@@ -523,6 +528,7 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String noSuchSubmodule; | |||
/***/ public String notABoolean; | |||
/***/ public String notABundle; | |||
/***/ public String notACommitGraph; | |||
/***/ public String notADIRCFile; | |||
/***/ public String notAGitDirectory; | |||
/***/ public String notAPACKFile; | |||
@@ -795,6 +801,7 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String unmergedPath; | |||
/***/ public String unmergedPaths; | |||
/***/ public String unpackException; | |||
/***/ public String unreadableCommitGraph; | |||
/***/ public String unreadablePackIndex; | |||
/***/ public String unrecognizedPackExtension; | |||
/***/ public String unrecognizedRef; | |||
@@ -802,6 +809,7 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String unsupportedAlternates; | |||
/***/ public String unsupportedArchiveFormat; | |||
/***/ public String unsupportedCommand0; | |||
/***/ public String unsupportedCommitGraphVersion; | |||
/***/ public String unsupportedEncryptionAlgorithm; | |||
/***/ public String unsupportedEncryptionVersion; | |||
/***/ public String unsupportedGC; |
@@ -0,0 +1,182 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import java.io.File; | |||
import java.io.FileNotFoundException; | |||
import java.io.IOException; | |||
import java.io.InputStream; | |||
import java.text.MessageFormat; | |||
import org.eclipse.jgit.errors.CommitGraphFormatException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.lib.AnyObjectId; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.util.IO; | |||
import org.eclipse.jgit.util.io.SilentFileInputStream; | |||
/** | |||
* <p> | |||
* The commit-graph stores a list of commit OIDs and some associated metadata, | |||
* including: | |||
* <ol> | |||
* <li>The generation number of the commit. Commits with no parents have | |||
* generation number 1; commits with parents have generation number one more | |||
* than the maximum generation number of its parents. We reserve zero as | |||
* special, and can be used to mark a generation number invalid or as "not | |||
* computed".</li> | |||
* <li>The root tree OID.</li> | |||
* <li>The commit date.</li> | |||
* <li>The parents of the commit, stored using positional references within the | |||
* graph file.</li> | |||
* </ol> | |||
* </p> | |||
*/ | |||
public abstract class CommitGraphData { | |||
/** | |||
* Open an existing commit-graph file for reading. | |||
* <p> | |||
* The format of the file will be automatically detected and a proper access | |||
* implementation for that format will be constructed and returned to the | |||
* caller. The file may or may not be held open by the returned instance. | |||
* | |||
* @param graphFile | |||
* existing commit-graph to read. | |||
* @return a copy of the commit-graph file in memory | |||
* @throws FileNotFoundException | |||
* the file does not exist. | |||
* @throws CommitGraphFormatException | |||
* commit-graph file's format is different than we expected. | |||
* @throws java.io.IOException | |||
* the file exists but could not be read due to security errors | |||
* or unexpected data corruption. | |||
*/ | |||
public static CommitGraphData open(File graphFile) | |||
throws FileNotFoundException, CommitGraphFormatException, | |||
IOException { | |||
try (SilentFileInputStream fd = new SilentFileInputStream(graphFile)) { | |||
try { | |||
return read(fd); | |||
} catch (CommitGraphFormatException fe) { | |||
throw fe; | |||
} catch (IOException ioe) { | |||
throw new IOException(MessageFormat.format( | |||
JGitText.get().unreadableCommitGraph, | |||
graphFile.getAbsolutePath()), ioe); | |||
} | |||
} | |||
} | |||
/** | |||
* Read an existing commit-graph file from a buffered stream. | |||
* <p> | |||
* The format of the file will be automatically detected and a proper access | |||
* implementation for that format will be constructed and returned to the | |||
* caller. The file may or may not be held open by the returned instance. | |||
* | |||
* @param fd | |||
* stream to read the commit-graph file from. The stream must be | |||
* buffered as some small IOs are performed against the stream. | |||
* The caller is responsible for closing the stream. | |||
* | |||
* @return a copy of the commit-graph file in memory | |||
* @throws CommitGraphFormatException | |||
* the commit-graph file's format is different than we expected. | |||
* @throws java.io.IOException | |||
* the stream cannot be read. | |||
*/ | |||
public static CommitGraphData read(InputStream fd) throws IOException { | |||
byte[] hdr = new byte[8]; | |||
IO.readFully(fd, hdr, 0, hdr.length); | |||
int v = hdr[4]; | |||
if (v != 1) { | |||
throw new CommitGraphFormatException(MessageFormat | |||
.format(JGitText.get().unsupportedCommitGraphVersion, v)); | |||
} | |||
return new CommitGraphDataV1(fd, hdr); | |||
} | |||
/** | |||
* Finds the position in the commit-graph of the object. | |||
* | |||
* @param objId | |||
* the id for which the commit-graph position will be found. | |||
* @return the commit-graph id or -1 if the object was not found. | |||
*/ | |||
public abstract int findGraphPosition(AnyObjectId objId); | |||
/** | |||
* Get the object at the commit-graph position. | |||
* | |||
* @param graphPos | |||
* the position in the commit-graph of the object. | |||
* @return the ObjectId or null if the object was not found. | |||
*/ | |||
public abstract ObjectId getObjectId(int graphPos); | |||
/** | |||
* Get the metadata of a commit。 | |||
* | |||
* @param graphPos | |||
* the position in the commit-graph of the object. | |||
* @return the metadata of a commit or null if it's not found. | |||
*/ | |||
public abstract CommitGraph.CommitData getCommitData(int graphPos); | |||
/** | |||
* Obtain the total number of commits described by this commit-graph. | |||
* | |||
* @return number of commits in this commit-graph | |||
*/ | |||
public abstract long getCommitCnt(); | |||
/** | |||
* Get the hash length of this commit-graph | |||
* | |||
* @return object hash length | |||
*/ | |||
public abstract int getHashLength(); | |||
static class CommitDataImpl extends CommitGraph.CommitData { | |||
ObjectId tree; | |||
int[] parents; | |||
long commitTime; | |||
int generation; | |||
@Override | |||
public ObjectId getTree() { | |||
return tree; | |||
} | |||
@Override | |||
public int[] getParents() { | |||
return parents; | |||
} | |||
@Override | |||
public long getCommitTime() { | |||
return commitTime; | |||
} | |||
@Override | |||
public int getGeneration() { | |||
return generation; | |||
} | |||
} | |||
} |
@@ -0,0 +1,393 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_DATA_EXTRA_LENGTH; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_GRAPH_MAGIC; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_CHUNK_LOOKUP_WIDTH; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EDGE_LAST_MASK; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EXTRA_EDGES_NEEDED; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_LAST_EDGE; | |||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_NO_PARENT; | |||
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; | |||
import java.io.IOException; | |||
import java.io.InputStream; | |||
import java.text.MessageFormat; | |||
import java.util.Arrays; | |||
import org.eclipse.jgit.errors.CommitGraphFormatException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.lib.AnyObjectId; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.util.IO; | |||
import org.eclipse.jgit.util.NB; | |||
/** | |||
* Support for the commit-graph v1 format. | |||
* | |||
* @see CommitGraphData | |||
*/ | |||
public class CommitGraphDataV1 extends CommitGraphData { | |||
private static final int chunkLookupWidth = GRAPH_CHUNK_LOOKUP_WIDTH; | |||
private static final int FANOUT = 256; | |||
private final long commitCnt; | |||
private final int hashLength; | |||
private final int commitDataLength; | |||
private long[] oidFanout; | |||
private byte[][] oidLookup; | |||
private byte[][] commitData; | |||
private byte[] extraEdgeList; | |||
CommitGraphDataV1(InputStream fd, byte[] hdr) throws IOException { | |||
int magic = NB.decodeInt32(hdr, 0); | |||
if (magic != COMMIT_GRAPH_MAGIC) { | |||
throw new CommitGraphFormatException( | |||
JGitText.get().notACommitGraph); | |||
} | |||
// Read the hash version (1 byte) | |||
// 1 => SHA-1 | |||
// 2 => SHA-256 nonsupport now | |||
int hashVersion = hdr[5]; | |||
if (hashVersion != 1) { | |||
throw new CommitGraphFormatException( | |||
JGitText.get().incorrectOBJECT_ID_LENGTH); | |||
} | |||
hashLength = OBJECT_ID_LENGTH; | |||
commitDataLength = hashLength + COMMIT_DATA_EXTRA_LENGTH; | |||
// Read the number of "chunkOffsets" (1 byte) | |||
int numberOfChunks = hdr[6]; | |||
byte[] chunkLookup = new byte[chunkLookupWidth * (numberOfChunks + 1)]; | |||
IO.readFully(fd, chunkLookup, 0, chunkLookup.length); | |||
int[] chunkId = new int[numberOfChunks + 1]; | |||
long[] chunkOffset = new long[numberOfChunks + 1]; | |||
for (int i = 0; i <= numberOfChunks; i++) { | |||
chunkId[i] = NB.decodeInt32(chunkLookup, i * 12); | |||
for (int j = 0; j < i; j++) { | |||
if (chunkId[i] == chunkId[j]) { | |||
throw new CommitGraphFormatException(MessageFormat.format( | |||
JGitText.get().commitGraphChunkRepeated, | |||
Integer.toHexString(chunkId[i]))); | |||
} | |||
} | |||
chunkOffset[i] = NB.decodeInt64(chunkLookup, i * 12 + 4); | |||
} | |||
oidLookup = new byte[FANOUT][]; | |||
commitData = new byte[FANOUT][]; | |||
for (int i = 0; i < numberOfChunks; i++) { | |||
long length = chunkOffset[i + 1] - chunkOffset[i]; | |||
long lengthReaded; | |||
if (chunkOffset[i] < 0 | |||
|| chunkOffset[i] > chunkOffset[numberOfChunks]) { | |||
throw new CommitGraphFormatException(MessageFormat.format( | |||
JGitText.get().commitGraphChunkImproperOffset, | |||
Integer.toHexString(chunkId[i]), chunkOffset[i])); | |||
} | |||
switch (chunkId[i]) { | |||
case CHUNK_ID_OID_FANOUT: | |||
lengthReaded = loadChunkOidFanout(fd); | |||
break; | |||
case CHUNK_ID_OID_LOOKUP: | |||
lengthReaded = loadChunkDataBasedOnFanout(fd, hashLength, | |||
oidLookup); | |||
break; | |||
case CHUNK_ID_COMMIT_DATA: | |||
lengthReaded = loadChunkDataBasedOnFanout(fd, commitDataLength, | |||
commitData); | |||
break; | |||
case CHUNK_ID_EXTRA_EDGE_LIST: | |||
lengthReaded = loadChunkExtraEdgeList(fd, length); | |||
break; | |||
default: | |||
throw new CommitGraphFormatException(MessageFormat.format( | |||
JGitText.get().commitGraphChunkUnknown, | |||
Integer.toHexString(chunkId[i]))); | |||
} | |||
if (length != lengthReaded) { | |||
throw new CommitGraphFormatException(MessageFormat.format( | |||
JGitText.get().commitGraphChunkImproperOffset, | |||
Integer.toHexString(chunkId[i + 1]), | |||
chunkOffset[i + 1])); | |||
} | |||
} | |||
if (oidFanout == null) { | |||
throw new CommitGraphFormatException( | |||
JGitText.get().commitGraphOidFanoutNeeded); | |||
} | |||
commitCnt = oidFanout[FANOUT - 1]; | |||
} | |||
private long loadChunkOidFanout(InputStream fd) throws IOException { | |||
int fanoutLen = FANOUT * 4; | |||
byte[] fanoutTable = new byte[fanoutLen]; | |||
IO.readFully(fd, fanoutTable, 0, fanoutLen); | |||
oidFanout = new long[256]; // really unsigned 32-bit... | |||
for (int k = 0; k < oidFanout.length; k++) { | |||
oidFanout[k] = NB.decodeUInt32(fanoutTable, k * 4); | |||
} | |||
return fanoutLen; | |||
} | |||
private long loadChunkDataBasedOnFanout(InputStream fd, int itemLength, | |||
byte[][] chunkData) throws IOException { | |||
if (oidFanout == null) { | |||
throw new CommitGraphFormatException( | |||
JGitText.get().commitGraphOidFanoutNeeded); | |||
} | |||
long readedLength = 0; | |||
for (int k = 0; k < oidFanout.length; k++) { | |||
long n; | |||
if (k == 0) { | |||
n = oidFanout[k]; | |||
} else { | |||
n = oidFanout[k] - oidFanout[k - 1]; | |||
} | |||
if (n > 0) { | |||
long len = n * itemLength; | |||
if (len > Integer.MAX_VALUE - 8) { // http://stackoverflow.com/a/8381338 | |||
throw new CommitGraphFormatException( | |||
JGitText.get().commitGraphFileIsTooLargeForJgit); | |||
} | |||
chunkData[k] = new byte[(int) len]; | |||
IO.readFully(fd, chunkData[k], 0, chunkData[k].length); | |||
readedLength += len; | |||
} | |||
} | |||
return readedLength; | |||
} | |||
private long loadChunkExtraEdgeList(InputStream fd, long len) | |||
throws IOException { | |||
if (len > Integer.MAX_VALUE - 8) { // http://stackoverflow.com/a/8381338 | |||
throw new CommitGraphFormatException( | |||
JGitText.get().commitGraphFileIsTooLargeForJgit); | |||
} | |||
extraEdgeList = new byte[(int) len]; | |||
IO.readFully(fd, extraEdgeList, 0, extraEdgeList.length); | |||
return len; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public int findGraphPosition(AnyObjectId objId) { | |||
int levelOne = objId.getFirstByte(); | |||
byte[] data = oidLookup[levelOne]; | |||
if (data == null) { | |||
return -1; | |||
} | |||
int high = data.length / (hashLength); | |||
int low = 0; | |||
do { | |||
int mid = (low + high) >>> 1; | |||
int pos = objIdOffset(mid); | |||
int cmp = objId.compareTo(data, pos); | |||
if (cmp < 0) { | |||
high = mid; | |||
} else if (cmp == 0) { | |||
if (levelOne == 0) { | |||
return mid; | |||
} | |||
return (int) (mid + oidFanout[levelOne - 1]); | |||
} else { | |||
low = mid + 1; | |||
} | |||
} while (low < high); | |||
return -1; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public ObjectId getObjectId(int graphPos) { | |||
if (graphPos < 0 || graphPos > commitCnt) { | |||
return null; | |||
} | |||
int levelOne = findLevelOne(graphPos); | |||
int p = getLevelTwo(graphPos, levelOne); | |||
int dataIdx = objIdOffset(p); | |||
return ObjectId.fromRaw(oidLookup[levelOne], dataIdx); | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public CommitGraph.CommitData getCommitData(int graphPos) { | |||
int levelOne = findLevelOne(graphPos); | |||
int p = getLevelTwo(graphPos, levelOne); | |||
int dataIdx = commitDataOffset(p); | |||
byte[] data = this.commitData[levelOne]; | |||
if (graphPos < 0) { | |||
return null; | |||
} | |||
CommitDataImpl commit = new CommitDataImpl(); | |||
// parse tree | |||
commit.tree = ObjectId.fromRaw(data, dataIdx); | |||
// parse date | |||
long dateHigh = NB.decodeInt32(data, dataIdx + hashLength + 8) & 0x3; | |||
long dateLow = NB.decodeInt32(data, dataIdx + hashLength + 12); | |||
commit.commitTime = dateHigh << 32 | dateLow; | |||
// parse generation | |||
commit.generation = NB.decodeInt32(data, dataIdx + hashLength + 8) >> 2; | |||
boolean noParents = false; | |||
int[] pList = new int[0]; | |||
int edgeValue = NB.decodeInt32(data, dataIdx + hashLength); | |||
if (edgeValue == GRAPH_NO_PARENT) { | |||
noParents = true; | |||
} | |||
// parse parents | |||
if (!noParents) { | |||
pList = new int[1]; | |||
int parent = edgeValue; | |||
pList[0] = parent; | |||
edgeValue = NB.decodeInt32(data, dataIdx + hashLength + 4); | |||
if (edgeValue != GRAPH_NO_PARENT) { | |||
if ((edgeValue & GRAPH_EXTRA_EDGES_NEEDED) != 0) { | |||
int pptr = edgeValue & GRAPH_EDGE_LAST_MASK; | |||
int[] extraEdgeList = findExtraEdgeList(pptr); | |||
if (extraEdgeList == null) { | |||
return null; | |||
} | |||
int[] old = pList; | |||
pList = new int[extraEdgeList.length + 1]; | |||
pList[0] = old[0]; | |||
for (int i = 0; i < extraEdgeList.length; i++) { | |||
parent = extraEdgeList[i]; | |||
pList[i + 1] = parent; | |||
} | |||
} else { | |||
parent = edgeValue; | |||
pList = new int[] { pList[0], parent }; | |||
} | |||
} | |||
} | |||
commit.parents = pList; | |||
return commit; | |||
} | |||
/** | |||
* Find the list of commit-graph position in extra edge list chunk. | |||
* <p> | |||
* The extra edge list chunk store the second through nth parents for all | |||
* octopus merges. | |||
* | |||
* @param pptr | |||
* the start position to iterate of extra edge list chunk | |||
* @return the list of commit-graph position or null if not found | |||
*/ | |||
int[] findExtraEdgeList(int pptr) { | |||
if (extraEdgeList == null) { | |||
return null; | |||
} | |||
int maxOffset = extraEdgeList.length - 4; | |||
int offset = pptr * 4; | |||
if (offset < 0 || offset > maxOffset) { | |||
return null; | |||
} | |||
int[] pList = new int[32]; | |||
int count = 0; | |||
int parentPosition; | |||
for (;;) { | |||
if (count >= pList.length) { | |||
int[] old = pList; | |||
pList = new int[pList.length + 32]; | |||
System.arraycopy(old, 0, pList, 0, count); | |||
} | |||
if (offset > maxOffset) { | |||
return null; | |||
} | |||
parentPosition = NB.decodeInt32(extraEdgeList, offset); | |||
if ((parentPosition & GRAPH_LAST_EDGE) != 0) { | |||
pList[count] = parentPosition & GRAPH_EDGE_LAST_MASK; | |||
count++; | |||
break; | |||
} | |||
pList[count++] = parentPosition; | |||
offset += 4; | |||
} | |||
int[] old = pList; | |||
pList = new int[count]; | |||
System.arraycopy(old, 0, pList, 0, count); | |||
return pList; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public long getCommitCnt() { | |||
return commitCnt; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public int getHashLength() { | |||
return hashLength; | |||
} | |||
private int findLevelOne(long nthPosition) { | |||
int levelOne = Arrays.binarySearch(oidFanout, nthPosition + 1); | |||
if (levelOne >= 0) { | |||
// If we hit the bucket exactly the item is in the bucket, or | |||
// any bucket before it which has the same object count. | |||
// | |||
long base = oidFanout[levelOne]; | |||
while (levelOne > 0 && base == oidFanout[levelOne - 1]) | |||
levelOne--; | |||
} else { | |||
// The item is in the bucket we would insert it into. | |||
// | |||
levelOne = -(levelOne + 1); | |||
} | |||
return levelOne; | |||
} | |||
private int getLevelTwo(long nthPosition, int levelOne) { | |||
long base = levelOne > 0 ? oidFanout[levelOne - 1] : 0; | |||
return (int) (nthPosition - base); | |||
} | |||
private int objIdOffset(int mid) { | |||
return hashLength * mid; | |||
} | |||
private int commitDataOffset(int mid) { | |||
return commitDataLength * mid; | |||
} | |||
} |
@@ -0,0 +1,64 @@ | |||
/* | |||
* Copyright (C) 2021, Tencent. | |||
* | |||
* This program and the accompanying materials are made available under the | |||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||
* | |||
* SPDX-License-Identifier: BSD-3-Clause | |||
*/ | |||
package org.eclipse.jgit.internal.storage.commitgraph; | |||
import org.eclipse.jgit.lib.AnyObjectId; | |||
import org.eclipse.jgit.lib.CommitGraph; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
/** | |||
* CommitGraph implementation by a single commit-graph file. | |||
* | |||
* @see CommitGraph | |||
*/ | |||
public class CommitGraphSingleImpl extends CommitGraph { | |||
private final CommitGraphData graphData; | |||
/** | |||
* Creates CommitGraph by a single commit-graph file. | |||
* | |||
* @param graphData | |||
* the commit-graph file in memory | |||
*/ | |||
public CommitGraphSingleImpl(CommitGraphData graphData) { | |||
this.graphData = graphData; | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public CommitData getCommitData(AnyObjectId commit) { | |||
int graphPos = graphData.findGraphPosition(commit); | |||
return getCommitData(graphPos); | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public CommitData getCommitData(int graphPos) { | |||
if (graphPos < 0 || graphPos > graphData.getCommitCnt()) { | |||
return null; | |||
} | |||
return graphData.getCommitData(graphPos); | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public ObjectId getObjectId(int graphPos) { | |||
return graphData.getObjectId(graphPos); | |||
} | |||
/** {@inheritDoc} */ | |||
@Override | |||
public long getCommitCnt() { | |||
return graphData.getCommitCnt(); | |||
} | |||
} |