This change teaches JGit to use a single commit-graph file to accelerate commit graph walks. When new commits are added to the repo, the commit-graph gets further and further behind, and then we write the whole commit-graph file again to include new commits. Multiple commit-graph files are not supported in the current implementation. With this feature, git will write a commit-graph after every git fetch command that downloads a pack-file from a remote. JGit can read a commit-graph file from a buffered stream, which means that we can provide this feature for both FileRepository and DfsRepository. Bug: 574368 Change-Id: Ib5c0d6678cb242870a0f5841bd413ad3885e95f6 Signed-off-by: kylezhao <kylezhao@tencent.com>changes/92/182892/3
/* | |||||
* Copyright (C) 2021, Tencent. | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.commitgraph; | |||||
import static org.junit.Assert.assertArrayEquals; | |||||
import static org.junit.Assert.assertEquals; | |||||
import java.io.ByteArrayInputStream; | |||||
import java.io.ByteArrayOutputStream; | |||||
import java.io.InputStream; | |||||
import java.util.Collections; | |||||
import java.util.HashSet; | |||||
import java.util.Set; | |||||
import org.eclipse.jgit.internal.storage.file.FileRepository; | |||||
import org.eclipse.jgit.junit.RepositoryTestCase; | |||||
import org.eclipse.jgit.junit.TestRepository; | |||||
import org.eclipse.jgit.lib.CommitGraph; | |||||
import org.eclipse.jgit.lib.ConfigConstants; | |||||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.StoredConfig; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
public class CommitGraphTest extends RepositoryTestCase { | |||||
private TestRepository<FileRepository> tr; | |||||
private CommitGraph commitGraph; | |||||
@Override | |||||
@Before | |||||
public void setUp() throws Exception { | |||||
super.setUp(); | |||||
tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); | |||||
} | |||||
@Test | |||||
public void testGraphWithManyParents() throws Exception { | |||||
int parentsNum = 40; | |||||
RevCommit root = commit(); | |||||
RevCommit[] parents = new RevCommit[parentsNum]; | |||||
for (int i = 0; i < parents.length; i++) { | |||||
parents[i] = commit(root); | |||||
} | |||||
RevCommit tip = commit(parents); | |||||
Set<ObjectId> wants = Collections.singleton(tip); | |||||
writeCommitGraph(wants); | |||||
assertEquals(parentsNum + 2, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
assertEquals(1, getGenerationNumber(root)); | |||||
for (RevCommit parent : parents) { | |||||
assertEquals(2, getGenerationNumber(parent)); | |||||
} | |||||
assertEquals(3, getGenerationNumber(tip)); | |||||
} | |||||
@Test | |||||
public void testGraphWithoutMerges() throws Exception { | |||||
int commitNum = 20; | |||||
RevCommit[] commits = new RevCommit[commitNum]; | |||||
for (int i = 0; i < commitNum; i++) { | |||||
if (i == 0) { | |||||
commits[i] = commit(); | |||||
} else { | |||||
commits[i] = commit(commits[i - 1]); | |||||
} | |||||
} | |||||
Set<ObjectId> wants = Collections.singleton(commits[commitNum - 1]); | |||||
writeCommitGraph(wants); | |||||
assertEquals(commitNum, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
for (int i = 0; i < commitNum; i++) { | |||||
assertEquals(i + 1, getGenerationNumber(commits[i])); | |||||
} | |||||
} | |||||
@Test | |||||
public void testGraphWithoutGeneration() throws Exception { | |||||
StoredConfig storedConfig = db.getConfig(); | |||||
storedConfig.setBoolean(ConfigConstants.CONFIG_COMMIT_GRAPH_SECTION, | |||||
null, ConfigConstants.CONFIG_KEY_COMPUTE_GENERATION, false); | |||||
storedConfig.save(); | |||||
int commitNum = 10; | |||||
RevCommit[] commits = new RevCommit[commitNum]; | |||||
for (int i = 0; i < commitNum; i++) { | |||||
if (i == 0) { | |||||
commits[i] = commit(); | |||||
} else { | |||||
commits[i] = commit(commits[i - 1]); | |||||
} | |||||
} | |||||
Set<ObjectId> wants = Collections.singleton(commits[commitNum - 1]); | |||||
writeCommitGraph(wants); | |||||
assertEquals(commitNum, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
for (int i = 0; i < commitNum; i++) { | |||||
assertEquals(CommitGraph.GENERATION_NUMBER_ZERO, | |||||
getGenerationNumber(commits[i])); | |||||
} | |||||
} | |||||
@Test | |||||
public void testGraphWithMerges() throws Exception { | |||||
RevCommit c1 = commit(); | |||||
RevCommit c2 = commit(c1); | |||||
RevCommit c3 = commit(c2); | |||||
RevCommit c4 = commit(c1); | |||||
RevCommit c5 = commit(c4); | |||||
RevCommit c6 = commit(c1); | |||||
RevCommit c7 = commit(c6); | |||||
RevCommit m1 = commit(c2, c4); | |||||
RevCommit m2 = commit(c4, c6); | |||||
RevCommit m3 = commit(c3, c5, c7); | |||||
Set<ObjectId> wants = new HashSet<>(); | |||||
/* | |||||
* <pre> | |||||
* current graph structure: | |||||
* M1 | |||||
* / \ | |||||
* 2 4 | |||||
* |___/ | |||||
* 1 | |||||
* </pre> | |||||
*/ | |||||
wants.add(m1); | |||||
writeCommitGraph(wants); | |||||
assertEquals(4, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
/* | |||||
* <pre> | |||||
* current graph structure: | |||||
* M1 M2 | |||||
* / \ / \ | |||||
* 2 4 6 | |||||
* |___/____/ | |||||
* 1 | |||||
* </pre> | |||||
*/ | |||||
wants.add(m2); | |||||
writeCommitGraph(wants); | |||||
assertEquals(6, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
/* | |||||
* <pre> | |||||
* current graph structure: | |||||
* | |||||
* __M3___ | |||||
* / | \ | |||||
* 3 M1 5 M2 7 | |||||
* |/ \|/ \| | |||||
* 2 4 6 | |||||
* |___/____/ | |||||
* 1 | |||||
* </pre> | |||||
*/ | |||||
wants.add(m3); | |||||
writeCommitGraph(wants); | |||||
assertEquals(10, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
/* | |||||
* <pre> | |||||
* current graph structure: | |||||
* 8 | |||||
* | | |||||
* __M3___ | |||||
* / | \ | |||||
* 3 M1 5 M2 7 | |||||
* |/ \|/ \| | |||||
* 2 4 6 | |||||
* |___/____/ | |||||
* 1 | |||||
* </pre> | |||||
*/ | |||||
RevCommit c8 = commit(m3); | |||||
wants.add(c8); | |||||
writeCommitGraph(wants); | |||||
assertEquals(11, commitGraph.getCommitCnt()); | |||||
verifyCommitGraph(); | |||||
assertEquals(getGenerationNumber(c1), 1); | |||||
assertEquals(getGenerationNumber(c2), 2); | |||||
assertEquals(getGenerationNumber(c4), 2); | |||||
assertEquals(getGenerationNumber(c6), 2); | |||||
assertEquals(getGenerationNumber(c3), 3); | |||||
assertEquals(getGenerationNumber(c5), 3); | |||||
assertEquals(getGenerationNumber(c7), 3); | |||||
assertEquals(getGenerationNumber(m1), 3); | |||||
assertEquals(getGenerationNumber(m2), 3); | |||||
assertEquals(getGenerationNumber(m3), 4); | |||||
assertEquals(getGenerationNumber(c8), 5); | |||||
} | |||||
void writeCommitGraph(Set<ObjectId> wants) throws Exception { | |||||
NullProgressMonitor m = NullProgressMonitor.INSTANCE; | |||||
CommitGraphWriter writer = new CommitGraphWriter(db); | |||||
ByteArrayOutputStream os = new ByteArrayOutputStream(); | |||||
writer.prepareCommitGraph(m, m, wants); | |||||
writer.writeCommitGraph(m, os); | |||||
InputStream inputStream = new ByteArrayInputStream(os.toByteArray()); | |||||
CommitGraphData graphData = CommitGraphData.read(inputStream); | |||||
commitGraph = new CommitGraphSingleImpl(graphData); | |||||
} | |||||
void verifyCommitGraph() throws Exception { | |||||
try (RevWalk walk = new RevWalk(db)) { | |||||
for (int i = 0; i < commitGraph.getCommitCnt(); i++) { | |||||
ObjectId objId = commitGraph.getObjectId(i); | |||||
CommitGraph.CommitData commit = commitGraph.getCommitData(i); | |||||
int[] pList = commit.getParents(); | |||||
RevCommit expect = walk.lookupCommit(objId); | |||||
walk.parseBody(expect); | |||||
assertEquals(expect.getCommitTime(), commit.getCommitTime()); | |||||
assertEquals(expect.getTree(), commit.getTree()); | |||||
assertEquals(expect.getParentCount(), pList.length); | |||||
if (pList.length > 0) { | |||||
ObjectId[] parents = new ObjectId[pList.length]; | |||||
for (int j = 0; j < parents.length; j++) { | |||||
parents[j] = commitGraph.getObjectId(pList[j]); | |||||
} | |||||
assertArrayEquals(expect.getParents(), parents); | |||||
} | |||||
} | |||||
} | |||||
} | |||||
int getGenerationNumber(ObjectId id) { | |||||
CommitGraph.CommitData commitData = commitGraph.getCommitData(id); | |||||
if (commitData != null) { | |||||
return commitData.getGeneration(); | |||||
} | |||||
return CommitGraph.GENERATION_NUMBER_INFINITY; | |||||
} | |||||
RevCommit commit(RevCommit... parents) throws Exception { | |||||
return tr.commit(parents); | |||||
} | |||||
} |
commandClosedStderrButDidntExit=Command {0} closed stderr stream but didn''t exit within timeout {1} seconds | commandClosedStderrButDidntExit=Command {0} closed stderr stream but didn''t exit within timeout {1} seconds | ||||
commandRejectedByHook=Rejected by "{0}" hook.\n{1} | commandRejectedByHook=Rejected by "{0}" hook.\n{1} | ||||
commandWasCalledInTheWrongState=Command {0} was called in the wrong state | commandWasCalledInTheWrongState=Command {0} was called in the wrong state | ||||
commitGraphChunkImproperOffset=commit-graph improper chunk 0x{0} offset {1} | |||||
commitGraphChunkRepeated=commit-graph chunk id 0x{0} appears multiple times | |||||
commitGraphChunkUnknown=unknown commit-graph chunk: 0x{0} | |||||
commitGraphFileIsTooLargeForJgit=commit-graph file is too large for jgit | |||||
commitGraphGeneratingCancelledDuringWriting=commit-graph generating cancelled during writing | commitGraphGeneratingCancelledDuringWriting=commit-graph generating cancelled during writing | ||||
commitGraphOidFanoutNeeded=commit-graph oid fanout chunk has not been loaded | |||||
commitMessageNotSpecified=commit message not specified | commitMessageNotSpecified=commit message not specified | ||||
commitOnRepoWithoutHEADCurrentlyNotSupported=Commit on repo without HEAD currently not supported | commitOnRepoWithoutHEADCurrentlyNotSupported=Commit on repo without HEAD currently not supported | ||||
commitAmendOnInitialNotPossible=Amending is not possible on initial commit. | commitAmendOnInitialNotPossible=Amending is not possible on initial commit. | ||||
noSuchSubmodule=no such submodule {0} | noSuchSubmodule=no such submodule {0} | ||||
notABoolean=Not a boolean: {0} | notABoolean=Not a boolean: {0} | ||||
notABundle=not a bundle | notABundle=not a bundle | ||||
notACommitGraph=not a commit-graph | |||||
notADIRCFile=Not a DIRC file. | notADIRCFile=Not a DIRC file. | ||||
notAGitDirectory=not a git directory | notAGitDirectory=not a git directory | ||||
notAPACKFile=Not a PACK file. | notAPACKFile=Not a PACK file. | ||||
unmergedPath=Unmerged path: {0} | unmergedPath=Unmerged path: {0} | ||||
unmergedPaths=Repository contains unmerged paths | unmergedPaths=Repository contains unmerged paths | ||||
unpackException=Exception while parsing pack stream | unpackException=Exception while parsing pack stream | ||||
unreadableCommitGraph=Unreadable commit graph: {0} | |||||
unreadablePackIndex=Unreadable pack index: {0} | unreadablePackIndex=Unreadable pack index: {0} | ||||
unrecognizedPackExtension=Unrecognized pack extension: {0} | unrecognizedPackExtension=Unrecognized pack extension: {0} | ||||
unrecognizedRef=Unrecognized ref: {0} | unrecognizedRef=Unrecognized ref: {0} | ||||
unsupportedAlternates=Alternates not supported | unsupportedAlternates=Alternates not supported | ||||
unsupportedArchiveFormat=Unknown archive format ''{0}'' | unsupportedArchiveFormat=Unknown archive format ''{0}'' | ||||
unsupportedCommand0=unsupported command 0 | unsupportedCommand0=unsupported command 0 | ||||
unsupportedCommitGraphVersion=Unsupported commit graph version: {0} | |||||
unsupportedEncryptionAlgorithm=Unsupported encryption algorithm: {0} | unsupportedEncryptionAlgorithm=Unsupported encryption algorithm: {0} | ||||
unsupportedEncryptionVersion=Unsupported encryption version: {0} | unsupportedEncryptionVersion=Unsupported encryption version: {0} | ||||
unsupportedGC=Unsupported garbage collector for repository type: {0} | unsupportedGC=Unsupported garbage collector for repository type: {0} |
/* | |||||
* Copyright (C) 2021, Tencent. | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.errors; | |||||
import java.io.IOException; | |||||
/** | |||||
* Thrown when a commit-graph file's format is different than we expected | |||||
* | |||||
* @since 5.13 | |||||
*/ | |||||
public class CommitGraphFormatException extends IOException { | |||||
private static final long serialVersionUID = 1L; | |||||
/** | |||||
* Construct an exception. | |||||
* | |||||
* @param why | |||||
* description of the type of error. | |||||
*/ | |||||
public CommitGraphFormatException(String why) { | |||||
super(why); | |||||
} | |||||
} |
/***/ public String commandClosedStderrButDidntExit; | /***/ public String commandClosedStderrButDidntExit; | ||||
/***/ public String commandRejectedByHook; | /***/ public String commandRejectedByHook; | ||||
/***/ public String commandWasCalledInTheWrongState; | /***/ public String commandWasCalledInTheWrongState; | ||||
/***/ public String commitGraphChunkImproperOffset; | |||||
/***/ public String commitGraphChunkRepeated; | |||||
/***/ public String commitGraphChunkUnknown; | |||||
/***/ public String commitGraphFileIsTooLargeForJgit; | |||||
/***/ public String commitGraphGeneratingCancelledDuringWriting; | /***/ public String commitGraphGeneratingCancelledDuringWriting; | ||||
/***/ public String commitGraphOidFanoutNeeded; | |||||
/***/ public String commitMessageNotSpecified; | /***/ public String commitMessageNotSpecified; | ||||
/***/ public String commitOnRepoWithoutHEADCurrentlyNotSupported; | /***/ public String commitOnRepoWithoutHEADCurrentlyNotSupported; | ||||
/***/ public String commitAmendOnInitialNotPossible; | /***/ public String commitAmendOnInitialNotPossible; | ||||
/***/ public String noSuchSubmodule; | /***/ public String noSuchSubmodule; | ||||
/***/ public String notABoolean; | /***/ public String notABoolean; | ||||
/***/ public String notABundle; | /***/ public String notABundle; | ||||
/***/ public String notACommitGraph; | |||||
/***/ public String notADIRCFile; | /***/ public String notADIRCFile; | ||||
/***/ public String notAGitDirectory; | /***/ public String notAGitDirectory; | ||||
/***/ public String notAPACKFile; | /***/ public String notAPACKFile; | ||||
/***/ public String unmergedPath; | /***/ public String unmergedPath; | ||||
/***/ public String unmergedPaths; | /***/ public String unmergedPaths; | ||||
/***/ public String unpackException; | /***/ public String unpackException; | ||||
/***/ public String unreadableCommitGraph; | |||||
/***/ public String unreadablePackIndex; | /***/ public String unreadablePackIndex; | ||||
/***/ public String unrecognizedPackExtension; | /***/ public String unrecognizedPackExtension; | ||||
/***/ public String unrecognizedRef; | /***/ public String unrecognizedRef; | ||||
/***/ public String unsupportedAlternates; | /***/ public String unsupportedAlternates; | ||||
/***/ public String unsupportedArchiveFormat; | /***/ public String unsupportedArchiveFormat; | ||||
/***/ public String unsupportedCommand0; | /***/ public String unsupportedCommand0; | ||||
/***/ public String unsupportedCommitGraphVersion; | |||||
/***/ public String unsupportedEncryptionAlgorithm; | /***/ public String unsupportedEncryptionAlgorithm; | ||||
/***/ public String unsupportedEncryptionVersion; | /***/ public String unsupportedEncryptionVersion; | ||||
/***/ public String unsupportedGC; | /***/ public String unsupportedGC; |
/* | |||||
* Copyright (C) 2021, Tencent. | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.commitgraph; | |||||
import java.io.File; | |||||
import java.io.FileNotFoundException; | |||||
import java.io.IOException; | |||||
import java.io.InputStream; | |||||
import java.text.MessageFormat; | |||||
import org.eclipse.jgit.errors.CommitGraphFormatException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.CommitGraph; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.util.IO; | |||||
import org.eclipse.jgit.util.io.SilentFileInputStream; | |||||
/** | |||||
* <p> | |||||
* The commit-graph stores a list of commit OIDs and some associated metadata, | |||||
* including: | |||||
* <ol> | |||||
* <li>The generation number of the commit. Commits with no parents have | |||||
* generation number 1; commits with parents have generation number one more | |||||
* than the maximum generation number of its parents. We reserve zero as | |||||
* special, and can be used to mark a generation number invalid or as "not | |||||
* computed".</li> | |||||
* <li>The root tree OID.</li> | |||||
* <li>The commit date.</li> | |||||
* <li>The parents of the commit, stored using positional references within the | |||||
* graph file.</li> | |||||
* </ol> | |||||
* </p> | |||||
*/ | |||||
public abstract class CommitGraphData { | |||||
/** | |||||
* Open an existing commit-graph file for reading. | |||||
* <p> | |||||
* The format of the file will be automatically detected and a proper access | |||||
* implementation for that format will be constructed and returned to the | |||||
* caller. The file may or may not be held open by the returned instance. | |||||
* | |||||
* @param graphFile | |||||
* existing commit-graph to read. | |||||
* @return a copy of the commit-graph file in memory | |||||
* @throws FileNotFoundException | |||||
* the file does not exist. | |||||
* @throws CommitGraphFormatException | |||||
* commit-graph file's format is different than we expected. | |||||
* @throws java.io.IOException | |||||
* the file exists but could not be read due to security errors | |||||
* or unexpected data corruption. | |||||
*/ | |||||
public static CommitGraphData open(File graphFile) | |||||
throws FileNotFoundException, CommitGraphFormatException, | |||||
IOException { | |||||
try (SilentFileInputStream fd = new SilentFileInputStream(graphFile)) { | |||||
try { | |||||
return read(fd); | |||||
} catch (CommitGraphFormatException fe) { | |||||
throw fe; | |||||
} catch (IOException ioe) { | |||||
throw new IOException(MessageFormat.format( | |||||
JGitText.get().unreadableCommitGraph, | |||||
graphFile.getAbsolutePath()), ioe); | |||||
} | |||||
} | |||||
} | |||||
/** | |||||
* Read an existing commit-graph file from a buffered stream. | |||||
* <p> | |||||
* The format of the file will be automatically detected and a proper access | |||||
* implementation for that format will be constructed and returned to the | |||||
* caller. The file may or may not be held open by the returned instance. | |||||
* | |||||
* @param fd | |||||
* stream to read the commit-graph file from. The stream must be | |||||
* buffered as some small IOs are performed against the stream. | |||||
* The caller is responsible for closing the stream. | |||||
* | |||||
* @return a copy of the commit-graph file in memory | |||||
* @throws CommitGraphFormatException | |||||
* the commit-graph file's format is different than we expected. | |||||
* @throws java.io.IOException | |||||
* the stream cannot be read. | |||||
*/ | |||||
public static CommitGraphData read(InputStream fd) throws IOException { | |||||
byte[] hdr = new byte[8]; | |||||
IO.readFully(fd, hdr, 0, hdr.length); | |||||
int v = hdr[4]; | |||||
if (v != 1) { | |||||
throw new CommitGraphFormatException(MessageFormat.format( | |||||
JGitText.get().unsupportedCommitGraphVersion, | |||||
Integer.valueOf(v))); | |||||
} | |||||
return new CommitGraphDataV1(fd, hdr); | |||||
} | |||||
/** | |||||
* Finds the position in the commit-graph of the object. | |||||
* | |||||
* @param objId | |||||
* the id for which the commit-graph position will be found. | |||||
* @return the commit-graph id or -1 if the object was not found. | |||||
*/ | |||||
public abstract int findGraphPosition(AnyObjectId objId); | |||||
/** | |||||
* Get the object at the commit-graph position. | |||||
* | |||||
* @param graphPos | |||||
* the position in the commit-graph of the object. | |||||
* @return the ObjectId or null if the object was not found. | |||||
*/ | |||||
public abstract ObjectId getObjectId(int graphPos); | |||||
/** | |||||
* Get the metadata of a commit。 | |||||
* | |||||
* @param graphPos | |||||
* the position in the commit-graph of the object. | |||||
* @return the metadata of a commit or null if it's not found. | |||||
*/ | |||||
public abstract CommitGraph.CommitData getCommitData(int graphPos); | |||||
/** | |||||
* Obtain the total number of commits described by this commit-graph. | |||||
* | |||||
* @return number of commits in this commit-graph | |||||
*/ | |||||
public abstract long getCommitCnt(); | |||||
/** | |||||
* Get the hash length of this commit-graph | |||||
* | |||||
* @return object hash length | |||||
*/ | |||||
public abstract int getHashLength(); | |||||
static class CommitDataImpl implements CommitGraph.CommitData { | |||||
ObjectId tree; | |||||
int[] parents; | |||||
long commitTime; | |||||
int generation; | |||||
@Override | |||||
public ObjectId getTree() { | |||||
return tree; | |||||
} | |||||
@Override | |||||
public int[] getParents() { | |||||
return parents; | |||||
} | |||||
@Override | |||||
public long getCommitTime() { | |||||
return commitTime; | |||||
} | |||||
@Override | |||||
public int getGeneration() { | |||||
return generation; | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2021, Tencent. | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.commitgraph; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_DATA_EXTRA_LENGTH; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_GRAPH_MAGIC; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_CHUNK_LOOKUP_WIDTH; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EDGE_LAST_MASK; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EXTRA_EDGES_NEEDED; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_LAST_EDGE; | |||||
import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_NO_PARENT; | |||||
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; | |||||
import java.io.IOException; | |||||
import java.io.InputStream; | |||||
import java.text.MessageFormat; | |||||
import java.util.Arrays; | |||||
import org.eclipse.jgit.errors.CommitGraphFormatException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.CommitGraph; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.util.IO; | |||||
import org.eclipse.jgit.util.NB; | |||||
/** | |||||
* Support for the commit-graph v1 format. | |||||
* | |||||
* @see CommitGraphData | |||||
*/ | |||||
public class CommitGraphDataV1 extends CommitGraphData { | |||||
private static final int FANOUT = 256; | |||||
private final long commitCnt; | |||||
private final int hashLength; | |||||
private final int commitDataLength; | |||||
private long[] oidFanout; | |||||
private byte[][] oidLookup; | |||||
private byte[][] commitData; | |||||
private byte[] extraEdgeList; | |||||
CommitGraphDataV1(InputStream fd, byte[] hdr) throws IOException { | |||||
int magic = NB.decodeInt32(hdr, 0); | |||||
if (magic != COMMIT_GRAPH_MAGIC) { | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().notACommitGraph); | |||||
} | |||||
// Read the hash version (1 byte) | |||||
// 1 => SHA-1 | |||||
// 2 => SHA-256 nonsupport now | |||||
int hashVersion = hdr[5]; | |||||
if (hashVersion != 1) { | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().incorrectOBJECT_ID_LENGTH); | |||||
} | |||||
hashLength = OBJECT_ID_LENGTH; | |||||
commitDataLength = hashLength + COMMIT_DATA_EXTRA_LENGTH; | |||||
// Read the number of "chunkOffsets" (1 byte) | |||||
int numberOfChunks = hdr[6]; | |||||
byte[] chunkLookup = new byte[GRAPH_CHUNK_LOOKUP_WIDTH | |||||
* (numberOfChunks + 1)]; | |||||
IO.readFully(fd, chunkLookup, 0, chunkLookup.length); | |||||
int[] chunkId = new int[numberOfChunks + 1]; | |||||
long[] chunkOffset = new long[numberOfChunks + 1]; | |||||
for (int i = 0; i <= numberOfChunks; i++) { | |||||
chunkId[i] = NB.decodeInt32(chunkLookup, i * 12); | |||||
for (int j = 0; j < i; j++) { | |||||
if (chunkId[i] == chunkId[j]) { | |||||
throw new CommitGraphFormatException(MessageFormat.format( | |||||
JGitText.get().commitGraphChunkRepeated, | |||||
Integer.toHexString(chunkId[i]))); | |||||
} | |||||
} | |||||
chunkOffset[i] = NB.decodeInt64(chunkLookup, i * 12 + 4); | |||||
} | |||||
oidLookup = new byte[FANOUT][]; | |||||
commitData = new byte[FANOUT][]; | |||||
for (int i = 0; i < numberOfChunks; i++) { | |||||
long length = chunkOffset[i + 1] - chunkOffset[i]; | |||||
long lengthReaded; | |||||
if (chunkOffset[i] < 0 | |||||
|| chunkOffset[i] > chunkOffset[numberOfChunks]) { | |||||
throw new CommitGraphFormatException(MessageFormat.format( | |||||
JGitText.get().commitGraphChunkImproperOffset, | |||||
Integer.toHexString(chunkId[i]), | |||||
Long.valueOf(chunkOffset[i]))); | |||||
} | |||||
switch (chunkId[i]) { | |||||
case CHUNK_ID_OID_FANOUT: | |||||
lengthReaded = loadChunkOidFanout(fd); | |||||
break; | |||||
case CHUNK_ID_OID_LOOKUP: | |||||
lengthReaded = loadChunkDataBasedOnFanout(fd, hashLength, | |||||
oidLookup); | |||||
break; | |||||
case CHUNK_ID_COMMIT_DATA: | |||||
lengthReaded = loadChunkDataBasedOnFanout(fd, commitDataLength, | |||||
commitData); | |||||
break; | |||||
case CHUNK_ID_EXTRA_EDGE_LIST: | |||||
lengthReaded = loadChunkExtraEdgeList(fd, length); | |||||
break; | |||||
default: | |||||
throw new CommitGraphFormatException(MessageFormat.format( | |||||
JGitText.get().commitGraphChunkUnknown, | |||||
Integer.toHexString(chunkId[i]))); | |||||
} | |||||
if (length != lengthReaded) { | |||||
throw new CommitGraphFormatException(MessageFormat.format( | |||||
JGitText.get().commitGraphChunkImproperOffset, | |||||
Integer.toHexString(chunkId[i + 1]), | |||||
Long.valueOf(chunkOffset[i + 1]))); | |||||
} | |||||
} | |||||
if (oidFanout == null) { | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().commitGraphOidFanoutNeeded); | |||||
} | |||||
commitCnt = oidFanout[FANOUT - 1]; | |||||
} | |||||
private long loadChunkOidFanout(InputStream fd) throws IOException { | |||||
int fanoutLen = FANOUT * 4; | |||||
byte[] fanoutTable = new byte[fanoutLen]; | |||||
IO.readFully(fd, fanoutTable, 0, fanoutLen); | |||||
oidFanout = new long[FANOUT]; | |||||
for (int k = 0; k < oidFanout.length; k++) { | |||||
oidFanout[k] = NB.decodeUInt32(fanoutTable, k * 4); | |||||
} | |||||
return fanoutLen; | |||||
} | |||||
private long loadChunkDataBasedOnFanout(InputStream fd, int itemLength, | |||||
byte[][] chunkData) throws IOException { | |||||
if (oidFanout == null) { | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().commitGraphOidFanoutNeeded); | |||||
} | |||||
long readedLength = 0; | |||||
for (int k = 0; k < oidFanout.length; k++) { | |||||
long n; | |||||
if (k == 0) { | |||||
n = oidFanout[k]; | |||||
} else { | |||||
n = oidFanout[k] - oidFanout[k - 1]; | |||||
} | |||||
if (n > 0) { | |||||
long len = n * itemLength; | |||||
if (len > Integer.MAX_VALUE - 8) { // http://stackoverflow.com/a/8381338 | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().commitGraphFileIsTooLargeForJgit); | |||||
} | |||||
chunkData[k] = new byte[(int) len]; | |||||
IO.readFully(fd, chunkData[k], 0, chunkData[k].length); | |||||
readedLength += len; | |||||
} | |||||
} | |||||
return readedLength; | |||||
} | |||||
private long loadChunkExtraEdgeList(InputStream fd, long len) | |||||
throws IOException { | |||||
if (len > Integer.MAX_VALUE - 8) { // http://stackoverflow.com/a/8381338 | |||||
throw new CommitGraphFormatException( | |||||
JGitText.get().commitGraphFileIsTooLargeForJgit); | |||||
} | |||||
extraEdgeList = new byte[(int) len]; | |||||
IO.readFully(fd, extraEdgeList, 0, extraEdgeList.length); | |||||
return len; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public int findGraphPosition(AnyObjectId objId) { | |||||
int levelOne = objId.getFirstByte(); | |||||
byte[] data = oidLookup[levelOne]; | |||||
if (data == null) { | |||||
return -1; | |||||
} | |||||
int high = data.length / (hashLength); | |||||
int low = 0; | |||||
do { | |||||
int mid = (low + high) >>> 1; | |||||
int pos = objIdOffset(mid); | |||||
int cmp = objId.compareTo(data, pos); | |||||
if (cmp < 0) { | |||||
high = mid; | |||||
} else if (cmp == 0) { | |||||
if (levelOne == 0) { | |||||
return mid; | |||||
} | |||||
return (int) (mid + oidFanout[levelOne - 1]); | |||||
} else { | |||||
low = mid + 1; | |||||
} | |||||
} while (low < high); | |||||
return -1; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public ObjectId getObjectId(int graphPos) { | |||||
if (graphPos < 0 || graphPos > commitCnt) { | |||||
return null; | |||||
} | |||||
int levelOne = findLevelOne(graphPos); | |||||
int p = getLevelTwo(graphPos, levelOne); | |||||
int dataIdx = objIdOffset(p); | |||||
return ObjectId.fromRaw(oidLookup[levelOne], dataIdx); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public CommitGraph.CommitData getCommitData(int graphPos) { | |||||
int levelOne = findLevelOne(graphPos); | |||||
int p = getLevelTwo(graphPos, levelOne); | |||||
int dataIdx = commitDataOffset(p); | |||||
byte[] data = this.commitData[levelOne]; | |||||
if (graphPos < 0) { | |||||
return null; | |||||
} | |||||
CommitDataImpl commit = new CommitDataImpl(); | |||||
// parse tree | |||||
commit.tree = ObjectId.fromRaw(data, dataIdx); | |||||
// parse date | |||||
long dateHigh = NB.decodeInt32(data, dataIdx + hashLength + 8) & 0x3; | |||||
long dateLow = NB.decodeInt32(data, dataIdx + hashLength + 12); | |||||
commit.commitTime = dateHigh << 32 | dateLow; | |||||
// parse generation | |||||
commit.generation = NB.decodeInt32(data, dataIdx + hashLength + 8) >> 2; | |||||
boolean noParents = false; | |||||
int[] pList = new int[0]; | |||||
int edgeValue = NB.decodeInt32(data, dataIdx + hashLength); | |||||
if (edgeValue == GRAPH_NO_PARENT) { | |||||
noParents = true; | |||||
} | |||||
// parse parents | |||||
if (!noParents) { | |||||
pList = new int[1]; | |||||
int parent = edgeValue; | |||||
pList[0] = parent; | |||||
edgeValue = NB.decodeInt32(data, dataIdx + hashLength + 4); | |||||
if (edgeValue != GRAPH_NO_PARENT) { | |||||
if ((edgeValue & GRAPH_EXTRA_EDGES_NEEDED) != 0) { | |||||
int pptr = edgeValue & GRAPH_EDGE_LAST_MASK; | |||||
int[] eList = findExtraEdgeList(pptr); | |||||
if (eList == null) { | |||||
return null; | |||||
} | |||||
int[] old = pList; | |||||
pList = new int[eList.length + 1]; | |||||
pList[0] = old[0]; | |||||
for (int i = 0; i < eList.length; i++) { | |||||
parent = eList[i]; | |||||
pList[i + 1] = parent; | |||||
} | |||||
} else { | |||||
parent = edgeValue; | |||||
pList = new int[] { pList[0], parent }; | |||||
} | |||||
} | |||||
} | |||||
commit.parents = pList; | |||||
return commit; | |||||
} | |||||
/** | |||||
* Find the list of commit-graph position in extra edge list chunk. | |||||
* <p> | |||||
* The extra edge list chunk store the second through nth parents for all | |||||
* octopus merges. | |||||
* | |||||
* @param pptr | |||||
* the start position to iterate of extra edge list chunk | |||||
* @return the list of commit-graph position or null if not found | |||||
*/ | |||||
int[] findExtraEdgeList(int pptr) { | |||||
if (extraEdgeList == null) { | |||||
return null; | |||||
} | |||||
int maxOffset = extraEdgeList.length - 4; | |||||
int offset = pptr * 4; | |||||
if (offset < 0 || offset > maxOffset) { | |||||
return null; | |||||
} | |||||
int[] pList = new int[32]; | |||||
int count = 0; | |||||
int parentPosition; | |||||
for (;;) { | |||||
if (count >= pList.length) { | |||||
int[] old = pList; | |||||
pList = new int[pList.length + 32]; | |||||
System.arraycopy(old, 0, pList, 0, count); | |||||
} | |||||
if (offset > maxOffset) { | |||||
return null; | |||||
} | |||||
parentPosition = NB.decodeInt32(extraEdgeList, offset); | |||||
if ((parentPosition & GRAPH_LAST_EDGE) != 0) { | |||||
pList[count] = parentPosition & GRAPH_EDGE_LAST_MASK; | |||||
count++; | |||||
break; | |||||
} | |||||
pList[count++] = parentPosition; | |||||
offset += 4; | |||||
} | |||||
int[] old = pList; | |||||
pList = new int[count]; | |||||
System.arraycopy(old, 0, pList, 0, count); | |||||
return pList; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public long getCommitCnt() { | |||||
return commitCnt; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public int getHashLength() { | |||||
return hashLength; | |||||
} | |||||
private int findLevelOne(long nthPosition) { | |||||
int levelOne = Arrays.binarySearch(oidFanout, nthPosition + 1); | |||||
if (levelOne >= 0) { | |||||
// If we hit the bucket exactly the item is in the bucket, or | |||||
// any bucket before it which has the same object count. | |||||
// | |||||
long base = oidFanout[levelOne]; | |||||
while (levelOne > 0 && base == oidFanout[levelOne - 1]) { | |||||
levelOne--; | |||||
} | |||||
} else { | |||||
// The item is in the bucket we would insert it into. | |||||
// | |||||
levelOne = -(levelOne + 1); | |||||
} | |||||
return levelOne; | |||||
} | |||||
private int getLevelTwo(long nthPosition, int levelOne) { | |||||
long base = levelOne > 0 ? oidFanout[levelOne - 1] : 0; | |||||
return (int) (nthPosition - base); | |||||
} | |||||
private int objIdOffset(int mid) { | |||||
return hashLength * mid; | |||||
} | |||||
private int commitDataOffset(int mid) { | |||||
return commitDataLength * mid; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2021, Tencent. | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.commitgraph; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.CommitGraph; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
/** | |||||
* CommitGraph implementation by a single commit-graph file. | |||||
* | |||||
* @see CommitGraph | |||||
*/ | |||||
public class CommitGraphSingleImpl implements CommitGraph { | |||||
private final CommitGraphData graphData; | |||||
/** | |||||
* Creates CommitGraph by a single commit-graph file. | |||||
* | |||||
* @param graphData | |||||
* the commit-graph file in memory | |||||
*/ | |||||
public CommitGraphSingleImpl(CommitGraphData graphData) { | |||||
this.graphData = graphData; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public CommitData getCommitData(AnyObjectId commit) { | |||||
int graphPos = graphData.findGraphPosition(commit); | |||||
return getCommitData(graphPos); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public CommitData getCommitData(int graphPos) { | |||||
if (graphPos < 0 || graphPos > graphData.getCommitCnt()) { | |||||
return null; | |||||
} | |||||
return graphData.getCommitData(graphPos); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public ObjectId getObjectId(int graphPos) { | |||||
return graphData.getObjectId(graphPos); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public long getCommitCnt() { | |||||
return graphData.getCommitCnt(); | |||||
} | |||||
} |