123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516 |
- /*
- * Copyright (C) 2021, Tencent.
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Distribution License v. 1.0 which is available at
- * https://www.eclipse.org/org/documents/edl-v10.php.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
- package org.eclipse.jgit.internal.storage.commitgraph;
-
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_DATA_EXTRA_LENGTH;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_CHUNK_LOOKUP_WIDTH;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EXTRA_EDGES_NEEDED;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_LAST_EDGE;
- import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_NO_PARENT;
-
- import java.io.IOException;
- import java.io.OutputStream;
- import java.nio.ByteBuffer;
- import java.text.MessageFormat;
- import java.util.Collections;
- import java.util.List;
- import java.util.Set;
- import java.util.Stack;
-
- import org.eclipse.jgit.annotations.NonNull;
- import org.eclipse.jgit.errors.MissingObjectException;
- import org.eclipse.jgit.internal.JGitText;
- import org.eclipse.jgit.lib.CommitGraph;
- import org.eclipse.jgit.lib.Constants;
- import org.eclipse.jgit.lib.NullProgressMonitor;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.lib.ObjectIdOwnerMap;
- import org.eclipse.jgit.lib.ObjectReader;
- import org.eclipse.jgit.lib.ProgressMonitor;
- import org.eclipse.jgit.lib.Repository;
- import org.eclipse.jgit.revwalk.ObjectWalk;
- import org.eclipse.jgit.revwalk.RevCommit;
- import org.eclipse.jgit.revwalk.RevObject;
- import org.eclipse.jgit.revwalk.RevSort;
- import org.eclipse.jgit.util.BlockList;
- import org.eclipse.jgit.util.NB;
-
- /**
- * Writes a commit-graph formatted file.
- */
- public class CommitGraphWriter {
-
- private static final int COMMIT_GRAPH_VERSION_GENERATED = 1;
-
- private static final int OID_HASH_VERSION = 1;
-
- private static final int GENERATION_NUMBER_MAX = 0x3FFFFFFF;
-
- private static final int MAX_NUM_CHUNKS = 5;
-
- private static final int GRAPH_FANOUT_SIZE = 4 * 256;
-
- private static final int OID_HASH_LENGTH = Constants.OBJECT_ID_LENGTH;
-
- private final ObjectWalk walk;
-
- private List<ObjectToCommitData> commitDataList = new BlockList<>();
-
- private List<ObjectToCommitData> commitDataSortedByName;
-
- private ObjectIdOwnerMap<ObjectToCommitData> commitDataMap = new ObjectIdOwnerMap<>();
-
- private int numExtraEdges;
-
- private boolean computeGeneration;
-
- /**
- * Create writer for specified repository.
- *
- * @param repo
- * repository where objects are stored.
- */
- public CommitGraphWriter(Repository repo) {
- this(repo, repo.newObjectReader());
- }
-
- /**
- * Create writer for specified repository.
- *
- * @param repo
- * repository where objects are stored.
- * @param reader
- * reader to read from the repository with.
- */
- public CommitGraphWriter(Repository repo, ObjectReader reader) {
- this(new CommitGraphConfig(repo), reader);
- }
-
- /**
- * Create writer with a specified configuration.
- *
- * @param cfg
- * configuration for the commit-graph writer.
- * @param reader
- * reader to read from the repository with.
- */
- public CommitGraphWriter(CommitGraphConfig cfg, ObjectReader reader) {
- this.walk = new ObjectWalk(reader);
- this.computeGeneration = cfg.isComputeGeneration();
- }
-
- /**
- * Prepare the list of commits to be written to the commit-graph stream.
- *
- * @param findingMonitor
- * progress monitor to report the number of commits found.
- * @param computeGenerationMonitor
- * progress monitor to report generation computation work.
- * @param wants
- * the list of wanted objects, writer walks commits starting at
- * these. Must not be {@code null}.
- * @throws IOException
- */
- public void prepareCommitGraph(ProgressMonitor findingMonitor,
- ProgressMonitor computeGenerationMonitor,
- @NonNull Set<? extends ObjectId> wants) throws IOException {
- BlockList<RevCommit> commits = findCommits(findingMonitor, walk, wants);
- if (computeGeneration) {
- computeGenerationNumbers(computeGenerationMonitor, commits);
- }
- }
-
- /**
- * Write the prepared commits to the supplied stream.
- *
- * @param writeMonitor
- * progress monitor to report the number of items written.
- * @param commitGraphStream
- * output stream of commit-graph data. The stream should be
- * buffered by the caller. The caller is responsible for closing
- * the stream.
- * @throws IOException
- */
- public void writeCommitGraph(ProgressMonitor writeMonitor,
- OutputStream commitGraphStream) throws IOException {
- if (writeMonitor == null) {
- writeMonitor = NullProgressMonitor.INSTANCE;
- }
-
- ChunkInfo[] chunks = new ChunkInfo[MAX_NUM_CHUNKS];
- for (int i = 0; i < chunks.length; i++) {
- chunks[i] = new ChunkInfo();
- }
- int numChunks = 3;
- int hashsz = OID_HASH_LENGTH;
- long writeCount = 0;
- long chunkOffset;
- CommitGraphOutPutStream out = new CommitGraphOutPutStream(writeMonitor,
- commitGraphStream);
-
- chunks[0].id = CHUNK_ID_OID_FANOUT;
- chunks[0].size = GRAPH_FANOUT_SIZE;
- writeCount += 256;
-
- chunks[1].id = CHUNK_ID_OID_LOOKUP;
- chunks[1].size = hashsz * commitDataList.size();
- writeCount += commitDataList.size();
-
- chunks[2].id = CHUNK_ID_COMMIT_DATA;
- chunks[2].size = (hashsz + 16) * commitDataList.size();
- writeCount += commitDataList.size();
-
- if (numExtraEdges > 0) {
- chunks[numChunks].id = CHUNK_ID_EXTRA_EDGE_LIST;
- chunks[numChunks].size = numExtraEdges * 4;
- writeCount += numExtraEdges;
- numChunks++;
- }
- chunks[numChunks].id = 0;
- chunks[numChunks].size = 0L;
-
- beginPhase(MessageFormat.format(JGitText.get().writingOutCommitGraph,
- Integer.valueOf(numChunks)), writeMonitor, writeCount);
-
- try {
- // write header
- out.writeFileHeader(getVersion(), OID_HASH_VERSION, numChunks);
- out.flush();
-
- // write chunk lookup
- chunkOffset = 8 + (numChunks + 1) * GRAPH_CHUNK_LOOKUP_WIDTH;
- for (int i = 0; i <= numChunks; i++) {
- ChunkInfo chunk = chunks[i];
-
- ByteBuffer buffer = ByteBuffer
- .allocate(GRAPH_CHUNK_LOOKUP_WIDTH);
- buffer.putInt(chunk.id);
- buffer.putLong(chunkOffset);
- out.write(buffer.array());
- chunkOffset += chunk.size;
- }
-
- // write chunks
- for (int i = 0; i < numChunks; i++) {
- int chunkId = chunks[i].id;
-
- switch (chunkId) {
- case CHUNK_ID_OID_FANOUT:
- writeFanoutTable(out);
- break;
- case CHUNK_ID_OID_LOOKUP:
- writeOidLookUp(out, hashsz);
- break;
- case CHUNK_ID_COMMIT_DATA:
- writeCommitData(out, hashsz);
- break;
- case CHUNK_ID_EXTRA_EDGE_LIST:
- writeExtraEdges(out);
- break;
- }
- }
-
- // write check sum
- out.write(out.getDigest());
- out.flush();
- } finally {
- endPhase(writeMonitor);
- }
- }
-
- /**
- * Returns commits number that was created by this writer.
- *
- * @return number of commits.
- */
- public long getCommitCnt() {
- return commitDataList.size();
- }
-
- /**
- * Whether to compute generation numbers.
- *
- * Default setting: {@value CommitGraphConfig#DEFAULT_COMPUTE_GENERATION}
- *
- * @return {@code true} if the writer should compute generation numbers.
- */
- public boolean isComputeGeneration() {
- return computeGeneration;
- }
-
- /**
- * Whether the writer should compute generation numbers.
- *
- * Default setting: {@value CommitGraphConfig#DEFAULT_COMPUTE_GENERATION}
- *
- * @param computeGeneration
- * if {@code true} the commits in commit-graph will have the
- * computed generation number.
- */
- public void setComputeGeneration(boolean computeGeneration) {
- this.computeGeneration = computeGeneration;
- }
-
- /**
- * Whether to write the extra edge list.
- * <p>
- * This list of 4-byte values store the second through nth parents for all
- * octopus merges.
- *
- * @return {@code true} if the writer will write the extra edge list.
- */
- public boolean willWriteExtraEdgeList() {
- return numExtraEdges > 0;
- }
-
- private void writeFanoutTable(CommitGraphOutPutStream out)
- throws IOException {
- byte[] tmp = new byte[4];
- int[] fanout = new int[256];
- for (ObjectToCommitData oc : commitDataList)
- fanout[oc.getFirstByte() & 0xff]++;
- for (int i = 1; i < 256; i++)
- fanout[i] += fanout[i - 1];
- for (int n : fanout) {
- NB.encodeInt32(tmp, 0, n);
- out.write(tmp, 0, 4);
- out.updateMonitor();
- }
- }
-
- private void writeOidLookUp(CommitGraphOutPutStream out, int hashsz)
- throws IOException {
- byte[] tmp = new byte[4 + hashsz];
- List<ObjectToCommitData> sortedByName = commitDataSortByName();
-
- for (int i = 0; i < sortedByName.size(); i++) {
- ObjectToCommitData commitData = sortedByName.get(i);
- commitData.setOidPosition(i);
- commitData.copyRawTo(tmp, 0);
- out.write(tmp, 0, hashsz);
- out.updateMonitor();
- }
- commitDataList = sortedByName;
- }
-
- private void writeCommitData(CommitGraphOutPutStream out, int hashsz)
- throws IOException {
- int numExtraEdges = 0;
- byte[] tmp = new byte[hashsz + COMMIT_DATA_EXTRA_LENGTH];
- for (ObjectToCommitData oc : commitDataList) {
- int edgeValue;
- int[] packedDate = new int[2];
-
- RevCommit commit = walk.parseCommit(oc);
- ObjectId treeId = commit.getTree();
- treeId.copyRawTo(tmp, 0);
-
- RevCommit[] parents = commit.getParents();
- if (parents.length == 0) {
- edgeValue = GRAPH_NO_PARENT;
- } else {
- RevCommit parent = parents[0];
- edgeValue = getCommitOidPosition(parent);
- }
- NB.encodeInt32(tmp, hashsz, edgeValue);
- if (parents.length == 1) {
- edgeValue = GRAPH_NO_PARENT;
- } else if (parents.length == 2) {
- RevCommit parent = parents[1];
- edgeValue = getCommitOidPosition(parent);
- } else if (parents.length > 2) {
- edgeValue = GRAPH_EXTRA_EDGES_NEEDED | numExtraEdges;
- numExtraEdges += parents.length - 1;
- }
-
- NB.encodeInt32(tmp, hashsz + 4, edgeValue);
-
- packedDate[0] = 0; // commitTime is an int in JGit now
- packedDate[0] |= oc.getGeneration() << 2;
- packedDate[1] = commit.getCommitTime();
- NB.encodeInt32(tmp, hashsz + 8, packedDate[0]);
- NB.encodeInt32(tmp, hashsz + 12, packedDate[1]);
-
- out.write(tmp);
- out.updateMonitor();
- }
- }
-
- private void writeExtraEdges(CommitGraphOutPutStream out)
- throws IOException {
- byte[] tmp = new byte[4];
- for (ObjectToCommitData oc : commitDataList) {
- RevCommit commit = walk.parseCommit(oc);
- RevCommit[] parents = commit.getParents();
- if (parents.length > 2) {
- int edgeValue;
- for (int n = 1; n < parents.length; n++) {
- RevCommit parent = parents[n];
- edgeValue = getCommitOidPosition(parent);
- if (n == parents.length - 1) {
- edgeValue |= GRAPH_LAST_EDGE;
- }
- NB.encodeInt32(tmp, 0, edgeValue);
- out.write(tmp);
- out.updateMonitor();
- }
- }
- }
- }
-
- private BlockList<RevCommit> findCommits(ProgressMonitor findingMonitor,
- ObjectWalk walk, Set<? extends ObjectId> wants) throws IOException {
- if (findingMonitor == null) {
- findingMonitor = NullProgressMonitor.INSTANCE;
- }
-
- for (ObjectId id : wants) {
- RevObject o = walk.parseAny(id);
- if (o instanceof RevCommit) {
- walk.markStart((RevCommit) o);
- }
- }
-
- walk.sort(RevSort.COMMIT_TIME_DESC);
- BlockList<RevCommit> commits = new BlockList<>();
-
- RevCommit c;
- beginPhase(JGitText.get().findingCommitsForCommitGraph, findingMonitor,
- ProgressMonitor.UNKNOWN);
- while ((c = walk.next()) != null) {
- findingMonitor.update(1);
- commits.add(c);
- addCommitData(c);
- if (c.getParentCount() > 2) {
- numExtraEdges += c.getParentCount() - 1;
- }
- }
- endPhase(findingMonitor);
-
- return commits;
- }
-
- private void computeGenerationNumbers(
- ProgressMonitor computeGenerationMonitor, List<RevCommit> commits)
- throws MissingObjectException {
- if (computeGenerationMonitor == null) {
- computeGenerationMonitor = NullProgressMonitor.INSTANCE;
- }
-
- beginPhase(JGitText.get().computingCommitGeneration,
- computeGenerationMonitor, commits.size());
- for (RevCommit cmit : commits) {
- computeGenerationMonitor.update(1);
- int generation = getCommitGeneration(cmit);
- if (generation != CommitGraph.GENERATION_NUMBER_ZERO
- && generation != CommitGraph.GENERATION_NUMBER_INFINITY) {
- continue;
- }
-
- Stack<RevCommit> commitStack = new Stack<>();
- commitStack.push(cmit);
-
- while (!commitStack.empty()) {
- int maxGeneration = 0;
- boolean allParentComputed = true;
- RevCommit current = commitStack.peek();
- RevCommit parent;
-
- for (int i = 0; i < current.getParentCount(); i++) {
- parent = current.getParent(i);
- generation = getCommitGeneration(parent);
- if (generation == CommitGraph.GENERATION_NUMBER_ZERO
- || generation == CommitGraph.GENERATION_NUMBER_INFINITY) {
- allParentComputed = false;
- commitStack.push(parent);
- break;
- } else if (generation > maxGeneration) {
- maxGeneration = generation;
- }
- }
-
- if (allParentComputed) {
- RevCommit commit = commitStack.pop();
- generation = maxGeneration + 1;
- if (generation > GENERATION_NUMBER_MAX) {
- generation = GENERATION_NUMBER_MAX;
- }
- setCommitGeneration(commit, generation);
- }
- }
- }
- endPhase(computeGenerationMonitor);
- }
-
- private int getVersion() {
- return COMMIT_GRAPH_VERSION_GENERATED;
- }
-
- private static class ChunkInfo {
- int id;
-
- long size;
- }
-
- private int getCommitGeneration(RevCommit commit)
- throws MissingObjectException {
- ObjectToCommitData oc = commitDataMap.get(commit);
- if (oc == null) {
- throw new MissingObjectException(commit, Constants.OBJ_COMMIT);
- }
- return oc.getGeneration();
- }
-
- private void setCommitGeneration(RevCommit commit, int generation)
- throws MissingObjectException {
- ObjectToCommitData oc = commitDataMap.get(commit);
- if (oc == null) {
- throw new MissingObjectException(commit, Constants.OBJ_COMMIT);
- }
- oc.setGeneration(generation);
- }
-
- private int getCommitOidPosition(RevCommit commit)
- throws MissingObjectException {
- ObjectToCommitData oc = commitDataMap.get(commit);
- if (oc == null) {
- throw new MissingObjectException(commit, Constants.OBJ_COMMIT);
- }
- return oc.getOidPosition();
- }
-
- private void addCommitData(RevCommit commit) {
- ObjectToCommitData otc = new ObjectToCommitData(commit);
- commitDataList.add(otc);
- commitDataMap.add(otc);
- }
-
- private List<ObjectToCommitData> commitDataSortByName() {
- if (commitDataSortedByName == null) {
- commitDataSortedByName = new BlockList<>(commitDataList.size());
- commitDataSortedByName.addAll(commitDataList);
- Collections.sort(commitDataSortedByName);
- }
- return commitDataSortedByName;
- }
-
- private void beginPhase(String task, ProgressMonitor monitor, long cnt) {
- monitor.beginTask(task, (int) cnt);
- }
-
- private void endPhase(ProgressMonitor monitor) {
- monitor.endTask();
- }
- }
|