Browse Source

Shallow fetch/clone: Make --depth mean the total history depth

cgit changed the --depth parameter to mean the total depth of history
rather than the depth of ancestors to be returned [1]. JGit still uses
the latter meaning, so update it to match cgit.

depth=0 still means a non-shallow clone. depth=1 now means only the
wants rather than the wants and their direct parents.

This is accomplished by changing the semantic meaning of "depth" in
UploadPack and PackWriter to mean the total depth of history desired,
while keeping "depth" in DepthWalk.{RevWalk,ObjectWalk} to mean
the depth of traversal. Thus UploadPack and PackWriter always
initialize their DepthWalks with "depth-1".

[1] upload-pack: fix off-by-one depth calculation in shallow clone
https://code.googlesource.com/git/+/682c7d2f1a2d1a5443777237450505738af2ff1a

Change-Id: I87ed3c0f56c37e3491e367a41f5e555c4207ff44
Signed-off-by: Terry Parker <tparker@google.com>
tags/v4.5.0.201609210915-r
Terry Parker 7 years ago
parent
commit
e426aa8b90

+ 49
- 7
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java View File

} }


@Test @Test
public void testShallowIsMinimal() throws Exception {
public void testShallowIsMinimalDepth1() throws Exception {
FileRepository repo = setupRepoForShallowFetch(); FileRepository repo = setupRepoForShallowFetch();


PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE); PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE);
assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(),
contentA.getId(), contentB.getId()));

// Client already has blobs A and B, verify those are not packed.
idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2));
assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
contentC.getId(), contentD.getId(), contentE.getId()));
}

@Test
public void testShallowIsMinimalDepth2() throws Exception {
FileRepository repo = setupRepoForShallowFetch();

PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE);
assertContent(idx, assertContent(idx,
Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(), Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(),
c2.getTree().getId(), contentA.getId(), c2.getTree().getId(), contentA.getId(),
contentB.getId())); contentB.getId()));


// Client already has blobs A and B, verify those are not packed. // Client already has blobs A and B, verify those are not packed.
idx = writeShallowPack(repo, 1, wants(c5), haves(c1, c2), shallows(c1));
idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2), shallows(c1));
assertContent(idx, assertContent(idx,
Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
c5.getTree().getId(), contentC.getId(), c5.getTree().getId(), contentC.getId(),
} }


@Test @Test
public void testShallowFetchShallowParent() throws Exception {
public void testShallowFetchShallowParentDepth1() throws Exception {
FileRepository repo = setupRepoForShallowFetch(); FileRepository repo = setupRepoForShallowFetch();


PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
assertContent(idx,
Arrays.asList(c5.getId(), c5.getTree().getId(),
contentA.getId(), contentB.getId(), contentC.getId(),
contentD.getId(), contentE.getId()));

idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5));
assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId()));
}

@Test
public void testShallowFetchShallowParentDepth2() throws Exception {
FileRepository repo = setupRepoForShallowFetch();

PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
assertContent(idx, assertContent(idx,
Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
c5.getTree().getId(), contentA.getId(), c5.getTree().getId(), contentA.getId(),
contentB.getId(), contentC.getId(), contentD.getId(), contentB.getId(), contentC.getId(), contentD.getId(),
contentE.getId())); contentE.getId()));


idx = writeShallowPack(repo, 1, wants(c3), haves(c4, c5), shallows(c4));
idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5), shallows(c4));
assertContent(idx, Arrays.asList(c2.getId(), c3.getId(), assertContent(idx, Arrays.asList(c2.getId(), c3.getId(),
c2.getTree().getId(), c3.getTree().getId())); c2.getTree().getId(), c3.getTree().getId()));
} }


@Test @Test
public void testShallowFetchShallowAncestor() throws Exception {
public void testShallowFetchShallowAncestorDepth1() throws Exception {
FileRepository repo = setupRepoForShallowFetch(); FileRepository repo = setupRepoForShallowFetch();


PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
assertContent(idx,
Arrays.asList(c5.getId(), c5.getTree().getId(),
contentA.getId(), contentB.getId(), contentC.getId(),
contentD.getId(), contentE.getId()));

idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5));
assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId()));
}

@Test
public void testShallowFetchShallowAncestorDepth2() throws Exception {
FileRepository repo = setupRepoForShallowFetch();

PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
assertContent(idx, assertContent(idx,
Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
c5.getTree().getId(), contentA.getId(), c5.getTree().getId(), contentA.getId(),
contentB.getId(), contentC.getId(), contentD.getId(), contentB.getId(), contentC.getId(), contentD.getId(),
contentE.getId())); contentE.getId()));


idx = writeShallowPack(repo, 1, wants(c2), haves(c4, c5), shallows(c4));
idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5), shallows(c4));
assertContent(idx, Arrays.asList(c1.getId(), c2.getId(), assertContent(idx, Arrays.asList(c1.getId(), c2.getId(),
c1.getTree().getId(), c2.getTree().getId())); c1.getTree().getId(), c2.getTree().getId()));
} }
Set<? extends ObjectId> shallow) throws IOException { Set<? extends ObjectId> shallow) throws IOException {
// During negotiation, UploadPack would have set up a DepthWalk and // During negotiation, UploadPack would have set up a DepthWalk and
// marked the client's "shallow" commits. Emulate that here. // marked the client's "shallow" commits. Emulate that here.
DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth);
DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1);
walk.assumeShallow(shallow); walk.assumeShallow(shallow);
return writePack(repo, walk, depth, want, have, EMPTY_ID_SET); return writePack(repo, walk, depth, want, have, EMPTY_ID_SET);
} }

+ 1
- 0
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties View File

invalidChannel=Invalid channel {0} invalidChannel=Invalid channel {0}
invalidCharacterInBase64Data=Invalid character in Base64 data. invalidCharacterInBase64Data=Invalid character in Base64 data.
invalidCommitParentNumber=Invalid commit parent number invalidCommitParentNumber=Invalid commit parent number
invalidDepth=Invalid depth: {0}
invalidEncryption=Invalid encryption invalidEncryption=Invalid encryption
invalidExpandWildcard=ExpandFromSource on a refspec that can have mismatched wildcards does not make sense. invalidExpandWildcard=ExpandFromSource on a refspec that can have mismatched wildcards does not make sense.
invalidGitdirRef = Invalid .git reference in file ''{0}'' invalidGitdirRef = Invalid .git reference in file ''{0}''

+ 1
- 0
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java View File

/***/ public String invalidChannel; /***/ public String invalidChannel;
/***/ public String invalidCharacterInBase64Data; /***/ public String invalidCharacterInBase64Data;
/***/ public String invalidCommitParentNumber; /***/ public String invalidCommitParentNumber;
/***/ public String invalidDepth;
/***/ public String invalidEncryption; /***/ public String invalidEncryption;
/***/ public String invalidExpandWildcard; /***/ public String invalidExpandWildcard;
/***/ public String invalidGitdirRef; /***/ public String invalidGitdirRef;

+ 3
- 2
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java View File

* Configure this pack for a shallow clone. * Configure this pack for a shallow clone.
* *
* @param depth * @param depth
* maximum depth to traverse the commit graph
* maximum depth of history to return. 1 means return only the
* "wants".
* @param unshallow * @param unshallow
* objects which used to be shallow on the client, but are being * objects which used to be shallow on the client, but are being
* extended as part of this fetch * extended as part of this fetch
@NonNull Set<? extends ObjectId> have) throws IOException { @NonNull Set<? extends ObjectId> have) throws IOException {
ObjectWalk ow; ObjectWalk ow;
if (shallowPack) if (shallowPack)
ow = new DepthWalk.ObjectWalk(reader, depth);
ow = new DepthWalk.ObjectWalk(reader, depth - 1);
else else
ow = new ObjectWalk(reader); ow = new ObjectWalk(reader);
preparePack(countingMonitor, ow, want, have); preparePack(countingMonitor, ow, want, have);

+ 13
- 4
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java View File

} }


private void processShallow() throws IOException { private void processShallow() throws IOException {
int walkDepth = depth - 1;
try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk( try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
walk.getObjectReader(), depth)) {
walk.getObjectReader(), walkDepth)) {


// Find all the commits which will be shallow // Find all the commits which will be shallow
for (ObjectId o : wantIds) { for (ObjectId o : wantIds) {


// Commits at the boundary which aren't already shallow in // Commits at the boundary which aren't already shallow in
// the client need to be marked as such // the client need to be marked as such
if (c.getDepth() == depth && !clientShallowCommits.contains(c))
if (c.getDepth() == walkDepth
&& !clientShallowCommits.contains(c))
pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$ pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$


// Commits not on the boundary which are shallow in the client // Commits not on the boundary which are shallow in the client
// need to become unshallowed // need to become unshallowed
if (c.getDepth() < depth && clientShallowCommits.remove(c)) {
if (c.getDepth() < walkDepth
&& clientShallowCommits.remove(c)) {
unshallowCommits.add(c.copy()); unshallowCommits.add(c.copy());
pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$ pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
} }


if (line.startsWith("deepen ")) { //$NON-NLS-1$ if (line.startsWith("deepen ")) { //$NON-NLS-1$
depth = Integer.parseInt(line.substring(7)); depth = Integer.parseInt(line.substring(7));
if (depth <= 0) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidDepth,
Integer.valueOf(depth)));
}
continue; continue;
} }


} }


/** /**
* Returns the clone/fetch depth. Valid only after calling recvWants().
* Returns the clone/fetch depth. Valid only after calling recvWants(). A
* depth of 1 means return only the wants.
* *
* @return the depth requested by the client, or 0 if unbounded. * @return the depth requested by the client, or 0 if unbounded.
* @since 4.0 * @since 4.0

Loading…
Cancel
Save