Smart HTTP clients may request both multi_ack_detailed and no-done in the same request to prevent the client from needing to send a "done" line to the server in response to a server's "ACK %s ready". For smart HTTP, this can save 1 full HTTP RPC in the fetch exchange, improving overall latency when incrementally updating a client that has not diverged very far from the remote repository. Unfortuantely this capability cannot be enabled for the traditional bi-directional connections. multi_ack_detailed has the client sending more "have" lines at the same time that the server is creating the "ACK %s ready" and writing out the PACK stream, resulting in some race conditions and/or deadlock, depending on how the pipe buffers are implemented. For very small updates, a server might actually be able to send "ACK %s ready", then the PACK, and disconnect before the client even finishes sending its first batch of "have" lines. This may cause the client to fail with a broken pipe exception. To avoid all of these potential problems, "no-done" is restricted only to the smart HTTP variant of the protocol. Change-Id: Ie0d0a39320202bc096fec2e97cb58e9efd061b2d Signed-off-by: Shawn O. Pearce <spearce@spearce.org>tags/v0.12.1
ServiceNotEnabledException, ServiceNotAuthorizedException { | ServiceNotEnabledException, ServiceNotAuthorizedException { | ||||
UploadPack up = (UploadPack) req.getAttribute(ATTRIBUTE_HANDLER); | UploadPack up = (UploadPack) req.getAttribute(ATTRIBUTE_HANDLER); | ||||
try { | try { | ||||
up.setBiDirectionalPipe(false); | |||||
up.sendAdvertisedRefs(pck); | up.sendAdvertisedRefs(pck); | ||||
} finally { | } finally { | ||||
up.getRevWalk().release(); | up.getRevWalk().release(); |
.getRequestHeader(HDR_CONTENT_LENGTH)); | .getRequestHeader(HDR_CONTENT_LENGTH)); | ||||
assertNull("not chunked", service | assertNull("not chunked", service | ||||
.getRequestHeader(HDR_TRANSFER_ENCODING)); | .getRequestHeader(HDR_TRANSFER_ENCODING)); | ||||
assertNull("no compression (too small)", service | |||||
.getRequestHeader(HDR_CONTENT_ENCODING)); | |||||
assertEquals(200, service.getStatus()); | assertEquals(200, service.getStatus()); | ||||
assertEquals("application/x-git-upload-pack-result", service | assertEquals("application/x-git-upload-pack-result", service | ||||
} | } | ||||
@Test | @Test | ||||
public void testFetchUpdateExisting() throws Exception { | |||||
public void testFetch_FewLocalCommits() throws Exception { | |||||
// Bootstrap by doing the clone. | |||||
// | |||||
TestRepository dst = createTestRepository(); | |||||
Transport t = Transport.open(dst.getRepository(), remoteURI); | |||||
try { | |||||
t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); | |||||
} finally { | |||||
t.close(); | |||||
} | |||||
assertEquals(B, dst.getRepository().getRef(master).getObjectId()); | |||||
List<AccessEvent> cloneRequests = getRequests(); | |||||
// Only create a few new commits. | |||||
TestRepository.BranchBuilder b = dst.branch(master); | |||||
for (int i = 0; i < 4; i++) | |||||
b.commit().tick(3600 /* 1 hour */).message("c" + i).create(); | |||||
// Create a new commit on the remote. | |||||
// | |||||
b = new TestRepository(remoteRepository).branch(master); | |||||
RevCommit Z = b.commit().message("Z").create(); | |||||
// Now incrementally update. | |||||
// | |||||
t = Transport.open(dst.getRepository(), remoteURI); | |||||
try { | |||||
t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); | |||||
} finally { | |||||
t.close(); | |||||
} | |||||
assertEquals(Z, dst.getRepository().getRef(master).getObjectId()); | |||||
List<AccessEvent> requests = getRequests(); | |||||
requests.removeAll(cloneRequests); | |||||
assertEquals(2, requests.size()); | |||||
AccessEvent info = requests.get(0); | |||||
assertEquals("GET", info.getMethod()); | |||||
assertEquals(join(remoteURI, "info/refs"), info.getPath()); | |||||
assertEquals(1, info.getParameters().size()); | |||||
assertEquals("git-upload-pack", info.getParameter("service")); | |||||
assertEquals(200, info.getStatus()); | |||||
assertEquals("application/x-git-upload-pack-advertisement", | |||||
info.getResponseHeader(HDR_CONTENT_TYPE)); | |||||
// We should have needed one request to perform the fetch. | |||||
// | |||||
AccessEvent service = requests.get(1); | |||||
assertEquals("POST", service.getMethod()); | |||||
assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); | |||||
assertEquals(0, service.getParameters().size()); | |||||
assertNotNull("has content-length", | |||||
service.getRequestHeader(HDR_CONTENT_LENGTH)); | |||||
assertNull("not chunked", | |||||
service.getRequestHeader(HDR_TRANSFER_ENCODING)); | |||||
assertEquals(200, service.getStatus()); | |||||
assertEquals("application/x-git-upload-pack-result", | |||||
service.getResponseHeader(HDR_CONTENT_TYPE)); | |||||
} | |||||
@Test | |||||
public void testFetch_TooManyLocalCommits() throws Exception { | |||||
// Bootstrap by doing the clone. | // Bootstrap by doing the clone. | ||||
// | // | ||||
TestRepository dst = createTestRepository(); | TestRepository dst = createTestRepository(); |
static final String OPTION_NO_PROGRESS = "no-progress"; | static final String OPTION_NO_PROGRESS = "no-progress"; | ||||
static final String OPTION_NO_DONE = "no-done"; | |||||
static enum MultiAck { | static enum MultiAck { | ||||
OFF, CONTINUE, DETAILED; | OFF, CONTINUE, DETAILED; | ||||
} | } | ||||
private boolean allowOfsDelta; | private boolean allowOfsDelta; | ||||
private boolean noDone; | |||||
private String lockMessage; | private String lockMessage; | ||||
private PackLock packLock; | private PackLock packLock; | ||||
if (allowOfsDelta) | if (allowOfsDelta) | ||||
wantCapability(line, OPTION_OFS_DELTA); | wantCapability(line, OPTION_OFS_DELTA); | ||||
if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) | |||||
if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) { | |||||
multiAck = MultiAck.DETAILED; | multiAck = MultiAck.DETAILED; | ||||
else if (wantCapability(line, OPTION_MULTI_ACK)) | |||||
if (statelessRPC) | |||||
noDone = wantCapability(line, OPTION_NO_DONE); | |||||
} else if (wantCapability(line, OPTION_MULTI_ACK)) | |||||
multiAck = MultiAck.CONTINUE; | multiAck = MultiAck.CONTINUE; | ||||
else | else | ||||
multiAck = MultiAck.OFF; | multiAck = MultiAck.OFF; | ||||
int havesSinceLastContinue = 0; | int havesSinceLastContinue = 0; | ||||
boolean receivedContinue = false; | boolean receivedContinue = false; | ||||
boolean receivedAck = false; | boolean receivedAck = false; | ||||
boolean negotiate = true; | |||||
boolean receivedReady = false; | |||||
if (statelessRPC) | if (statelessRPC) | ||||
state.writeTo(out, null); | state.writeTo(out, null); | ||||
negotiateBegin(); | negotiateBegin(); | ||||
SEND_HAVES: while (negotiate) { | |||||
SEND_HAVES: while (!receivedReady) { | |||||
final RevCommit c = walk.next(); | final RevCommit c = walk.next(); | ||||
if (c == null) | if (c == null) | ||||
break SEND_HAVES; | break SEND_HAVES; | ||||
receivedContinue = true; | receivedContinue = true; | ||||
havesSinceLastContinue = 0; | havesSinceLastContinue = 0; | ||||
if (anr == AckNackResult.ACK_READY) | if (anr == AckNackResult.ACK_READY) | ||||
negotiate = false; | |||||
receivedReady = true; | |||||
break; | break; | ||||
} | } | ||||
if (monitor.isCancelled()) | if (monitor.isCancelled()) | ||||
throw new CancelledException(); | throw new CancelledException(); | ||||
// When statelessRPC is true we should always leave SEND_HAVES | |||||
// loop above while in the middle of a request. This allows us | |||||
// to just write done immediately. | |||||
// | |||||
pckOut.writeString("done\n"); | |||||
pckOut.flush(); | |||||
if (!receivedReady || !noDone) { | |||||
// When statelessRPC is true we should always leave SEND_HAVES | |||||
// loop above while in the middle of a request. This allows us | |||||
// to just write done immediately. | |||||
// | |||||
pckOut.writeString("done\n"); | |||||
pckOut.flush(); | |||||
} | |||||
if (!receivedAck) { | if (!receivedAck) { | ||||
// Apparently if we have never received an ACK earlier | // Apparently if we have never received an ACK earlier |
static final String OPTION_NO_PROGRESS = BasePackFetchConnection.OPTION_NO_PROGRESS; | static final String OPTION_NO_PROGRESS = BasePackFetchConnection.OPTION_NO_PROGRESS; | ||||
static final String OPTION_NO_DONE = BasePackFetchConnection.OPTION_NO_DONE; | |||||
/** Database we read the objects from. */ | /** Database we read the objects from. */ | ||||
private final Repository db; | private final Repository db; | ||||
/** null if {@link #commonBase} should be examined again. */ | /** null if {@link #commonBase} should be examined again. */ | ||||
private Boolean okToGiveUp; | private Boolean okToGiveUp; | ||||
private boolean sentReady; | |||||
/** Objects we sent in our advertisement list, clients can ask for these. */ | /** Objects we sent in our advertisement list, clients can ask for these. */ | ||||
private Set<ObjectId> advertised; | private Set<ObjectId> advertised; | ||||
private MultiAck multiAck = MultiAck.OFF; | private MultiAck multiAck = MultiAck.OFF; | ||||
private boolean noDone; | |||||
private PackWriter.Statistics statistics; | private PackWriter.Statistics statistics; | ||||
private UploadPackLogger logger; | private UploadPackLogger logger; | ||||
return; | return; | ||||
} | } | ||||
if (options.contains(OPTION_MULTI_ACK_DETAILED)) | |||||
if (options.contains(OPTION_MULTI_ACK_DETAILED)) { | |||||
multiAck = MultiAck.DETAILED; | multiAck = MultiAck.DETAILED; | ||||
else if (options.contains(OPTION_MULTI_ACK)) | |||||
noDone = options.contains(OPTION_NO_DONE); | |||||
} else if (options.contains(OPTION_MULTI_ACK)) | |||||
multiAck = MultiAck.CONTINUE; | multiAck = MultiAck.CONTINUE; | ||||
else | else | ||||
multiAck = MultiAck.OFF; | multiAck = MultiAck.OFF; | ||||
adv.advertiseCapability(OPTION_SIDE_BAND_64K); | adv.advertiseCapability(OPTION_SIDE_BAND_64K); | ||||
adv.advertiseCapability(OPTION_THIN_PACK); | adv.advertiseCapability(OPTION_THIN_PACK); | ||||
adv.advertiseCapability(OPTION_NO_PROGRESS); | adv.advertiseCapability(OPTION_NO_PROGRESS); | ||||
if (!biDirectionalPipe) | |||||
adv.advertiseCapability(OPTION_NO_DONE); | |||||
adv.setDerefTags(true); | adv.setDerefTags(true); | ||||
advertised = adv.send(getAdvertisedRefs()); | advertised = adv.send(getAdvertisedRefs()); | ||||
adv.end(); | adv.end(); | ||||
last = processHaveLines(peerHas, last); | last = processHaveLines(peerHas, last); | ||||
if (commonBase.isEmpty() || multiAck != MultiAck.OFF) | if (commonBase.isEmpty() || multiAck != MultiAck.OFF) | ||||
pckOut.writeString("NAK\n"); | pckOut.writeString("NAK\n"); | ||||
if (noDone && sentReady) { | |||||
pckOut.writeString("ACK " + last.name() + "\n"); | |||||
return true; | |||||
} | |||||
if (!biDirectionalPipe) | if (!biDirectionalPipe) | ||||
return false; | return false; | ||||
pckOut.flush(); | pckOut.flush(); | ||||
List<ObjectId> toParse = peerHas; | List<ObjectId> toParse = peerHas; | ||||
HashSet<ObjectId> peerHasSet = null; | HashSet<ObjectId> peerHasSet = null; | ||||
boolean needMissing = false; | boolean needMissing = false; | ||||
sentReady = false; | |||||
if (wantAll.isEmpty() && !wantIds.isEmpty()) { | if (wantAll.isEmpty() && !wantIds.isEmpty()) { | ||||
// We have not yet parsed the want list. Parse it now. | // We have not yet parsed the want list. Parse it now. | ||||
// telling us about its history. | // telling us about its history. | ||||
// | // | ||||
boolean didOkToGiveUp = false; | boolean didOkToGiveUp = false; | ||||
boolean sentReady = false; | |||||
if (0 < missCnt) { | if (0 < missCnt) { | ||||
for (int i = peerHas.size() - 1; i >= 0; i--) { | for (int i = peerHas.size() - 1; i >= 0; i--) { | ||||
ObjectId id = peerHas.get(i); | ObjectId id = peerHas.get(i); | ||||
if (multiAck == MultiAck.DETAILED && !didOkToGiveUp && okToGiveUp()) { | if (multiAck == MultiAck.DETAILED && !didOkToGiveUp && okToGiveUp()) { | ||||
ObjectId id = peerHas.get(peerHas.size() - 1); | ObjectId id = peerHas.get(peerHas.size() - 1); | ||||
sentReady = true; | |||||
pckOut.writeString("ACK " + id.name() + " ready\n"); | pckOut.writeString("ACK " + id.name() + " ready\n"); | ||||
sentReady = true; | sentReady = true; | ||||
} | } |