Teach UploadPack to take a provider of URIs corresponding to cached packs. When fetching, if the client supports the packfile-uri feature, and if such a cached pack were to be streamed, instead send the corresponding URI. This packfile-uri feature is implemented in the jt/fetch-cdn-offload branch of Git. There is interest in this feature [1], but it is not yet merged. [1] https://public-inbox.org/git/cover.1552073690.git.jonathantanmy@google.com/ Change-Id: I9a32dae131c9c56ad2ff4a8a9638ae3b5e44dc15 Signed-off-by: Jonathan Tan <jonathantanmy@google.com>tags/v5.5.0.201908280940-m3
@@ -35,6 +35,8 @@ import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector; | |||
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; | |||
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; | |||
import org.eclipse.jgit.internal.storage.file.PackLock; | |||
import org.eclipse.jgit.internal.storage.pack.CachedPack; | |||
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; | |||
import org.eclipse.jgit.junit.TestRepository; | |||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
@@ -2149,6 +2151,58 @@ public class UploadPackTest { | |||
parsePack(recvStream); | |||
} | |||
@Test | |||
public void testV2FetchPackfileUris() throws Exception { | |||
// Inside the pack | |||
RevCommit commit = remote.commit().message("x").create(); | |||
remote.update("master", commit); | |||
generateBitmaps(server); | |||
// Outside the pack | |||
RevCommit commit2 = remote.commit().message("x").parent(commit).create(); | |||
remote.update("master", commit2); | |||
server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true); | |||
ByteArrayInputStream recvStream = uploadPackV2( | |||
(UploadPack up) -> { | |||
up.setCachedPackUriProvider(new CachedPackUriProvider() { | |||
@Override | |||
public PackInfo getInfo(CachedPack pack, | |||
Collection<String> protocolsSupported) | |||
throws IOException { | |||
assertThat(protocolsSupported, hasItems("https")); | |||
if (!protocolsSupported.contains("https")) | |||
return null; | |||
return new PackInfo("myhash", "myuri"); | |||
} | |||
}); | |||
}, | |||
"command=fetch\n", | |||
PacketLineIn.DELIM, | |||
"want " + commit2.getName() + "\n", | |||
"sideband-all\n", | |||
"packfile-uris https\n", | |||
"done\n", | |||
PacketLineIn.END); | |||
PacketLineIn pckIn = new PacketLineIn(recvStream); | |||
String s; | |||
// skip all \002 strings | |||
for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) { | |||
// do nothing | |||
} | |||
assertThat(s, is("\001packfile-uris")); | |||
assertThat(pckIn.readString(), is("\001myhash myuri")); | |||
assertTrue(PacketLineIn.isDelimiter(pckIn.readString())); | |||
assertThat(pckIn.readString(), is("\001packfile")); | |||
parsePack(recvStream); | |||
assertFalse(client.getObjectDatabase().has(commit.toObjectId())); | |||
assertTrue(client.getObjectDatabase().has(commit2.toObjectId())); | |||
} | |||
@Test | |||
public void testGetPeerAgentProtocolV0() throws Exception { | |||
RevCommit one = remote.commit().message("1").create(); |
@@ -312,6 +312,14 @@ public abstract class PackIndex | |||
public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, | |||
int matchLimit) throws IOException; | |||
/** | |||
* @return the checksum of the pack; caller must not modify it | |||
* @since 5.5 | |||
*/ | |||
public byte[] getChecksum() { | |||
return packChecksum; | |||
} | |||
/** | |||
* Represent mutable entry of pack index consisting of object id and offset | |||
* in pack (both mutable). |
@@ -0,0 +1,103 @@ | |||
/* | |||
* Copyright (C) 2019, Google LLC. | |||
* and other copyright owners as documented in the project's IP log. | |||
* | |||
* This program and the accompanying materials are made available | |||
* under the terms of the Eclipse Distribution License v1.0 which | |||
* accompanies this distribution, is reproduced below, and is | |||
* available at http://www.eclipse.org/org/documents/edl-v10.php | |||
* | |||
* All rights reserved. | |||
* | |||
* Redistribution and use in source and binary forms, with or | |||
* without modification, are permitted provided that the following | |||
* conditions are met: | |||
* | |||
* - Redistributions of source code must retain the above copyright | |||
* notice, this list of conditions and the following disclaimer. | |||
* | |||
* - Redistributions in binary form must reproduce the above | |||
* copyright notice, this list of conditions and the following | |||
* disclaimer in the documentation and/or other materials provided | |||
* with the distribution. | |||
* | |||
* - Neither the name of the Eclipse Foundation, Inc. nor the | |||
* names of its contributors may be used to endorse or promote | |||
* products derived from this software without specific prior | |||
* written permission. | |||
* | |||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND | |||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, | |||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR | |||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | |||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
*/ | |||
package org.eclipse.jgit.internal.storage.pack; | |||
import java.io.IOException; | |||
import java.util.Collection; | |||
import org.eclipse.jgit.annotations.Nullable; | |||
/** | |||
* Provider of URIs corresponding to cached packs. For use with the | |||
* "packfile-uris" feature. | |||
* @since 5.5 | |||
*/ | |||
public interface CachedPackUriProvider { | |||
/** | |||
* @param pack the cached pack for which to check if a corresponding URI | |||
* exists | |||
* @param protocolsSupported the protocols that the client has declared | |||
* support for; if a URI is returned, it must be of one of these | |||
* protocols | |||
* @throws IOException implementations may throw this | |||
* @return if a URI corresponds to the cached pack, an object | |||
* containing the URI and some other information; null otherwise | |||
* @since 5.5 | |||
*/ | |||
@Nullable | |||
PackInfo getInfo(CachedPack pack, Collection<String> protocolsSupported) | |||
throws IOException; | |||
/** | |||
* Information about a packfile. | |||
* @since 5.5 | |||
*/ | |||
public static class PackInfo { | |||
private final String hash; | |||
private final String uri; | |||
/** | |||
* Constructs an object containing information about a packfile. | |||
* @param hash the hash of the packfile as a hexadecimal string | |||
* @param uri the URI corresponding to the packfile | |||
*/ | |||
public PackInfo(String hash, String uri) { | |||
this.hash = hash; | |||
this.uri = uri; | |||
} | |||
/** | |||
* @return the hash of the packfile as a hexadecimal string | |||
*/ | |||
public String getHash() { | |||
return hash; | |||
} | |||
/** | |||
* @return the URI corresponding to the packfile | |||
*/ | |||
public String getUri() { | |||
return uri; | |||
} | |||
} | |||
} |
@@ -122,6 +122,7 @@ import org.eclipse.jgit.storage.pack.PackConfig; | |||
import org.eclipse.jgit.storage.pack.PackStatistics; | |||
import org.eclipse.jgit.transport.FilterSpec; | |||
import org.eclipse.jgit.transport.ObjectCountCallback; | |||
import org.eclipse.jgit.transport.PacketLineOut; | |||
import org.eclipse.jgit.transport.WriteAbortedException; | |||
import org.eclipse.jgit.util.BlockList; | |||
import org.eclipse.jgit.util.TemporaryBuffer; | |||
@@ -307,6 +308,8 @@ public class PackWriter implements AutoCloseable { | |||
private FilterSpec filterSpec = FilterSpec.NO_FILTER; | |||
private PackfileUriConfig packfileUriConfig; | |||
/** | |||
* Create writer for specified repository. | |||
* <p> | |||
@@ -650,6 +653,14 @@ public class PackWriter implements AutoCloseable { | |||
filterSpec = requireNonNull(filter); | |||
} | |||
/** | |||
* @param config configuration related to packfile URIs | |||
* @since 5.5 | |||
*/ | |||
public void setPackfileUriConfig(PackfileUriConfig config) { | |||
packfileUriConfig = config; | |||
} | |||
/** | |||
* Returns objects number in a pack file that was created by this writer. | |||
* | |||
@@ -673,6 +684,26 @@ public class PackWriter implements AutoCloseable { | |||
return stats.totalObjects; | |||
} | |||
private long getUnoffloadedObjectCount() throws IOException { | |||
long objCnt = 0; | |||
objCnt += objectsLists[OBJ_COMMIT].size(); | |||
objCnt += objectsLists[OBJ_TREE].size(); | |||
objCnt += objectsLists[OBJ_BLOB].size(); | |||
objCnt += objectsLists[OBJ_TAG].size(); | |||
for (CachedPack pack : cachedPacks) { | |||
CachedPackUriProvider.PackInfo packInfo = | |||
packfileUriConfig.cachedPackUriProvider.getInfo( | |||
pack, packfileUriConfig.protocolsSupported); | |||
if (packInfo == null) { | |||
objCnt += pack.getObjectCount(); | |||
} | |||
} | |||
return objCnt; | |||
} | |||
/** | |||
* Returns the object ids in the pack file that was created by this writer. | |||
* <p> | |||
@@ -1177,13 +1208,38 @@ public class PackWriter implements AutoCloseable { | |||
: new CheckedOutputStream(packStream, crc32), | |||
this); | |||
long objCnt = getObjectCount(); | |||
long objCnt = packfileUriConfig == null ? getObjectCount() : | |||
getUnoffloadedObjectCount(); | |||
stats.totalObjects = objCnt; | |||
if (callback != null) | |||
callback.setObjectCount(objCnt); | |||
beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); | |||
long writeStart = System.currentTimeMillis(); | |||
try { | |||
List<CachedPack> unwrittenCachedPacks; | |||
if (packfileUriConfig != null) { | |||
unwrittenCachedPacks = new ArrayList<>(); | |||
CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider; | |||
PacketLineOut o = packfileUriConfig.pckOut; | |||
o.writeString("packfile-uris\n"); | |||
for (CachedPack pack : cachedPacks) { | |||
CachedPackUriProvider.PackInfo packInfo = p.getInfo( | |||
pack, packfileUriConfig.protocolsSupported); | |||
if (packInfo != null) { | |||
o.writeString(packInfo.getHash() + ' ' + | |||
packInfo.getUri() + '\n'); | |||
} else { | |||
unwrittenCachedPacks.add(pack); | |||
} | |||
} | |||
packfileUriConfig.pckOut.writeDelim(); | |||
packfileUriConfig.pckOut.writeString("packfile\n"); | |||
} else { | |||
unwrittenCachedPacks = cachedPacks; | |||
} | |||
out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); | |||
out.flush(); | |||
@@ -1197,7 +1253,7 @@ public class PackWriter implements AutoCloseable { | |||
} | |||
stats.reusedPacks = Collections.unmodifiableList(cachedPacks); | |||
for (CachedPack pack : cachedPacks) { | |||
for (CachedPack pack : unwrittenCachedPacks) { | |||
long deltaCnt = pack.getDeltaCount(); | |||
stats.reusedObjects += pack.getObjectCount(); | |||
stats.reusedDeltas += deltaCnt; | |||
@@ -2426,4 +2482,37 @@ public class PackWriter implements AutoCloseable { | |||
return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; | |||
} | |||
} | |||
/** | |||
* Configuration related to the packfile URI feature. | |||
* | |||
* @since 5.5 | |||
*/ | |||
public static class PackfileUriConfig { | |||
@NonNull | |||
private final PacketLineOut pckOut; | |||
@NonNull | |||
private final Collection<String> protocolsSupported; | |||
@NonNull | |||
private final CachedPackUriProvider cachedPackUriProvider; | |||
/** | |||
* @param pckOut where to write "packfile-uri" lines to (should | |||
* output to the same stream as the one passed to | |||
* PackWriter#writePack) | |||
* @param protocolsSupported list of protocols supported (e.g. "https") | |||
* @param cachedPackUriProvider provider of URIs corresponding | |||
* to cached packs | |||
* @since 5.5 | |||
*/ | |||
public PackfileUriConfig(@NonNull PacketLineOut pckOut, | |||
@NonNull Collection<String> protocolsSupported, | |||
@NonNull CachedPackUriProvider cachedPackUriProvider) { | |||
this.pckOut = pckOut; | |||
this.protocolsSupported = protocolsSupported; | |||
this.cachedPackUriProvider = cachedPackUriProvider; | |||
} | |||
} | |||
} |
@@ -74,6 +74,9 @@ public final class FetchV2Request extends FetchRequest { | |||
private final boolean sidebandAll; | |||
@NonNull | |||
private final List<String> packfileUriProtocols; | |||
FetchV2Request(@NonNull List<ObjectId> peerHas, | |||
@NonNull List<String> wantedRefs, | |||
@NonNull Set<ObjectId> wantIds, | |||
@@ -82,7 +85,7 @@ public final class FetchV2Request extends FetchRequest { | |||
@NonNull FilterSpec filterSpec, | |||
boolean doneReceived, @NonNull Set<String> clientCapabilities, | |||
@Nullable String agent, @NonNull List<String> serverOptions, | |||
boolean sidebandAll) { | |||
boolean sidebandAll, @NonNull List<String> packfileUriProtocols) { | |||
super(wantIds, depth, clientShallowCommits, filterSpec, | |||
clientCapabilities, deepenSince, | |||
deepenNotRefs, agent); | |||
@@ -91,6 +94,7 @@ public final class FetchV2Request extends FetchRequest { | |||
this.doneReceived = doneReceived; | |||
this.serverOptions = requireNonNull(serverOptions); | |||
this.sidebandAll = sidebandAll; | |||
this.packfileUriProtocols = packfileUriProtocols; | |||
} | |||
/** | |||
@@ -138,6 +142,11 @@ public final class FetchV2Request extends FetchRequest { | |||
return sidebandAll; | |||
} | |||
@NonNull | |||
List<String> getPackfileUriProtocols() { | |||
return packfileUriProtocols; | |||
} | |||
/** @return A builder of {@link FetchV2Request}. */ | |||
static Builder builder() { | |||
return new Builder(); | |||
@@ -172,6 +181,8 @@ public final class FetchV2Request extends FetchRequest { | |||
boolean sidebandAll; | |||
final List<String> packfileUriProtocols = new ArrayList<>(); | |||
private Builder() { | |||
} | |||
@@ -339,6 +350,11 @@ public final class FetchV2Request extends FetchRequest { | |||
return this; | |||
} | |||
Builder addPackfileUriProtocol(@NonNull String value) { | |||
packfileUriProtocols.add(value); | |||
return this; | |||
} | |||
/** | |||
* @return Initialized fetch request | |||
*/ | |||
@@ -347,7 +363,8 @@ public final class FetchV2Request extends FetchRequest { | |||
clientShallowCommits, deepenSince, deepenNotRefs, | |||
depth, filterSpec, doneReceived, clientCapabilities, | |||
agent, Collections.unmodifiableList(serverOptions), | |||
sidebandAll); | |||
sidebandAll, | |||
Collections.unmodifiableList(packfileUriProtocols)); | |||
} | |||
} | |||
} |
@@ -214,6 +214,10 @@ final class ProtocolV2Parser { | |||
} else if (transferConfig.isAllowSidebandAll() | |||
&& line2.equals(OPTION_SIDEBAND_ALL)) { | |||
reqBuilder.setSidebandAll(true); | |||
} else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$ | |||
for (String s : line2.substring(14).split(",")) { | |||
reqBuilder.addPackfileUriProtocol(s); | |||
} | |||
} else { | |||
throw new PackProtocolException(MessageFormat | |||
.format(JGitText.get().unexpectedPacketLine, line2)); |
@@ -93,6 +93,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; | |||
import org.eclipse.jgit.errors.MissingObjectException; | |||
import org.eclipse.jgit.errors.PackProtocolException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; | |||
import org.eclipse.jgit.internal.storage.pack.PackWriter; | |||
import org.eclipse.jgit.internal.transport.parser.FirstWant; | |||
import org.eclipse.jgit.lib.BitmapIndex; | |||
@@ -359,6 +360,8 @@ public class UploadPack { | |||
*/ | |||
private FetchRequest currentRequest; | |||
private CachedPackUriProvider cachedPackUriProvider; | |||
/** | |||
* Create a new pack upload for an open repository. | |||
* | |||
@@ -736,6 +739,15 @@ public class UploadPack { | |||
this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ | |||
} | |||
/** | |||
* @param p provider of URIs corresponding to cached packs (to support | |||
* the packfile URIs feature) | |||
* @since 5.5 | |||
*/ | |||
public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) { | |||
cachedPackUriProvider = p; | |||
} | |||
private boolean useProtocolV2() { | |||
return ProtocolVersion.V2.equals(transferConfig.protocolVersion) | |||
&& clientRequestedV2; | |||
@@ -1285,6 +1297,7 @@ public class UploadPack { | |||
(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ | |||
(advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ | |||
(transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$ | |||
(cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$ | |||
OPTION_SHALLOW); | |||
caps.add(CAPABILITY_SERVER_OPTION); | |||
return caps; | |||
@@ -2298,7 +2311,21 @@ public class UploadPack { | |||
} | |||
if (pckOut.isUsingSideband()) { | |||
pckOut.writeString("packfile\n"); //$NON-NLS-1$ | |||
if (req instanceof FetchV2Request && | |||
cachedPackUriProvider != null && | |||
!((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) { | |||
FetchV2Request reqV2 = (FetchV2Request) req; | |||
pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig( | |||
pckOut, | |||
reqV2.getPackfileUriProtocols(), | |||
cachedPackUriProvider)); | |||
} else { | |||
// PackWriter will write "packfile-uris\n" and "packfile\n" | |||
// for us if provided a PackfileUriConfig. In this case, we | |||
// are not providing a PackfileUriConfig, so we have to | |||
// write this line ourselves. | |||
pckOut.writeString("packfile\n"); //$NON-NLS-1$ | |||
} | |||
} | |||
pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); | |||