import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; | import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; | ||||
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; | import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; | ||||
import org.eclipse.jgit.internal.storage.file.PackLock; | import org.eclipse.jgit.internal.storage.file.PackLock; | ||||
import org.eclipse.jgit.internal.storage.pack.CachedPack; | |||||
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; | |||||
import org.eclipse.jgit.junit.TestRepository; | import org.eclipse.jgit.junit.TestRepository; | ||||
import org.eclipse.jgit.lib.NullProgressMonitor; | import org.eclipse.jgit.lib.NullProgressMonitor; | ||||
import org.eclipse.jgit.lib.ObjectId; | import org.eclipse.jgit.lib.ObjectId; | ||||
parsePack(recvStream); | parsePack(recvStream); | ||||
} | } | ||||
@Test | |||||
public void testV2FetchPackfileUris() throws Exception { | |||||
// Inside the pack | |||||
RevCommit commit = remote.commit().message("x").create(); | |||||
remote.update("master", commit); | |||||
generateBitmaps(server); | |||||
// Outside the pack | |||||
RevCommit commit2 = remote.commit().message("x").parent(commit).create(); | |||||
remote.update("master", commit2); | |||||
server.getConfig().setBoolean("uploadpack", null, "allowsidebandall", true); | |||||
ByteArrayInputStream recvStream = uploadPackV2( | |||||
(UploadPack up) -> { | |||||
up.setCachedPackUriProvider(new CachedPackUriProvider() { | |||||
@Override | |||||
public PackInfo getInfo(CachedPack pack, | |||||
Collection<String> protocolsSupported) | |||||
throws IOException { | |||||
assertThat(protocolsSupported, hasItems("https")); | |||||
if (!protocolsSupported.contains("https")) | |||||
return null; | |||||
return new PackInfo("myhash", "myuri"); | |||||
} | |||||
}); | |||||
}, | |||||
"command=fetch\n", | |||||
PacketLineIn.DELIM, | |||||
"want " + commit2.getName() + "\n", | |||||
"sideband-all\n", | |||||
"packfile-uris https\n", | |||||
"done\n", | |||||
PacketLineIn.END); | |||||
PacketLineIn pckIn = new PacketLineIn(recvStream); | |||||
String s; | |||||
// skip all \002 strings | |||||
for (s = pckIn.readString(); s.startsWith("\002"); s = pckIn.readString()) { | |||||
// do nothing | |||||
} | |||||
assertThat(s, is("\001packfile-uris")); | |||||
assertThat(pckIn.readString(), is("\001myhash myuri")); | |||||
assertTrue(PacketLineIn.isDelimiter(pckIn.readString())); | |||||
assertThat(pckIn.readString(), is("\001packfile")); | |||||
parsePack(recvStream); | |||||
assertFalse(client.getObjectDatabase().has(commit.toObjectId())); | |||||
assertTrue(client.getObjectDatabase().has(commit2.toObjectId())); | |||||
} | |||||
@Test | @Test | ||||
public void testGetPeerAgentProtocolV0() throws Exception { | public void testGetPeerAgentProtocolV0() throws Exception { | ||||
RevCommit one = remote.commit().message("1").create(); | RevCommit one = remote.commit().message("1").create(); |
public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, | public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, | ||||
int matchLimit) throws IOException; | int matchLimit) throws IOException; | ||||
/** | |||||
* @return the checksum of the pack; caller must not modify it | |||||
* @since 5.5 | |||||
*/ | |||||
public byte[] getChecksum() { | |||||
return packChecksum; | |||||
} | |||||
/** | /** | ||||
* Represent mutable entry of pack index consisting of object id and offset | * Represent mutable entry of pack index consisting of object id and offset | ||||
* in pack (both mutable). | * in pack (both mutable). |
/* | |||||
* Copyright (C) 2019, Google LLC. | |||||
* and other copyright owners as documented in the project's IP log. | |||||
* | |||||
* This program and the accompanying materials are made available | |||||
* under the terms of the Eclipse Distribution License v1.0 which | |||||
* accompanies this distribution, is reproduced below, and is | |||||
* available at http://www.eclipse.org/org/documents/edl-v10.php | |||||
* | |||||
* All rights reserved. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or | |||||
* without modification, are permitted provided that the following | |||||
* conditions are met: | |||||
* | |||||
* - Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* | |||||
* - Redistributions in binary form must reproduce the above | |||||
* copyright notice, this list of conditions and the following | |||||
* disclaimer in the documentation and/or other materials provided | |||||
* with the distribution. | |||||
* | |||||
* - Neither the name of the Eclipse Foundation, Inc. nor the | |||||
* names of its contributors may be used to endorse or promote | |||||
* products derived from this software without specific prior | |||||
* written permission. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND | |||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, | |||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR | |||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | |||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.pack; | |||||
import java.io.IOException; | |||||
import java.util.Collection; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
/** | |||||
* Provider of URIs corresponding to cached packs. For use with the | |||||
* "packfile-uris" feature. | |||||
* @since 5.5 | |||||
*/ | |||||
public interface CachedPackUriProvider { | |||||
/** | |||||
* @param pack the cached pack for which to check if a corresponding URI | |||||
* exists | |||||
* @param protocolsSupported the protocols that the client has declared | |||||
* support for; if a URI is returned, it must be of one of these | |||||
* protocols | |||||
* @throws IOException implementations may throw this | |||||
* @return if a URI corresponds to the cached pack, an object | |||||
* containing the URI and some other information; null otherwise | |||||
* @since 5.5 | |||||
*/ | |||||
@Nullable | |||||
PackInfo getInfo(CachedPack pack, Collection<String> protocolsSupported) | |||||
throws IOException; | |||||
/** | |||||
* Information about a packfile. | |||||
* @since 5.5 | |||||
*/ | |||||
public static class PackInfo { | |||||
private final String hash; | |||||
private final String uri; | |||||
/** | |||||
* Constructs an object containing information about a packfile. | |||||
* @param hash the hash of the packfile as a hexadecimal string | |||||
* @param uri the URI corresponding to the packfile | |||||
*/ | |||||
public PackInfo(String hash, String uri) { | |||||
this.hash = hash; | |||||
this.uri = uri; | |||||
} | |||||
/** | |||||
* @return the hash of the packfile as a hexadecimal string | |||||
*/ | |||||
public String getHash() { | |||||
return hash; | |||||
} | |||||
/** | |||||
* @return the URI corresponding to the packfile | |||||
*/ | |||||
public String getUri() { | |||||
return uri; | |||||
} | |||||
} | |||||
} |
import org.eclipse.jgit.storage.pack.PackStatistics; | import org.eclipse.jgit.storage.pack.PackStatistics; | ||||
import org.eclipse.jgit.transport.FilterSpec; | import org.eclipse.jgit.transport.FilterSpec; | ||||
import org.eclipse.jgit.transport.ObjectCountCallback; | import org.eclipse.jgit.transport.ObjectCountCallback; | ||||
import org.eclipse.jgit.transport.PacketLineOut; | |||||
import org.eclipse.jgit.transport.WriteAbortedException; | import org.eclipse.jgit.transport.WriteAbortedException; | ||||
import org.eclipse.jgit.util.BlockList; | import org.eclipse.jgit.util.BlockList; | ||||
import org.eclipse.jgit.util.TemporaryBuffer; | import org.eclipse.jgit.util.TemporaryBuffer; | ||||
private FilterSpec filterSpec = FilterSpec.NO_FILTER; | private FilterSpec filterSpec = FilterSpec.NO_FILTER; | ||||
private PackfileUriConfig packfileUriConfig; | |||||
/** | /** | ||||
* Create writer for specified repository. | * Create writer for specified repository. | ||||
* <p> | * <p> | ||||
filterSpec = requireNonNull(filter); | filterSpec = requireNonNull(filter); | ||||
} | } | ||||
/** | |||||
* @param config configuration related to packfile URIs | |||||
* @since 5.5 | |||||
*/ | |||||
public void setPackfileUriConfig(PackfileUriConfig config) { | |||||
packfileUriConfig = config; | |||||
} | |||||
/** | /** | ||||
* Returns objects number in a pack file that was created by this writer. | * Returns objects number in a pack file that was created by this writer. | ||||
* | * | ||||
return stats.totalObjects; | return stats.totalObjects; | ||||
} | } | ||||
private long getUnoffloadedObjectCount() throws IOException { | |||||
long objCnt = 0; | |||||
objCnt += objectsLists[OBJ_COMMIT].size(); | |||||
objCnt += objectsLists[OBJ_TREE].size(); | |||||
objCnt += objectsLists[OBJ_BLOB].size(); | |||||
objCnt += objectsLists[OBJ_TAG].size(); | |||||
for (CachedPack pack : cachedPacks) { | |||||
CachedPackUriProvider.PackInfo packInfo = | |||||
packfileUriConfig.cachedPackUriProvider.getInfo( | |||||
pack, packfileUriConfig.protocolsSupported); | |||||
if (packInfo == null) { | |||||
objCnt += pack.getObjectCount(); | |||||
} | |||||
} | |||||
return objCnt; | |||||
} | |||||
/** | /** | ||||
* Returns the object ids in the pack file that was created by this writer. | * Returns the object ids in the pack file that was created by this writer. | ||||
* <p> | * <p> | ||||
: new CheckedOutputStream(packStream, crc32), | : new CheckedOutputStream(packStream, crc32), | ||||
this); | this); | ||||
long objCnt = getObjectCount(); | |||||
long objCnt = packfileUriConfig == null ? getObjectCount() : | |||||
getUnoffloadedObjectCount(); | |||||
stats.totalObjects = objCnt; | stats.totalObjects = objCnt; | ||||
if (callback != null) | if (callback != null) | ||||
callback.setObjectCount(objCnt); | callback.setObjectCount(objCnt); | ||||
beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); | beginPhase(PackingPhase.WRITING, writeMonitor, objCnt); | ||||
long writeStart = System.currentTimeMillis(); | long writeStart = System.currentTimeMillis(); | ||||
try { | try { | ||||
List<CachedPack> unwrittenCachedPacks; | |||||
if (packfileUriConfig != null) { | |||||
unwrittenCachedPacks = new ArrayList<>(); | |||||
CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider; | |||||
PacketLineOut o = packfileUriConfig.pckOut; | |||||
o.writeString("packfile-uris\n"); | |||||
for (CachedPack pack : cachedPacks) { | |||||
CachedPackUriProvider.PackInfo packInfo = p.getInfo( | |||||
pack, packfileUriConfig.protocolsSupported); | |||||
if (packInfo != null) { | |||||
o.writeString(packInfo.getHash() + ' ' + | |||||
packInfo.getUri() + '\n'); | |||||
} else { | |||||
unwrittenCachedPacks.add(pack); | |||||
} | |||||
} | |||||
packfileUriConfig.pckOut.writeDelim(); | |||||
packfileUriConfig.pckOut.writeString("packfile\n"); | |||||
} else { | |||||
unwrittenCachedPacks = cachedPacks; | |||||
} | |||||
out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); | out.writeFileHeader(PACK_VERSION_GENERATED, objCnt); | ||||
out.flush(); | out.flush(); | ||||
} | } | ||||
stats.reusedPacks = Collections.unmodifiableList(cachedPacks); | stats.reusedPacks = Collections.unmodifiableList(cachedPacks); | ||||
for (CachedPack pack : cachedPacks) { | |||||
for (CachedPack pack : unwrittenCachedPacks) { | |||||
long deltaCnt = pack.getDeltaCount(); | long deltaCnt = pack.getDeltaCount(); | ||||
stats.reusedObjects += pack.getObjectCount(); | stats.reusedObjects += pack.getObjectCount(); | ||||
stats.reusedDeltas += deltaCnt; | stats.reusedDeltas += deltaCnt; | ||||
return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; | return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]"; | ||||
} | } | ||||
} | } | ||||
/** | |||||
* Configuration related to the packfile URI feature. | |||||
* | |||||
* @since 5.5 | |||||
*/ | |||||
public static class PackfileUriConfig { | |||||
@NonNull | |||||
private final PacketLineOut pckOut; | |||||
@NonNull | |||||
private final Collection<String> protocolsSupported; | |||||
@NonNull | |||||
private final CachedPackUriProvider cachedPackUriProvider; | |||||
/** | |||||
* @param pckOut where to write "packfile-uri" lines to (should | |||||
* output to the same stream as the one passed to | |||||
* PackWriter#writePack) | |||||
* @param protocolsSupported list of protocols supported (e.g. "https") | |||||
* @param cachedPackUriProvider provider of URIs corresponding | |||||
* to cached packs | |||||
* @since 5.5 | |||||
*/ | |||||
public PackfileUriConfig(@NonNull PacketLineOut pckOut, | |||||
@NonNull Collection<String> protocolsSupported, | |||||
@NonNull CachedPackUriProvider cachedPackUriProvider) { | |||||
this.pckOut = pckOut; | |||||
this.protocolsSupported = protocolsSupported; | |||||
this.cachedPackUriProvider = cachedPackUriProvider; | |||||
} | |||||
} | |||||
} | } |
private final boolean sidebandAll; | private final boolean sidebandAll; | ||||
@NonNull | |||||
private final List<String> packfileUriProtocols; | |||||
FetchV2Request(@NonNull List<ObjectId> peerHas, | FetchV2Request(@NonNull List<ObjectId> peerHas, | ||||
@NonNull List<String> wantedRefs, | @NonNull List<String> wantedRefs, | ||||
@NonNull Set<ObjectId> wantIds, | @NonNull Set<ObjectId> wantIds, | ||||
@NonNull FilterSpec filterSpec, | @NonNull FilterSpec filterSpec, | ||||
boolean doneReceived, @NonNull Set<String> clientCapabilities, | boolean doneReceived, @NonNull Set<String> clientCapabilities, | ||||
@Nullable String agent, @NonNull List<String> serverOptions, | @Nullable String agent, @NonNull List<String> serverOptions, | ||||
boolean sidebandAll) { | |||||
boolean sidebandAll, @NonNull List<String> packfileUriProtocols) { | |||||
super(wantIds, depth, clientShallowCommits, filterSpec, | super(wantIds, depth, clientShallowCommits, filterSpec, | ||||
clientCapabilities, deepenSince, | clientCapabilities, deepenSince, | ||||
deepenNotRefs, agent); | deepenNotRefs, agent); | ||||
this.doneReceived = doneReceived; | this.doneReceived = doneReceived; | ||||
this.serverOptions = requireNonNull(serverOptions); | this.serverOptions = requireNonNull(serverOptions); | ||||
this.sidebandAll = sidebandAll; | this.sidebandAll = sidebandAll; | ||||
this.packfileUriProtocols = packfileUriProtocols; | |||||
} | } | ||||
/** | /** | ||||
return sidebandAll; | return sidebandAll; | ||||
} | } | ||||
@NonNull | |||||
List<String> getPackfileUriProtocols() { | |||||
return packfileUriProtocols; | |||||
} | |||||
/** @return A builder of {@link FetchV2Request}. */ | /** @return A builder of {@link FetchV2Request}. */ | ||||
static Builder builder() { | static Builder builder() { | ||||
return new Builder(); | return new Builder(); | ||||
boolean sidebandAll; | boolean sidebandAll; | ||||
final List<String> packfileUriProtocols = new ArrayList<>(); | |||||
private Builder() { | private Builder() { | ||||
} | } | ||||
return this; | return this; | ||||
} | } | ||||
Builder addPackfileUriProtocol(@NonNull String value) { | |||||
packfileUriProtocols.add(value); | |||||
return this; | |||||
} | |||||
/** | /** | ||||
* @return Initialized fetch request | * @return Initialized fetch request | ||||
*/ | */ | ||||
clientShallowCommits, deepenSince, deepenNotRefs, | clientShallowCommits, deepenSince, deepenNotRefs, | ||||
depth, filterSpec, doneReceived, clientCapabilities, | depth, filterSpec, doneReceived, clientCapabilities, | ||||
agent, Collections.unmodifiableList(serverOptions), | agent, Collections.unmodifiableList(serverOptions), | ||||
sidebandAll); | |||||
sidebandAll, | |||||
Collections.unmodifiableList(packfileUriProtocols)); | |||||
} | } | ||||
} | } | ||||
} | } |
} else if (transferConfig.isAllowSidebandAll() | } else if (transferConfig.isAllowSidebandAll() | ||||
&& line2.equals(OPTION_SIDEBAND_ALL)) { | && line2.equals(OPTION_SIDEBAND_ALL)) { | ||||
reqBuilder.setSidebandAll(true); | reqBuilder.setSidebandAll(true); | ||||
} else if (line2.startsWith("packfile-uris ")) { //$NON-NLS-1$ | |||||
for (String s : line2.substring(14).split(",")) { | |||||
reqBuilder.addPackfileUriProtocol(s); | |||||
} | |||||
} else { | } else { | ||||
throw new PackProtocolException(MessageFormat | throw new PackProtocolException(MessageFormat | ||||
.format(JGitText.get().unexpectedPacketLine, line2)); | .format(JGitText.get().unexpectedPacketLine, line2)); |
import org.eclipse.jgit.errors.MissingObjectException; | import org.eclipse.jgit.errors.MissingObjectException; | ||||
import org.eclipse.jgit.errors.PackProtocolException; | import org.eclipse.jgit.errors.PackProtocolException; | ||||
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; | |||||
import org.eclipse.jgit.internal.storage.pack.PackWriter; | import org.eclipse.jgit.internal.storage.pack.PackWriter; | ||||
import org.eclipse.jgit.internal.transport.parser.FirstWant; | import org.eclipse.jgit.internal.transport.parser.FirstWant; | ||||
import org.eclipse.jgit.lib.BitmapIndex; | import org.eclipse.jgit.lib.BitmapIndex; | ||||
*/ | */ | ||||
private FetchRequest currentRequest; | private FetchRequest currentRequest; | ||||
private CachedPackUriProvider cachedPackUriProvider; | |||||
/** | /** | ||||
* Create a new pack upload for an open repository. | * Create a new pack upload for an open repository. | ||||
* | * | ||||
this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ | this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ | ||||
} | } | ||||
/** | |||||
* @param p provider of URIs corresponding to cached packs (to support | |||||
* the packfile URIs feature) | |||||
* @since 5.5 | |||||
*/ | |||||
public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) { | |||||
cachedPackUriProvider = p; | |||||
} | |||||
private boolean useProtocolV2() { | private boolean useProtocolV2() { | ||||
return ProtocolVersion.V2.equals(transferConfig.protocolVersion) | return ProtocolVersion.V2.equals(transferConfig.protocolVersion) | ||||
&& clientRequestedV2; | && clientRequestedV2; | ||||
(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ | (transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ | ||||
(advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ | (advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ | ||||
(transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$ | (transferConfig.isAllowSidebandAll() ? OPTION_SIDEBAND_ALL + ' ' : "") + //$NON-NLS-1$ | ||||
(cachedPackUriProvider != null ? "packfile-uris " : "") + // $NON-NLS-1$ | |||||
OPTION_SHALLOW); | OPTION_SHALLOW); | ||||
caps.add(CAPABILITY_SERVER_OPTION); | caps.add(CAPABILITY_SERVER_OPTION); | ||||
return caps; | return caps; | ||||
} | } | ||||
if (pckOut.isUsingSideband()) { | if (pckOut.isUsingSideband()) { | ||||
pckOut.writeString("packfile\n"); //$NON-NLS-1$ | |||||
if (req instanceof FetchV2Request && | |||||
cachedPackUriProvider != null && | |||||
!((FetchV2Request) req).getPackfileUriProtocols().isEmpty()) { | |||||
FetchV2Request reqV2 = (FetchV2Request) req; | |||||
pw.setPackfileUriConfig(new PackWriter.PackfileUriConfig( | |||||
pckOut, | |||||
reqV2.getPackfileUriProtocols(), | |||||
cachedPackUriProvider)); | |||||
} else { | |||||
// PackWriter will write "packfile-uris\n" and "packfile\n" | |||||
// for us if provided a PackfileUriConfig. In this case, we | |||||
// are not providing a PackfileUriConfig, so we have to | |||||
// write this line ourselves. | |||||
pckOut.writeString("packfile\n"); //$NON-NLS-1$ | |||||
} | |||||
} | } | ||||
pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); | pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); | ||||