diff options
6 files changed, 164 insertions, 26 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java index 5fd6816ec7..9fb323c4db 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java @@ -106,13 +106,21 @@ public class UnionInputStreamTest { u.add(new ByteArrayInputStream(new byte[] { 4, 5 })); final byte[] r = new byte[5]; - assertEquals(5, u.read(r, 0, 5)); - assertTrue(Arrays.equals(new byte[] { 1, 0, 2, 3, 4 }, r)); + assertEquals(3, u.read(r, 0, 5)); + assertTrue(Arrays.equals(new byte[] { 1, 0, 2, }, slice(r, 3))); assertEquals(1, u.read(r, 0, 5)); - assertEquals(5, r[0]); + assertEquals(3, r[0]); + assertEquals(2, u.read(r, 0, 5)); + assertTrue(Arrays.equals(new byte[] { 4, 5, }, slice(r, 2))); assertEquals(-1, u.read(r, 0, 5)); } + private static byte[] slice(byte[] in, int len) { + byte[] r = new byte[len]; + System.arraycopy(in, 0, r, 0, len); + return r; + } + @Test public void testArrayConstructor() throws IOException { final UnionInputStream u = new UnionInputStream( @@ -121,10 +129,12 @@ public class UnionInputStreamTest { new ByteArrayInputStream(new byte[] { 4, 5 })); final byte[] r = new byte[5]; - assertEquals(5, u.read(r, 0, 5)); - assertTrue(Arrays.equals(new byte[] { 1, 0, 2, 3, 4 }, r)); + assertEquals(3, u.read(r, 0, 5)); + assertTrue(Arrays.equals(new byte[] { 1, 0, 2, }, slice(r, 3))); assertEquals(1, u.read(r, 0, 5)); - assertEquals(5, r[0]); + assertEquals(3, r[0]); + assertEquals(2, u.read(r, 0, 5)); + assertTrue(Arrays.equals(new byte[] { 4, 5, }, slice(r, 2))); assertEquals(-1, u.read(r, 0, 5)); } @@ -143,9 +153,9 @@ public class UnionInputStreamTest { u.add(new ByteArrayInputStream(new byte[] { 3 })); u.add(new ByteArrayInputStream(new byte[] { 4, 5 })); assertEquals(0, u.skip(0)); - assertEquals(4, u.skip(4)); - assertEquals(4, u.read()); - assertEquals(1, u.skip(5)); + assertEquals(3, u.skip(3)); + assertEquals(3, u.read()); + assertEquals(2, u.skip(5)); assertEquals(0, u.skip(5)); assertEquals(-1, u.read()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index 0e34554395..ab40831823 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -714,6 +714,7 @@ public class PackWriter { if (!cachedPacks.isEmpty()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); + long writeStart = System.currentTimeMillis(); final List<ObjectToPack> list = sortByName(); final PackIndexWriter iw; int indexVersion = config.getIndexVersion(); @@ -722,6 +723,7 @@ public class PackWriter { else iw = PackIndexWriter.createVersion(indexStream, indexVersion); iw.write(list, packcsum); + stats.timeWriting += System.currentTimeMillis() - writeStart; } private List<ObjectToPack> sortByName() { @@ -828,6 +830,7 @@ public class PackWriter { stats.timeWriting = System.currentTimeMillis() - writeStart; stats.totalBytes = out.length(); stats.reusedPacks = Collections.unmodifiableList(cachedPacks); + stats.depth = depth; for (Statistics.ObjectType typeStat : stats.objectTypes) { if (typeStat == null) @@ -1870,6 +1873,8 @@ public class PackWriter { Collection<CachedPack> reusedPacks; + int depth; + int deltaSearchNonEdgeObjects; int deltasFound; @@ -2012,6 +2017,16 @@ public class PackWriter { return objectTypes[typeCode]; } + /** @return true if the resulting pack file was a shallow pack. */ + public boolean isShallow() { + return depth > 0; + } + + /** @return depth (in commits) the pack includes if shallow. */ + public int getDepth() { + return depth; + } + /** * @return time in milliseconds spent enumerating the objects that need * to be included in the output. This time includes any restarts @@ -2060,6 +2075,15 @@ public class PackWriter { return timeWriting; } + /** @return total time spent processing this pack. */ + public long getTimeTotal() { + return timeCounting + + timeSearchingForReuse + + timeSearchingForSizes + + timeCompressing + + timeWriting; + } + /** * @return get the average output speed in terms of bytes-per-second. * {@code getTotalBytes() / (getTimeWriting() / 1000.0)}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 67bedaca11..4a86da7c6d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -433,7 +433,9 @@ public abstract class BasePackFetchConnection extends BasePackConnection // ACK status to tell us common objects for reuse in future // requests. If its not enabled, we can't talk to the peer. // - throw new PackProtocolException(uri, MessageFormat.format(JGitText.get().statelessRPCRequiresOptionToBeEnabled, OPTION_MULTI_ACK_DETAILED)); + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().statelessRPCRequiresOptionToBeEnabled, + OPTION_MULTI_ACK_DETAILED)); } return line.toString(); @@ -453,7 +455,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection state.writeTo(out, null); negotiateBegin(); - SEND_HAVES: while (!receivedReady) { + SEND_HAVES: for (;;) { final RevCommit c = walk.next(); if (c == null) break SEND_HAVES; @@ -528,6 +530,8 @@ public abstract class BasePackFetchConnection extends BasePackConnection throw new CancelledException(); } + if (noDone & receivedReady) + break SEND_HAVES; if (statelessRPC) state.writeTo(out, null); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 43ad98985d..1b78fcfc9a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -851,11 +851,6 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } class HttpExecuteStream extends InputStream { - public int available() throws IOException { - execute(); - return 0; - } - public int read() throws IOException { execute(); return -1; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 459ee6a95d..ca3d24790c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -49,6 +49,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -107,6 +108,16 @@ public class UploadPack { static final String OPTION_SHALLOW = BasePackFetchConnection.OPTION_SHALLOW; + /** Policy the server uses to validate client requests */ + public static enum RequestPolicy { + /** Client may only ask for objects the server advertised a reference for. */ + ADVERTISED, + /** Client may ask for any commit reachable from a reference. */ + REACHABLE_COMMIT, + /** Client may ask for any SHA-1 in the repository. */ + ANY; + } + /** Database we read the objects from. */ private final Repository db; @@ -198,6 +209,8 @@ public class UploadPack { private final RevFlagSet SAVE; + private RequestPolicy requestPolicy = RequestPolicy.ADVERTISED; + private MultiAck multiAck = MultiAck.OFF; private boolean noDone; @@ -243,12 +256,22 @@ public class UploadPack { /** @return all refs which were advertised to the client. */ public final Map<String, Ref> getAdvertisedRefs() { - if (refs == null) { - refs = refFilter.filter(db.getAllRefs()); - } + if (refs == null) + setAdvertisedRefs(db.getAllRefs()); return refs; } + /** + * @param allRefs + * explicit set of references to claim as advertised by this + * UploadPack instance. This overrides any references that + * may exist in the source repository. The map is passed + * to the configured {@link #getRefFilter()}. + */ + public void setAdvertisedRefs(Map<String, Ref> allRefs) { + refs = refFilter.filter(allRefs); + } + /** @return timeout (in seconds) before aborting an IO operation. */ public int getTimeout() { return timeout; @@ -285,6 +308,26 @@ public class UploadPack { */ public void setBiDirectionalPipe(final boolean twoWay) { biDirectionalPipe = twoWay; + if (!biDirectionalPipe && requestPolicy == RequestPolicy.ADVERTISED) + requestPolicy = RequestPolicy.REACHABLE_COMMIT; + } + + /** @return policy used by the service to validate client requests. */ + public RequestPolicy getRequestPolicy() { + return requestPolicy; + } + + /** + * @param policy + * the policy used to enforce validation of a client's want list. + * By default the policy is {@link RequestPolicy#ADVERTISED}, + * which is the Git default requiring clients to only ask for an + * object that a reference directly points to. This may be relaxed + * to {@link RequestPolicy#REACHABLE_COMMIT} when callers + * have {@link #setBiDirectionalPipe(boolean)} set to false. + */ + public void setRequestPolicy(RequestPolicy policy) { + requestPolicy = policy != null ? policy : RequestPolicy.ADVERTISED; } /** @return the filter used while advertising the refs to the client */ @@ -406,6 +449,8 @@ public class UploadPack { private void service() throws IOException { if (biDirectionalPipe) sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut)); + else if (requestPolicy == RequestPolicy.ANY) + advertised = Collections.emptySet(); else { advertised = new HashSet<ObjectId>(); for (Ref ref : getAdvertisedRefs().values()) { @@ -659,6 +704,7 @@ public class UploadPack { needMissing = true; } + Set<RevObject> notAdvertisedWants = null; int haveCnt = 0; AsyncRevObjectQueue q = walk.parseAny(toParse, needMissing); try { @@ -682,10 +728,10 @@ public class UploadPack { // list wasn't parsed earlier, and was done in this batch. // if (wantIds.remove(obj)) { - if (!advertised.contains(obj)) { - String msg = MessageFormat.format( - JGitText.get().wantNotValid, obj.name()); - throw new PackProtocolException(msg); + if (!advertised.contains(obj) && requestPolicy != RequestPolicy.ANY) { + if (notAdvertisedWants == null) + notAdvertisedWants = new HashSet<RevObject>(); + notAdvertisedWants.add(obj); } if (!obj.has(WANT)) { @@ -745,6 +791,26 @@ public class UploadPack { } finally { q.release(); } + + // If the client asked for non advertised object, check our policy. + if (notAdvertisedWants != null && !notAdvertisedWants.isEmpty()) { + switch (requestPolicy) { + case ADVERTISED: + default: + throw new PackProtocolException(MessageFormat.format( + JGitText.get().wantNotValid, + notAdvertisedWants.iterator().next().name())); + + case REACHABLE_COMMIT: + checkNotAdvertisedWants(notAdvertisedWants); + break; + + case ANY: + // Allow whatever was asked for. + break; + } + } + int missCnt = peerHas.size() - haveCnt; // If we don't have one of the objects but we're also willing to @@ -787,6 +853,40 @@ public class UploadPack { return last; } + private void checkNotAdvertisedWants(Set<RevObject> notAdvertisedWants) + throws MissingObjectException, IncorrectObjectTypeException, IOException { + // Walk the requested commits back to the advertised commits. + // If any commit exists, a branch was deleted or rewound and + // the repository owner no longer exports that requested item. + // If the requested commit is merged into an advertised branch + // it will be marked UNINTERESTING and no commits return. + + for (RevObject o : notAdvertisedWants) { + if (!(o instanceof RevCommit)) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().wantNotValid, + notAdvertisedWants.iterator().next().name())); + } + walk.markStart((RevCommit) o); + } + + for (ObjectId id : advertised) { + try { + walk.markUninteresting(walk.parseCommit(id)); + } catch (IncorrectObjectTypeException notCommit) { + continue; + } + } + + RevCommit bad = walk.next(); + if (bad != null) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().wantNotValid, + bad.name())); + } + walk.reset(); + } + private void addCommonBase(final RevObject o) { if (!o.has(COMMON)) { o.add(COMMON); @@ -941,7 +1041,7 @@ public class UploadPack { pw.setThin(options.contains(OPTION_THIN_PACK)); pw.setReuseValidatingObjects(false); - if (commonBase.isEmpty()) { + if (commonBase.isEmpty() && refs != null) { Set<ObjectId> tagTargets = new HashSet<ObjectId>(); for (Ref ref : refs.values()) { if (ref.getPeeledObjectId() != null) @@ -968,7 +1068,7 @@ public class UploadPack { rw = ow; } - if (options.contains(OPTION_INCLUDE_TAG)) { + if (options.contains(OPTION_INCLUDE_TAG) && refs != null) { for (Ref ref : refs.values()) { ObjectId objectId = ref.getObjectId(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java index f0183717a7..0df3775f11 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java @@ -148,8 +148,11 @@ public class UnionInputStream extends InputStream { len -= n; } else if (in == EOF) return 0 < cnt ? cnt : -1; - else + else { pop(); + if (0 < cnt) + break; + } } return cnt; } @@ -180,6 +183,8 @@ public class UnionInputStream extends InputStream { final int r = in.read(); if (r < 0) { pop(); + if (0 < cnt) + break; } else { cnt += 1; len -= 1; |