dump HTTP: Avoid being confused by Content-Length of a gzipped stream

TransportHttp sets 'Accept-Encoding: gzip' to allow the server to
compress HTTP responses. When fetching a loose object over HTTP, it
uses the following code to read the response:

       InputStream in = openInputStream(c);
       int len = c.getContentLength();
       return new FileStream(in, len);

If the content is gzipped, openInputStream decompresses it and produces
the correct content for the object. Unfortunately the Content-Length
header contains the length of the compressed stream instead of the
actual content length. Use a length of -1 instead since we don't know
the actual length.

Loose objects are already compressed, so the gzip encoding typically
produces a longer compressed payload. The value from the Content-Length
is too high, producing EOFException: Short read of block.

Change-Id: I8d5284dad608e3abd8217823da2b365e8cd998b0
Signed-off-by: Zhen Chen <czhen@google.com>
Helped-by: Jonathan Nieder <jrn@google.com>

org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java

@@ -44,6 +44,7 @@
 package org.eclipse.jgit.http.server;
 
 import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
+import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP;
 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
 import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING;
 import static org.eclipse.jgit.util.HttpSupport.HDR_ETAG;
@@ -111,7 +112,7 @@ public final class ServletUtils {
 			throws IOException {
 		InputStream in = req.getInputStream();
 		final String enc = req.getHeader(HDR_CONTENT_ENCODING);
-		if (ENCODING_GZIP.equals(enc) || "x-gzip".equals(enc)) //$NON-NLS-1$
+		if (ENCODING_GZIP.equals(enc) || ENCODING_X_GZIP.equals(enc)) //$NON-NLS-1$
 			in = new GZIPInputStream(in);
 		else if (enc != null)
 			throw new IOException(MessageFormat.format(HttpServerText.get().encodingNotSupportedByThisLibrary

org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java

@@ -46,6 +46,7 @@
 package org.eclipse.jgit.transport;
 
 import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
+import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP;
 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
 import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING;
@@ -575,7 +576,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 	final InputStream openInputStream(HttpConnection conn)
 			throws IOException {
 		InputStream input = conn.getInputStream();
-		if (ENCODING_GZIP.equals(conn.getHeaderField(HDR_CONTENT_ENCODING)))
+		if (isGzipContent(conn))
 			input = new GZIPInputStream(input);
 		return input;
 	}
@@ -591,6 +592,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 		return expType.equals(actType);
 	}
 
+	private boolean isGzipContent(final HttpConnection c) {
+		return ENCODING_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING))
+				|| ENCODING_X_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING));
+	}
+
 	private void readSmartHeaders(final InputStream in, final String service)
 			throws IOException {
 		// A smart reply will have a '#' after the first 4 bytes, but
@@ -685,8 +691,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 			switch (HttpSupport.response(c)) {
 			case HttpConnection.HTTP_OK:
 				final InputStream in = openInputStream(c);
-				final int len = c.getContentLength();
-				return new FileStream(in, len);
+				// If content is being gzipped and then transferred, the content
+				// length in the header is the zipped content length, not the
+				// actual content length.
+				if (!isGzipContent(c)) {
+					final int len = c.getContentLength();
+					return new FileStream(in, len);
+				}
+				return new FileStream(in);
 			case HttpConnection.HTTP_NOT_FOUND:
 				throw new FileNotFoundException(u.toString());
 			default:

org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java

@@ -144,6 +144,9 @@ public class HttpSupport {
 	/** The {@code gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. */
 	public static final String ENCODING_GZIP = "gzip"; //$NON-NLS-1$
 
+	/** The {@code x-gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. */
+	public static final String ENCODING_X_GZIP = "x-gzip"; //$NON-NLS-1$
+
 	/** The standard {@code text/plain} MIME type. */
 	public static final String TEXT_PLAIN = "text/plain"; //$NON-NLS-1$