From d62130558884e0600665236ab62b20ba0f5bac45 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Tue, 22 Nov 2016 16:39:27 -0800 Subject: [PATCH] Decide whether to "Accept-Encoding: gzip" on a request-by-request basis When the reply is already compressed (e.g. a packfile fetched using dumb HTTP), "Content-Encoding: gzip" wastes bandwidth relative to sending the content raw. So don't "Accept-Encoding: gzip" for such requests. Change-Id: Id25702c0b0ed2895df8e9790052c3417d713572c Signed-off-by: Zhen Chen --- .../eclipse/jgit/transport/TransportHttp.java | 89 +++++++++++++++---- .../transport/WalkRemoteObjectDatabase.java | 7 +- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index eb2d62cc32..96a6fe1d01 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -45,6 +45,7 @@ package org.eclipse.jgit.transport; +import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; @@ -129,6 +130,25 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$ + /** + * Accept-Encoding header in the HTTP request + * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html). + * + * @since 4.6 + */ + public enum AcceptEncoding { + /** + * Do not specify an Accept-Encoding header. In most servers this + * results in the content being transmitted as-is. + */ + UNSPECIFIED, + + /** + * Accept gzip content encoding. + */ + GZIP + } + static final TransportProtocol PROTO_HTTP = new TransportProtocol() { private final String[] schemeNames = { "http", "https" }; //$NON-NLS-1$ //$NON-NLS-2$ @@ -325,12 +345,15 @@ public class TransportHttp extends HttpTransport implements WalkTransport, br.close(); } - if (!refs.containsKey(Constants.HEAD)) { + if (!refs.containsKey(HEAD)) { // If HEAD was not published in the info/refs file (it usually // is not there) download HEAD by itself as a loose file and do // the resolution by hand. // - HttpConnection conn = httpOpen(new URL(baseUrl, Constants.HEAD)); + HttpConnection conn = httpOpen( + METHOD_GET, + new URL(baseUrl, HEAD), + AcceptEncoding.GZIP); int status = HttpSupport.response(conn); switch (status) { case HttpConnection.HTTP_OK: { @@ -342,11 +365,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport, Ref r = refs.get(target); if (r == null) r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null); - r = new SymbolicRef(Constants.HEAD, r); + r = new SymbolicRef(HEAD, r); refs.put(r.getName(), r); } else if (line != null && ObjectId.isId(line)) { Ref r = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, - Constants.HEAD, ObjectId.fromString(line)); + HEAD, ObjectId.fromString(line)); refs.put(r.getName(), r); } } finally { @@ -456,7 +479,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, Collection ignoreTypes = null; for (;;) { try { - final HttpConnection conn = httpOpen(u); + final HttpConnection conn = httpOpen(METHOD_GET, u, AcceptEncoding.GZIP); if (useSmartHttp) { String exp = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$ conn.setRequestProperty(HDR_ACCEPT, exp + ", */*"); //$NON-NLS-1$ @@ -530,21 +553,37 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } } - final HttpConnection httpOpen(URL u) throws IOException { - return httpOpen(METHOD_GET, u); + /** + * Open an HTTP connection, setting the accept-encoding request header to gzip. + * + * @param method HTTP request method + * @param u url of the HTTP connection + * @return the HTTP connection + * @throws IOException + * @since 3.3 + * @deprecated use {@link #httpOpen(String, URL, AcceptEncoding)} instead. + */ + @Deprecated + protected HttpConnection httpOpen(String method, URL u) throws IOException { + return httpOpen(method, u, AcceptEncoding.GZIP); } /** * Open an HTTP connection. * - * @param method - * @param u - * @return the connection + * @param method HTTP request method + * @param u url of the HTTP connection + * @param acceptEncoding accept-encoding header option + * @return the HTTP connection * @throws IOException - * @since 3.3 + * @since 4.6 */ - protected HttpConnection httpOpen(String method, URL u) - throws IOException { + protected HttpConnection httpOpen(String method, URL u, + AcceptEncoding acceptEncoding) throws IOException { + if (method == null || u == null || acceptEncoding == null) { + throw new NullPointerException(); + } + final Proxy proxy = HttpSupport.proxyFor(proxySelector, u); HttpConnection conn = connectionFactory.create(u, proxy); @@ -554,7 +593,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, conn.setRequestMethod(method); conn.setUseCaches(false); - conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP); + if (acceptEncoding == AcceptEncoding.GZIP) { + conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP); + } conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$ if (UserAgent.get() != null) { conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get()); @@ -660,6 +701,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return new HttpObjectDB(new URL(httpObjectsUrl, location)); } + @Override + BufferedReader openReader(String path) throws IOException { + // Line oriented readable content is likely to compress well. + // Request gzip encoding. + InputStream is = open(path, AcceptEncoding.GZIP).in; + return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); + } + @Override Collection getPackNames() throws IOException { final Collection packs = new ArrayList(); @@ -685,9 +734,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport, @Override FileStream open(final String path) throws IOException { + return open(path, AcceptEncoding.UNSPECIFIED); + } + + FileStream open(String path, AcceptEncoding acceptEncoding) + throws IOException { final URL base = httpObjectsUrl; final URL u = new URL(base, path); - final HttpConnection c = httpOpen(u); + final HttpConnection c = httpOpen(METHOD_GET, u, acceptEncoding); switch (HttpSupport.response(c)) { case HttpConnection.HTTP_OK: final InputStream in = openInputStream(c); @@ -844,7 +898,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } void openStream() throws IOException { - conn = httpOpen(METHOD_POST, new URL(baseUrl, serviceName)); + conn = httpOpen( + METHOD_POST, + new URL(baseUrl, serviceName), + AcceptEncoding.GZIP); conn.setInstanceFollowRedirects(false); conn.setDoOutput(true); conn.setRequestProperty(HDR_CONTENT_TYPE, requestType); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java index e47913ea39..24f30ed206 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java @@ -133,6 +133,9 @@ abstract class WalkRemoteObjectDatabase { * Callers such as {@link WalkFetchConnection} are prepared to handle this * by validating the content received, and assuming content that fails to * match its hash is an incorrectly phrased FileNotFoundException. + *

+ * This method is recommended for already compressed files like loose objects + * and pack files. For text files, see {@link #openReader(String)}. * * @param path * location of the file to read, relative to this objects @@ -346,8 +349,8 @@ abstract class WalkRemoteObjectDatabase { /** * Open a buffered reader around a file. *

- * This is shorthand for calling {@link #open(String)} and then wrapping it - * in a reader suitable for line oriented files like the alternates list. + * This method is suitable for for reading line-oriented resources like + * info/packs, info/refs, and the alternates list. * * @return a stream to read from the file. Never null. * @param path -- 2.39.5