Support "http.userAgent" and "http.extraHeader" from the git config

Validate the extra headers and log but otherwise ignore invalid
headers. An empty http.extraHeader starts the list afresh.

The http.userAgent is restricted to printable 7-bit ASCII, other
characters are replaced by '.'.

Moves a support method from the ssh.apache bundle to HttpSupport in
the main JGit bundle.

Bug:541500
Change-Id: Id2d8df12914e2cdbd936ff00dc824d8f871bd580
Signed-off-by: James Wynn <james@jameswynn.com>
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

7 files changed, 209 insertions, 2 deletions

diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters
index 05fc371c81..538c6f7bfb 100644
--- a/org.eclipse.jgit/.settings/.api_filters
+++ b/org.eclipse.jgit/.settings/.api_filters
@@ -22,4 +22,18 @@
             </message_arguments>
         </filter>
     </resource>
+    <resource path="src/org/eclipse/jgit/transport/HttpConfig.java" type="org.eclipse.jgit.transport.HttpConfig">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.HttpConfig"/>
+                <message_argument value="EXTRA_HEADER"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.HttpConfig"/>
+                <message_argument value="USER_AGENT"/>
+            </message_arguments>
+        </filter>
+    </resource>
 </component>
diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index beafff3811..c9e48a5c09 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -347,6 +347,9 @@ invalidFilter=Invalid filter: {0}
 invalidGitdirRef = Invalid .git reference in file ''{0}''
 invalidGitModules=Invalid .gitmodules file
 invalidGitType=invalid git type: {0}
+invalidHeaderFormat=Invalid header from git config http.extraHeader ignored: no colon or empty key in header ''{0}''
+invalidHeaderKey=Invalid header from git config http.extraHeader ignored: key contains illegal characters; see RFC 7230: ''{0}''
+invalidHeaderValue=Invalid header from git config http.extraHeader ignored: value should be 7bit-ASCII characters only: ''{0}''
 invalidHexString=Invalid hex string: {0}
 invalidHomeDirectory=Invalid home directory: {0}
 invalidHooksPath=Invalid git config core.hooksPath = {0}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 3f2565fdde..b761210e63 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -375,6 +375,9 @@ public class JGitText extends TranslationBundle {
 	/***/ public String invalidGitdirRef;
 	/***/ public String invalidGitModules;
 	/***/ public String invalidGitType;
+	/***/ public String invalidHeaderFormat;
+	/***/ public String invalidHeaderKey;
+	/***/ public String invalidHeaderValue;
 	/***/ public String invalidHexString;
 	/***/ public String invalidHomeDirectory;
 	/***/ public String invalidHooksPath;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java
index 79cba80e11..58fc250255 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java
@@ -14,9 +14,13 @@ package org.eclipse.jgit.transport;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.text.MessageFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.function.Supplier;
 
+import org.eclipse.jgit.annotations.NonNull;
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Config;
@@ -56,6 +60,20 @@ public class HttpConfig {
 	public static final String SSL_VERIFY_KEY = "sslVerify"; //$NON-NLS-1$
 
 	/**
+	 * git config key for the "userAgent" setting.
+	 *
+	 * @since 5.10
+	 */
+	public static final String USER_AGENT = "userAgent"; //$NON-NLS-1$
+
+	/**
+	 * git config key for the "extraHeader" setting.
+	 *
+	 * @since 5.10
+	 */
+	public static final String EXTRA_HEADER = "extraHeader"; //$NON-NLS-1$
+
+	/**
 	 * git config key for the "cookieFile" setting.
 	 *
 	 * @since 5.4
@@ -143,6 +161,10 @@ public class HttpConfig {
 
 	private int maxRedirects;
 
+	private String userAgent;
+
+	private List<String> extraHeaders;
+
 	private String cookieFile;
 
 	private boolean saveCookies;
@@ -186,6 +208,27 @@ public class HttpConfig {
 	}
 
 	/**
+	 * Get the "http.userAgent" setting
+	 *
+	 * @return the value of the "http.userAgent" setting
+	 * @since 5.10
+	 */
+	public String getUserAgent() {
+		return userAgent;
+	}
+
+	/**
+	 * Get the "http.extraHeader" setting
+	 *
+	 * @return the value of the "http.extraHeader" setting
+	 * @since 5.10
+	 */
+	@NonNull
+	public List<String> getExtraHeaders() {
+		return extraHeaders == null ? Collections.emptyList() : extraHeaders;
+	}
+
+	/**
 	 * Get the "http.cookieFile" setting
 	 *
 	 * @return the value of the "http.cookieFile" setting
@@ -265,11 +308,25 @@ public class HttpConfig {
 		if (redirectLimit < 0) {
 			redirectLimit = MAX_REDIRECTS;
 		}
+		String agent = config.getString(HTTP, null, USER_AGENT);
+		if (agent != null) {
+			agent = UserAgent.clean(agent);
+		}
+		userAgent = agent;
+		String[] headers = config.getStringList(HTTP, null, EXTRA_HEADER);
+		// https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpextraHeader
+		// "an empty value will reset the extra headers to the empty list."
+		int start = findLastEmpty(headers) + 1;
+		if (start > 0) {
+			headers = Arrays.copyOfRange(headers, start, headers.length);
+		}
+		extraHeaders = Arrays.asList(headers);
 		cookieFile = config.getString(HTTP, null, COOKIE_FILE_KEY);
 		saveCookies = config.getBoolean(HTTP, SAVE_COOKIES_KEY, false);
 		cookieFileCacheLimit = config.getInt(HTTP, COOKIE_FILE_CACHE_LIMIT_KEY,
 				DEFAULT_COOKIE_FILE_CACHE_LIMIT);
 		String match = findMatch(config.getSubsections(HTTP), uri);
+
 		if (match != null) {
 			// Override with more specific items
 			postBufferSize = config.getInt(HTTP, match, POST_BUFFER_KEY,
@@ -283,6 +340,22 @@ public class HttpConfig {
 			if (newMaxRedirects >= 0) {
 				redirectLimit = newMaxRedirects;
 			}
+			String uriSpecificUserAgent = config.getString(HTTP, match,
+					USER_AGENT);
+			if (uriSpecificUserAgent != null) {
+				userAgent = UserAgent.clean(uriSpecificUserAgent);
+			}
+			String[] uriSpecificExtraHeaders = config.getStringList(HTTP, match,
+					EXTRA_HEADER);
+			if (uriSpecificExtraHeaders.length > 0) {
+				start = findLastEmpty(uriSpecificExtraHeaders) + 1;
+				if (start > 0) {
+					uriSpecificExtraHeaders = Arrays.copyOfRange(
+							uriSpecificExtraHeaders, start,
+							uriSpecificExtraHeaders.length);
+				}
+				extraHeaders = Arrays.asList(uriSpecificExtraHeaders);
+			}
 			String urlSpecificCookieFile = config.getString(HTTP, match,
 					COOKIE_FILE_KEY);
 			if (urlSpecificCookieFile != null) {
@@ -297,6 +370,15 @@ public class HttpConfig {
 		maxRedirects = redirectLimit;
 	}
 
+	private int findLastEmpty(String[] values) {
+		for (int i = values.length - 1; i >= 0; i--) {
+			if (values[i] == null) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
 	/**
 	 * Determines the best match from a set of subsection names (representing
 	 * prefix URLs) for the given {@link URIish}.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java
index 16169f028b..6768387e65 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java
@@ -49,6 +49,7 @@ import java.net.SocketException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -900,7 +901,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 			conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP);
 		}
 		conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$
-		if (UserAgent.get() != null) {
+		if (http.getUserAgent() != null) {
+			conn.setRequestProperty(HDR_USER_AGENT, http.getUserAgent());
+		} else if (UserAgent.get() != null) {
 			conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get());
 		}
 		int timeOut = getTimeout();
@@ -909,6 +912,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 			conn.setConnectTimeout(effTimeOut);
 			conn.setReadTimeout(effTimeOut);
 		}
+		addHeaders(conn, http.getExtraHeaders());
 		// set cookie header if necessary
 		if (!relevantCookies.isEmpty()) {
 			setCookieHeader(conn);
@@ -923,6 +927,44 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
 		return conn;
 	}
 
+	/**
+	 * Adds a list of header strings to the connection. Headers are expected to
+	 * separate keys from values, i.e. "Key: Value". Headers without colon or
+	 * key are ignored (and logged), as are headers with keys that are not RFC
+	 * 7230 tokens or with non-ASCII values.
+	 *
+	 * @param conn
+	 *            The target HttpConnection
+	 * @param headersToAdd
+	 *            A list of header strings
+	 */
+	static void addHeaders(HttpConnection conn, List<String> headersToAdd) {
+		for (String header : headersToAdd) {
+			// Empty values are allowed according to
+			// https://tools.ietf.org/html/rfc7230
+			int colon = header.indexOf(':');
+			String key = null;
+			if (colon > 0) {
+				key = header.substring(0, colon).trim();
+			}
+			if (key == null || key.isEmpty()) {
+				LOG.warn(MessageFormat.format(
+						JGitText.get().invalidHeaderFormat, header));
+			} else if (HttpSupport.scanToken(key, 0) != key.length()) {
+				LOG.warn(MessageFormat.format(JGitText.get().invalidHeaderKey,
+						header));
+			} else {
+				String value = header.substring(colon + 1).trim();
+				if (!StandardCharsets.US_ASCII.newEncoder().canEncode(value)) {
+					LOG.warn(MessageFormat
+							.format(JGitText.get().invalidHeaderValue, header));
+				} else {
+					conn.setRequestProperty(key, value);
+				}
+			}
+		}
+	}
+
 	private void setCookieHeader(HttpConnection conn) {
 		StringBuilder cookieHeaderValue = new StringBuilder();
 		for (HttpCookie cookie : relevantCookies) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java
index 5b63635e5a..604eb3a66c 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java
@@ -46,7 +46,7 @@ public class UserAgent {
 		return "unknown"; //$NON-NLS-1$
 	}
 
-	private static String clean(String s) {
+	static String clean(String s) {
 		s = s.trim();
 		StringBuilder b = new StringBuilder(s.length());
 		for (int i = 0; i < s.length(); i++) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java
index 8ff649bccc..03672f8ba5 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java
@@ -424,6 +424,69 @@ public class HttpSupport {
 		}
 	}
 
+	/**
+	 * Scan a RFC 7230 token as it appears in HTTP headers.
+	 *
+	 * @param header
+	 *            to scan in
+	 * @param from
+	 *            index in {@code header} to start scanning at
+	 * @return the index after the token, that is, on the first non-token
+	 *         character or {@code header.length}
+	 * @throws IndexOutOfBoundsException
+	 *             if {@code from < 0} or {@code from > header.length()}
+	 *
+	 * @see <a href="https://tools.ietf.org/html/rfc7230#appendix-B">RFC 7230,
+	 *      Appendix B: Collected Grammar; "token" production</a>
+	 * @since 5.10
+	 */
+	public static int scanToken(String header, int from) {
+		int length = header.length();
+		int i = from;
+		if (i < 0 || i > length) {
+			throw new IndexOutOfBoundsException();
+		}
+		while (i < length) {
+			char c = header.charAt(i);
+			switch (c) {
+			case '!':
+			case '#':
+			case '$':
+			case '%':
+			case '&':
+			case '\'':
+			case '*':
+			case '+':
+			case '-':
+			case '.':
+			case '^':
+			case '_':
+			case '`':
+			case '|':
+			case '~':
+			case '0':
+			case '1':
+			case '2':
+			case '3':
+			case '4':
+			case '5':
+			case '6':
+			case '7':
+			case '8':
+			case '9':
+				i++;
+				break;
+			default:
+				if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
+					i++;
+					break;
+				}
+				return i;
+			}
+		}
+		return i;
+	}
+
 	private HttpSupport() {
 		// Utility class only.
 	}