You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

TransportHttp.java 52KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665
  1. /*
  2. * Copyright (C) 2008-2010, Google Inc.
  3. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  4. * Copyright (C) 2013, Matthias Sohn <matthias.sohn@sap.com>
  5. * Copyright (C) 2017, Thomas Wolf <thomas.wolf@paranor.ch> and others
  6. *
  7. * This program and the accompanying materials are made available under the
  8. * terms of the Eclipse Distribution License v. 1.0 which is available at
  9. * https://www.eclipse.org/org/documents/edl-v10.php.
  10. *
  11. * SPDX-License-Identifier: BSD-3-Clause
  12. */
  13. package org.eclipse.jgit.transport;
  14. import static java.nio.charset.StandardCharsets.UTF_8;
  15. import static org.eclipse.jgit.lib.Constants.HEAD;
  16. import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES;
  17. import static org.eclipse.jgit.lib.Constants.INFO_HTTP_ALTERNATES;
  18. import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
  19. import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP;
  20. import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
  21. import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
  22. import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING;
  23. import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
  24. import static org.eclipse.jgit.util.HttpSupport.HDR_COOKIE;
  25. import static org.eclipse.jgit.util.HttpSupport.HDR_LOCATION;
  26. import static org.eclipse.jgit.util.HttpSupport.HDR_PRAGMA;
  27. import static org.eclipse.jgit.util.HttpSupport.HDR_SET_COOKIE;
  28. import static org.eclipse.jgit.util.HttpSupport.HDR_SET_COOKIE2;
  29. import static org.eclipse.jgit.util.HttpSupport.HDR_USER_AGENT;
  30. import static org.eclipse.jgit.util.HttpSupport.HDR_WWW_AUTHENTICATE;
  31. import static org.eclipse.jgit.util.HttpSupport.METHOD_GET;
  32. import static org.eclipse.jgit.util.HttpSupport.METHOD_POST;
  33. import java.io.BufferedReader;
  34. import java.io.ByteArrayInputStream;
  35. import java.io.FileNotFoundException;
  36. import java.io.IOException;
  37. import java.io.InputStream;
  38. import java.io.InputStreamReader;
  39. import java.io.OutputStream;
  40. import java.net.HttpCookie;
  41. import java.net.MalformedURLException;
  42. import java.net.Proxy;
  43. import java.net.ProxySelector;
  44. import java.net.URI;
  45. import java.net.URISyntaxException;
  46. import java.net.URL;
  47. import java.nio.file.InvalidPathException;
  48. import java.nio.file.Path;
  49. import java.nio.file.Paths;
  50. import java.security.cert.CertPathBuilderException;
  51. import java.security.cert.CertPathValidatorException;
  52. import java.security.cert.CertificateException;
  53. import java.text.MessageFormat;
  54. import java.util.ArrayList;
  55. import java.util.Arrays;
  56. import java.util.Collection;
  57. import java.util.Collections;
  58. import java.util.EnumSet;
  59. import java.util.HashSet;
  60. import java.util.LinkedHashSet;
  61. import java.util.LinkedList;
  62. import java.util.List;
  63. import java.util.Locale;
  64. import java.util.Map;
  65. import java.util.Set;
  66. import java.util.TreeMap;
  67. import java.util.zip.GZIPInputStream;
  68. import java.util.zip.GZIPOutputStream;
  69. import javax.net.ssl.SSLHandshakeException;
  70. import org.eclipse.jgit.errors.ConfigInvalidException;
  71. import org.eclipse.jgit.errors.NoRemoteRepositoryException;
  72. import org.eclipse.jgit.errors.NotSupportedException;
  73. import org.eclipse.jgit.errors.PackProtocolException;
  74. import org.eclipse.jgit.errors.TransportException;
  75. import org.eclipse.jgit.internal.JGitText;
  76. import org.eclipse.jgit.internal.storage.file.RefDirectory;
  77. import org.eclipse.jgit.internal.transport.http.NetscapeCookieFile;
  78. import org.eclipse.jgit.internal.transport.http.NetscapeCookieFileCache;
  79. import org.eclipse.jgit.lib.Constants;
  80. import org.eclipse.jgit.lib.ObjectId;
  81. import org.eclipse.jgit.lib.ObjectIdRef;
  82. import org.eclipse.jgit.lib.ProgressMonitor;
  83. import org.eclipse.jgit.lib.Ref;
  84. import org.eclipse.jgit.lib.Repository;
  85. import org.eclipse.jgit.lib.StoredConfig;
  86. import org.eclipse.jgit.lib.SymbolicRef;
  87. import org.eclipse.jgit.transport.HttpAuthMethod.Type;
  88. import org.eclipse.jgit.transport.HttpConfig.HttpRedirectMode;
  89. import org.eclipse.jgit.transport.http.HttpConnection;
  90. import org.eclipse.jgit.util.HttpSupport;
  91. import org.eclipse.jgit.util.IO;
  92. import org.eclipse.jgit.util.RawParseUtils;
  93. import org.eclipse.jgit.util.StringUtils;
  94. import org.eclipse.jgit.util.SystemReader;
  95. import org.eclipse.jgit.util.TemporaryBuffer;
  96. import org.eclipse.jgit.util.io.DisabledOutputStream;
  97. import org.eclipse.jgit.util.io.UnionInputStream;
  98. import org.slf4j.Logger;
  99. import org.slf4j.LoggerFactory;
  100. /**
  101. * Transport over HTTP and FTP protocols.
  102. * <p>
  103. * If the transport is using HTTP and the remote HTTP service is Git-aware
  104. * (speaks the "smart-http protocol") this client will automatically take
  105. * advantage of the additional Git-specific HTTP extensions. If the remote
  106. * service does not support these extensions, the client will degrade to direct
  107. * file fetching.
  108. * <p>
  109. * If the remote (server side) repository does not have the specialized Git
  110. * support, object files are retrieved directly through standard HTTP GET (or
  111. * binary FTP GET) requests. This make it easy to serve a Git repository through
  112. * a standard web host provider that does not offer specific support for Git.
  113. *
  114. * @see WalkFetchConnection
  115. */
  116. public class TransportHttp extends HttpTransport implements WalkTransport,
  117. PackTransport {
  118. private static final Logger LOG = LoggerFactory
  119. .getLogger(TransportHttp.class);
  120. private static final String SVC_UPLOAD_PACK = "git-upload-pack"; //$NON-NLS-1$
  121. private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$
  122. /**
  123. * Accept-Encoding header in the HTTP request
  124. * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html).
  125. *
  126. * @since 4.6
  127. */
  128. public enum AcceptEncoding {
  129. /**
  130. * Do not specify an Accept-Encoding header. In most servers this
  131. * results in the content being transmitted as-is.
  132. */
  133. UNSPECIFIED,
  134. /**
  135. * Accept gzip content encoding.
  136. */
  137. GZIP
  138. }
  139. static final TransportProtocol PROTO_HTTP = new TransportProtocol() {
  140. private final String[] schemeNames = { "http", "https" }; //$NON-NLS-1$ //$NON-NLS-2$
  141. private final Set<String> schemeSet = Collections
  142. .unmodifiableSet(new LinkedHashSet<>(Arrays
  143. .asList(schemeNames)));
  144. @Override
  145. public String getName() {
  146. return JGitText.get().transportProtoHTTP;
  147. }
  148. @Override
  149. public Set<String> getSchemes() {
  150. return schemeSet;
  151. }
  152. @Override
  153. public Set<URIishField> getRequiredFields() {
  154. return Collections.unmodifiableSet(EnumSet.of(URIishField.HOST,
  155. URIishField.PATH));
  156. }
  157. @Override
  158. public Set<URIishField> getOptionalFields() {
  159. return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
  160. URIishField.PASS, URIishField.PORT));
  161. }
  162. @Override
  163. public int getDefaultPort() {
  164. return 80;
  165. }
  166. @Override
  167. public Transport open(URIish uri, Repository local, String remoteName)
  168. throws NotSupportedException {
  169. return new TransportHttp(local, uri);
  170. }
  171. @Override
  172. public Transport open(URIish uri) throws NotSupportedException {
  173. return new TransportHttp(uri);
  174. }
  175. };
  176. static final TransportProtocol PROTO_FTP = new TransportProtocol() {
  177. @Override
  178. public String getName() {
  179. return JGitText.get().transportProtoFTP;
  180. }
  181. @Override
  182. public Set<String> getSchemes() {
  183. return Collections.singleton("ftp"); //$NON-NLS-1$
  184. }
  185. @Override
  186. public Set<URIishField> getRequiredFields() {
  187. return Collections.unmodifiableSet(EnumSet.of(URIishField.HOST,
  188. URIishField.PATH));
  189. }
  190. @Override
  191. public Set<URIishField> getOptionalFields() {
  192. return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
  193. URIishField.PASS, URIishField.PORT));
  194. }
  195. @Override
  196. public int getDefaultPort() {
  197. return 21;
  198. }
  199. @Override
  200. public Transport open(URIish uri, Repository local, String remoteName)
  201. throws NotSupportedException {
  202. return new TransportHttp(local, uri);
  203. }
  204. };
  205. /**
  206. * The current URI we're talking to. The inherited (final) field
  207. * {@link #uri} stores the original URI; {@code currentUri} may be different
  208. * after redirects.
  209. */
  210. private URIish currentUri;
  211. private URL baseUrl;
  212. private URL objectsUrl;
  213. private final HttpConfig http;
  214. private final ProxySelector proxySelector;
  215. private boolean useSmartHttp = true;
  216. private HttpAuthMethod authMethod = HttpAuthMethod.Type.NONE.method(null);
  217. private Map<String, String> headers;
  218. private boolean sslVerify;
  219. private boolean sslFailure = false;
  220. /**
  221. * All stored cookies bound to this repo (independent of the baseUrl)
  222. */
  223. private final NetscapeCookieFile cookieFile;
  224. /**
  225. * The cookies to be sent with each request to the given {@link #baseUrl}.
  226. * Filtered view on top of {@link #cookieFile} where only cookies which
  227. * apply to the current url are left. This set needs to be filtered for
  228. * expired entries each time prior to sending them.
  229. */
  230. private final Set<HttpCookie> relevantCookies;
  231. TransportHttp(Repository local, URIish uri)
  232. throws NotSupportedException {
  233. super(local, uri);
  234. setURI(uri);
  235. http = new HttpConfig(local.getConfig(), uri);
  236. proxySelector = ProxySelector.getDefault();
  237. sslVerify = http.isSslVerify();
  238. cookieFile = getCookieFileFromConfig(http);
  239. relevantCookies = filterCookies(cookieFile, baseUrl);
  240. }
  241. private URL toURL(URIish urish) throws MalformedURLException {
  242. String uriString = urish.toString();
  243. if (!uriString.endsWith("/")) { //$NON-NLS-1$
  244. uriString += '/';
  245. }
  246. return new URL(uriString);
  247. }
  248. /**
  249. * Set uri a {@link org.eclipse.jgit.transport.URIish} object.
  250. *
  251. * @param uri
  252. * a {@link org.eclipse.jgit.transport.URIish} object.
  253. * @throws org.eclipse.jgit.errors.NotSupportedException
  254. * @since 4.9
  255. */
  256. protected void setURI(URIish uri) throws NotSupportedException {
  257. try {
  258. currentUri = uri;
  259. baseUrl = toURL(uri);
  260. objectsUrl = new URL(baseUrl, "objects/"); //$NON-NLS-1$
  261. } catch (MalformedURLException e) {
  262. throw new NotSupportedException(MessageFormat.format(JGitText.get().invalidURL, uri), e);
  263. }
  264. }
  265. /**
  266. * Create a minimal HTTP transport with default configuration values.
  267. *
  268. * @param uri
  269. * @throws NotSupportedException
  270. */
  271. TransportHttp(URIish uri) throws NotSupportedException {
  272. super(uri);
  273. setURI(uri);
  274. http = new HttpConfig(uri);
  275. proxySelector = ProxySelector.getDefault();
  276. sslVerify = http.isSslVerify();
  277. cookieFile = getCookieFileFromConfig(http);
  278. relevantCookies = filterCookies(cookieFile, baseUrl);
  279. }
  280. /**
  281. * Toggle whether or not smart HTTP transport should be used.
  282. * <p>
  283. * This flag exists primarily to support backwards compatibility testing
  284. * within a testing framework, there is no need to modify it in most
  285. * applications.
  286. *
  287. * @param on
  288. * if {@code true} (default), smart HTTP is enabled.
  289. */
  290. public void setUseSmartHttp(boolean on) {
  291. useSmartHttp = on;
  292. }
  293. @SuppressWarnings("resource") // Closed by caller
  294. private FetchConnection getConnection(HttpConnection c, InputStream in,
  295. String service) throws IOException {
  296. BaseConnection f;
  297. if (isSmartHttp(c, service)) {
  298. readSmartHeaders(in, service);
  299. f = new SmartHttpFetchConnection(in);
  300. } else {
  301. // Assume this server doesn't support smart HTTP fetch
  302. // and fall back on dumb object walking.
  303. f = newDumbConnection(in);
  304. }
  305. f.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER));
  306. return (FetchConnection) f;
  307. }
  308. /** {@inheritDoc} */
  309. @Override
  310. public FetchConnection openFetch() throws TransportException,
  311. NotSupportedException {
  312. final String service = SVC_UPLOAD_PACK;
  313. try {
  314. final HttpConnection c = connect(service);
  315. try (InputStream in = openInputStream(c)) {
  316. return getConnection(c, in, service);
  317. }
  318. } catch (NotSupportedException | TransportException err) {
  319. throw err;
  320. } catch (IOException err) {
  321. throw new TransportException(uri, JGitText.get().errorReadingInfoRefs, err);
  322. }
  323. }
  324. private WalkFetchConnection newDumbConnection(InputStream in)
  325. throws IOException, PackProtocolException {
  326. HttpObjectDB d = new HttpObjectDB(objectsUrl);
  327. Map<String, Ref> refs;
  328. try (BufferedReader br = toBufferedReader(in)) {
  329. refs = d.readAdvertisedImpl(br);
  330. }
  331. if (!refs.containsKey(HEAD)) {
  332. // If HEAD was not published in the info/refs file (it usually
  333. // is not there) download HEAD by itself as a loose file and do
  334. // the resolution by hand.
  335. //
  336. HttpConnection conn = httpOpen(
  337. METHOD_GET,
  338. new URL(baseUrl, HEAD),
  339. AcceptEncoding.GZIP);
  340. int status = HttpSupport.response(conn);
  341. switch (status) {
  342. case HttpConnection.HTTP_OK: {
  343. try (BufferedReader br = toBufferedReader(
  344. openInputStream(conn))) {
  345. String line = br.readLine();
  346. if (line != null && line.startsWith(RefDirectory.SYMREF)) {
  347. String target = line.substring(RefDirectory.SYMREF.length());
  348. Ref r = refs.get(target);
  349. if (r == null)
  350. r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
  351. r = new SymbolicRef(HEAD, r);
  352. refs.put(r.getName(), r);
  353. } else if (line != null && ObjectId.isId(line)) {
  354. Ref r = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK,
  355. HEAD, ObjectId.fromString(line));
  356. refs.put(r.getName(), r);
  357. }
  358. }
  359. break;
  360. }
  361. case HttpConnection.HTTP_NOT_FOUND:
  362. break;
  363. default:
  364. throw new TransportException(uri, MessageFormat.format(
  365. JGitText.get().cannotReadHEAD, Integer.valueOf(status),
  366. conn.getResponseMessage()));
  367. }
  368. }
  369. WalkFetchConnection wfc = new WalkFetchConnection(this, d);
  370. wfc.available(refs);
  371. return wfc;
  372. }
  373. private BufferedReader toBufferedReader(InputStream in) {
  374. return new BufferedReader(new InputStreamReader(in, UTF_8));
  375. }
  376. /** {@inheritDoc} */
  377. @Override
  378. public PushConnection openPush() throws NotSupportedException,
  379. TransportException {
  380. final String service = SVC_RECEIVE_PACK;
  381. try {
  382. final HttpConnection c = connect(service);
  383. try (InputStream in = openInputStream(c)) {
  384. if (isSmartHttp(c, service)) {
  385. return smartPush(service, c, in);
  386. } else if (!useSmartHttp) {
  387. final String msg = JGitText.get().smartHTTPPushDisabled;
  388. throw new NotSupportedException(msg);
  389. } else {
  390. final String msg = JGitText.get().remoteDoesNotSupportSmartHTTPPush;
  391. throw new NotSupportedException(msg);
  392. }
  393. }
  394. } catch (NotSupportedException | TransportException err) {
  395. throw err;
  396. } catch (IOException err) {
  397. throw new TransportException(uri, JGitText.get().errorReadingInfoRefs, err);
  398. }
  399. }
  400. private PushConnection smartPush(String service, HttpConnection c,
  401. InputStream in) throws IOException, TransportException {
  402. readSmartHeaders(in, service);
  403. SmartHttpPushConnection p = new SmartHttpPushConnection(in);
  404. p.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER));
  405. return p;
  406. }
  407. /** {@inheritDoc} */
  408. @Override
  409. public void close() {
  410. // No explicit connections are maintained.
  411. }
  412. /**
  413. * Set additional headers on the HTTP connection
  414. *
  415. * @param headers
  416. * a map of name:values that are to be set as headers on the HTTP
  417. * connection
  418. * @since 3.4
  419. */
  420. public void setAdditionalHeaders(Map<String, String> headers) {
  421. this.headers = headers;
  422. }
  423. private NoRemoteRepositoryException createNotFoundException(URIish u,
  424. URL url, String msg) {
  425. String text;
  426. if (msg != null && !msg.isEmpty()) {
  427. text = MessageFormat.format(JGitText.get().uriNotFoundWithMessage,
  428. url, msg);
  429. } else {
  430. text = MessageFormat.format(JGitText.get().uriNotFound, url);
  431. }
  432. return new NoRemoteRepositoryException(u, text);
  433. }
  434. private HttpConnection connect(String service)
  435. throws TransportException, NotSupportedException {
  436. URL u = getServiceURL(service);
  437. int authAttempts = 1;
  438. int redirects = 0;
  439. Collection<Type> ignoreTypes = null;
  440. for (;;) {
  441. try {
  442. final HttpConnection conn = httpOpen(METHOD_GET, u, AcceptEncoding.GZIP);
  443. if (useSmartHttp) {
  444. String exp = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$
  445. conn.setRequestProperty(HDR_ACCEPT, exp + ", */*"); //$NON-NLS-1$
  446. } else {
  447. conn.setRequestProperty(HDR_ACCEPT, "*/*"); //$NON-NLS-1$
  448. }
  449. final int status = HttpSupport.response(conn);
  450. processResponseCookies(conn);
  451. switch (status) {
  452. case HttpConnection.HTTP_OK:
  453. // Check if HttpConnection did some authentication in the
  454. // background (e.g Kerberos/SPNEGO).
  455. // That may not work for streaming requests and jgit
  456. // explicit authentication would be required
  457. if (authMethod.getType() == HttpAuthMethod.Type.NONE
  458. && conn.getHeaderField(HDR_WWW_AUTHENTICATE) != null)
  459. authMethod = HttpAuthMethod.scanResponse(conn, ignoreTypes);
  460. return conn;
  461. case HttpConnection.HTTP_NOT_FOUND:
  462. throw createNotFoundException(uri, u,
  463. conn.getResponseMessage());
  464. case HttpConnection.HTTP_UNAUTHORIZED:
  465. authMethod = HttpAuthMethod.scanResponse(conn, ignoreTypes);
  466. if (authMethod.getType() == HttpAuthMethod.Type.NONE)
  467. throw new TransportException(uri, MessageFormat.format(
  468. JGitText.get().authenticationNotSupported, uri));
  469. CredentialsProvider credentialsProvider = getCredentialsProvider();
  470. if (credentialsProvider == null)
  471. throw new TransportException(uri,
  472. JGitText.get().noCredentialsProvider);
  473. if (authAttempts > 1)
  474. credentialsProvider.reset(currentUri);
  475. if (3 < authAttempts
  476. || !authMethod.authorize(currentUri,
  477. credentialsProvider)) {
  478. throw new TransportException(uri,
  479. JGitText.get().notAuthorized);
  480. }
  481. authAttempts++;
  482. continue;
  483. case HttpConnection.HTTP_FORBIDDEN:
  484. throw new TransportException(uri, MessageFormat.format(
  485. JGitText.get().serviceNotPermitted, baseUrl,
  486. service));
  487. case HttpConnection.HTTP_MOVED_PERM:
  488. case HttpConnection.HTTP_MOVED_TEMP:
  489. case HttpConnection.HTTP_SEE_OTHER:
  490. case HttpConnection.HTTP_11_MOVED_PERM:
  491. case HttpConnection.HTTP_11_MOVED_TEMP:
  492. // SEE_OTHER should actually never be sent by a git server,
  493. // and in general should occur only on POST requests. But it
  494. // doesn't hurt to accept it here as a redirect.
  495. if (http.getFollowRedirects() == HttpRedirectMode.FALSE) {
  496. throw new TransportException(uri,
  497. MessageFormat.format(
  498. JGitText.get().redirectsOff,
  499. Integer.valueOf(status)));
  500. }
  501. URIish newUri = redirect(u,
  502. conn.getHeaderField(HDR_LOCATION),
  503. Constants.INFO_REFS, redirects++);
  504. setURI(newUri);
  505. u = getServiceURL(service);
  506. authAttempts = 1;
  507. break;
  508. default:
  509. String err = status + " " + conn.getResponseMessage(); //$NON-NLS-1$
  510. throw new TransportException(uri, err);
  511. }
  512. } catch (NotSupportedException | TransportException e) {
  513. throw e;
  514. } catch (SSLHandshakeException e) {
  515. handleSslFailure(e);
  516. continue; // Re-try
  517. } catch (IOException e) {
  518. if (authMethod.getType() != HttpAuthMethod.Type.NONE) {
  519. if (ignoreTypes == null) {
  520. ignoreTypes = new HashSet<>();
  521. }
  522. ignoreTypes.add(authMethod.getType());
  523. // reset auth method & attempts for next authentication type
  524. authMethod = HttpAuthMethod.Type.NONE.method(null);
  525. authAttempts = 1;
  526. continue;
  527. }
  528. throw new TransportException(uri, MessageFormat.format(JGitText.get().cannotOpenService, service), e);
  529. }
  530. }
  531. }
  532. void processResponseCookies(HttpConnection conn) {
  533. if (cookieFile != null && http.getSaveCookies()) {
  534. List<HttpCookie> foundCookies = new LinkedList<>();
  535. List<String> cookieHeaderValues = conn
  536. .getHeaderFields(HDR_SET_COOKIE);
  537. if (!cookieHeaderValues.isEmpty()) {
  538. foundCookies.addAll(
  539. extractCookies(HDR_SET_COOKIE, cookieHeaderValues));
  540. }
  541. cookieHeaderValues = conn.getHeaderFields(HDR_SET_COOKIE2);
  542. if (!cookieHeaderValues.isEmpty()) {
  543. foundCookies.addAll(
  544. extractCookies(HDR_SET_COOKIE2, cookieHeaderValues));
  545. }
  546. if (!foundCookies.isEmpty()) {
  547. try {
  548. // update cookie lists with the newly received cookies!
  549. Set<HttpCookie> cookies = cookieFile.getCookies(false);
  550. cookies.addAll(foundCookies);
  551. cookieFile.write(baseUrl);
  552. relevantCookies.addAll(foundCookies);
  553. } catch (IOException | IllegalArgumentException
  554. | InterruptedException e) {
  555. LOG.warn(MessageFormat.format(
  556. JGitText.get().couldNotPersistCookies,
  557. cookieFile.getPath()), e);
  558. }
  559. }
  560. }
  561. }
  562. private List<HttpCookie> extractCookies(String headerKey,
  563. List<String> headerValues) {
  564. List<HttpCookie> foundCookies = new LinkedList<>();
  565. for (String headerValue : headerValues) {
  566. foundCookies
  567. .addAll(HttpCookie.parse(headerKey + ':' + headerValue));
  568. }
  569. // HttpCookies.parse(...) is only compliant with RFC 2965. Make it RFC
  570. // 6265 compliant by applying the logic from
  571. // https://tools.ietf.org/html/rfc6265#section-5.2.3
  572. for (HttpCookie foundCookie : foundCookies) {
  573. String domain = foundCookie.getDomain();
  574. if (domain != null && domain.startsWith(".")) { //$NON-NLS-1$
  575. foundCookie.setDomain(domain.substring(1));
  576. }
  577. }
  578. return foundCookies;
  579. }
  580. private static class CredentialItems {
  581. CredentialItem.InformationalMessage message;
  582. /** Trust the server for this git operation */
  583. CredentialItem.YesNoType now;
  584. /**
  585. * Trust the server for all git operations from this repository; may be
  586. * {@code null} if the transport was created via
  587. * {@link #TransportHttp(URIish)}.
  588. */
  589. CredentialItem.YesNoType forRepo;
  590. /** Always trust the server from now on. */
  591. CredentialItem.YesNoType always;
  592. public CredentialItem[] items() {
  593. if (forRepo == null) {
  594. return new CredentialItem[] { message, now, always };
  595. }
  596. return new CredentialItem[] { message, now, forRepo, always };
  597. }
  598. }
  599. private void handleSslFailure(Throwable e) throws TransportException {
  600. if (sslFailure || !trustInsecureSslConnection(e.getCause())) {
  601. throw new TransportException(uri,
  602. MessageFormat.format(
  603. JGitText.get().sslFailureExceptionMessage,
  604. currentUri.setPass(null)),
  605. e);
  606. }
  607. sslFailure = true;
  608. }
  609. private boolean trustInsecureSslConnection(Throwable cause) {
  610. if (cause instanceof CertificateException
  611. || cause instanceof CertPathBuilderException
  612. || cause instanceof CertPathValidatorException) {
  613. // Certificate expired or revoked, PKIX path building not
  614. // possible, self-signed certificate, host does not match ...
  615. CredentialsProvider provider = getCredentialsProvider();
  616. if (provider != null) {
  617. CredentialItems trust = constructSslTrustItems(cause);
  618. CredentialItem[] items = trust.items();
  619. if (provider.supports(items)) {
  620. boolean answered = provider.get(uri, items);
  621. if (answered) {
  622. // Not canceled
  623. boolean trustNow = trust.now.getValue();
  624. boolean trustLocal = trust.forRepo != null
  625. && trust.forRepo.getValue();
  626. boolean trustAlways = trust.always.getValue();
  627. if (trustNow || trustLocal || trustAlways) {
  628. sslVerify = false;
  629. if (trustAlways) {
  630. updateSslVerifyUser(false);
  631. } else if (trustLocal) {
  632. updateSslVerify(local.getConfig(), false);
  633. }
  634. return true;
  635. }
  636. }
  637. }
  638. }
  639. }
  640. return false;
  641. }
  642. private CredentialItems constructSslTrustItems(Throwable cause) {
  643. CredentialItems items = new CredentialItems();
  644. String info = MessageFormat.format(JGitText.get().sslFailureInfo,
  645. currentUri.setPass(null));
  646. String sslMessage = cause.getLocalizedMessage();
  647. if (sslMessage == null) {
  648. sslMessage = cause.toString();
  649. }
  650. sslMessage = MessageFormat.format(JGitText.get().sslFailureCause,
  651. sslMessage);
  652. items.message = new CredentialItem.InformationalMessage(info + '\n'
  653. + sslMessage + '\n'
  654. + JGitText.get().sslFailureTrustExplanation);
  655. items.now = new CredentialItem.YesNoType(JGitText.get().sslTrustNow);
  656. if (local != null) {
  657. items.forRepo = new CredentialItem.YesNoType(
  658. MessageFormat.format(JGitText.get().sslTrustForRepo,
  659. local.getDirectory()));
  660. }
  661. items.always = new CredentialItem.YesNoType(
  662. JGitText.get().sslTrustAlways);
  663. return items;
  664. }
  665. private void updateSslVerify(StoredConfig config, boolean value) {
  666. // Since git uses the original URI for matching, we must also use the
  667. // original URI and cannot use the current URI (which might be different
  668. // after redirects).
  669. String uriPattern = uri.getScheme() + "://" + uri.getHost(); //$NON-NLS-1$
  670. int port = uri.getPort();
  671. if (port > 0) {
  672. uriPattern += ":" + port; //$NON-NLS-1$
  673. }
  674. config.setBoolean(HttpConfig.HTTP, uriPattern,
  675. HttpConfig.SSL_VERIFY_KEY, value);
  676. try {
  677. config.save();
  678. } catch (IOException e) {
  679. LOG.error(JGitText.get().sslVerifyCannotSave, e);
  680. }
  681. }
  682. private void updateSslVerifyUser(boolean value) {
  683. StoredConfig userConfig = null;
  684. try {
  685. userConfig = SystemReader.getInstance().getUserConfig();
  686. updateSslVerify(userConfig, value);
  687. } catch (IOException | ConfigInvalidException e) {
  688. // Log it, but otherwise ignore here.
  689. LOG.error(e.getMessage(), e);
  690. }
  691. }
  692. private URIish redirect(URL currentUrl, String location, String checkFor,
  693. int redirects)
  694. throws TransportException {
  695. if (location == null || location.isEmpty()) {
  696. throw new TransportException(uri,
  697. MessageFormat.format(JGitText.get().redirectLocationMissing,
  698. baseUrl));
  699. }
  700. if (redirects >= http.getMaxRedirects()) {
  701. throw new TransportException(uri,
  702. MessageFormat.format(JGitText.get().redirectLimitExceeded,
  703. Integer.valueOf(http.getMaxRedirects()), baseUrl,
  704. location));
  705. }
  706. try {
  707. URI redirectTo = new URI(location);
  708. redirectTo = currentUrl.toURI().resolve(redirectTo);
  709. String redirected = redirectTo.toASCIIString();
  710. if (!isValidRedirect(baseUrl, redirected, checkFor)) {
  711. throw new TransportException(uri,
  712. MessageFormat.format(JGitText.get().redirectBlocked,
  713. baseUrl, redirected));
  714. }
  715. redirected = redirected.substring(0, redirected.indexOf(checkFor));
  716. URIish result = new URIish(redirected);
  717. if (LOG.isInfoEnabled()) {
  718. LOG.info(MessageFormat.format(JGitText.get().redirectHttp,
  719. uri.setPass(null),
  720. Integer.valueOf(redirects), baseUrl, result));
  721. }
  722. return result;
  723. } catch (URISyntaxException e) {
  724. throw new TransportException(uri,
  725. MessageFormat.format(JGitText.get().invalidRedirectLocation,
  726. baseUrl, location),
  727. e);
  728. }
  729. }
  730. private boolean isValidRedirect(URL current, String next, String checkFor) {
  731. // Protocols must be the same, or current is "http" and next "https". We
  732. // do not follow redirects from https back to http.
  733. String oldProtocol = current.getProtocol().toLowerCase(Locale.ROOT);
  734. int schemeEnd = next.indexOf("://"); //$NON-NLS-1$
  735. if (schemeEnd < 0) {
  736. return false;
  737. }
  738. String newProtocol = next.substring(0, schemeEnd)
  739. .toLowerCase(Locale.ROOT);
  740. if (!oldProtocol.equals(newProtocol)) {
  741. if (!"https".equals(newProtocol)) { //$NON-NLS-1$
  742. return false;
  743. }
  744. }
  745. // git allows only rewriting the root, i.e., everything before INFO_REFS
  746. // or the service name
  747. if (!next.contains(checkFor)) {
  748. return false;
  749. }
  750. // Basically we should test here that whatever follows INFO_REFS is
  751. // unchanged. But since we re-construct the query part
  752. // anyway, it doesn't matter.
  753. return true;
  754. }
  755. private URL getServiceURL(String service)
  756. throws NotSupportedException {
  757. try {
  758. final StringBuilder b = new StringBuilder();
  759. b.append(baseUrl);
  760. if (b.charAt(b.length() - 1) != '/') {
  761. b.append('/');
  762. }
  763. b.append(Constants.INFO_REFS);
  764. if (useSmartHttp) {
  765. b.append(b.indexOf("?") < 0 ? '?' : '&'); //$NON-NLS-1$
  766. b.append("service="); //$NON-NLS-1$
  767. b.append(service);
  768. }
  769. return new URL(b.toString());
  770. } catch (MalformedURLException e) {
  771. throw new NotSupportedException(MessageFormat.format(JGitText.get().invalidURL, uri), e);
  772. }
  773. }
  774. /**
  775. * Open an HTTP connection.
  776. *
  777. * @param method HTTP request method
  778. * @param u url of the HTTP connection
  779. * @param acceptEncoding accept-encoding header option
  780. * @return the HTTP connection
  781. * @throws java.io.IOException
  782. * @since 4.6
  783. */
  784. protected HttpConnection httpOpen(String method, URL u,
  785. AcceptEncoding acceptEncoding) throws IOException {
  786. if (method == null || u == null || acceptEncoding == null) {
  787. throw new NullPointerException();
  788. }
  789. final Proxy proxy = HttpSupport.proxyFor(proxySelector, u);
  790. HttpConnection conn = connectionFactory.create(u, proxy);
  791. if (!sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$
  792. HttpSupport.disableSslVerify(conn);
  793. }
  794. // We must do our own redirect handling to implement git rules and to
  795. // handle http->https redirects
  796. conn.setInstanceFollowRedirects(false);
  797. conn.setRequestMethod(method);
  798. conn.setUseCaches(false);
  799. if (acceptEncoding == AcceptEncoding.GZIP) {
  800. conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP);
  801. }
  802. conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$
  803. if (UserAgent.get() != null) {
  804. conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get());
  805. }
  806. int timeOut = getTimeout();
  807. if (timeOut != -1) {
  808. int effTimeOut = timeOut * 1000;
  809. conn.setConnectTimeout(effTimeOut);
  810. conn.setReadTimeout(effTimeOut);
  811. }
  812. // set cookie header if necessary
  813. if (!relevantCookies.isEmpty()) {
  814. setCookieHeader(conn);
  815. }
  816. if (this.headers != null && !this.headers.isEmpty()) {
  817. for (Map.Entry<String, String> entry : this.headers.entrySet()) {
  818. conn.setRequestProperty(entry.getKey(), entry.getValue());
  819. }
  820. }
  821. authMethod.configureRequest(conn);
  822. return conn;
  823. }
  824. private void setCookieHeader(HttpConnection conn) {
  825. StringBuilder cookieHeaderValue = new StringBuilder();
  826. for (HttpCookie cookie : relevantCookies) {
  827. if (!cookie.hasExpired()) {
  828. if (cookieHeaderValue.length() > 0) {
  829. cookieHeaderValue.append(';');
  830. }
  831. cookieHeaderValue.append(cookie.toString());
  832. }
  833. }
  834. if (cookieHeaderValue.length() > 0) {
  835. conn.setRequestProperty(HDR_COOKIE, cookieHeaderValue.toString());
  836. }
  837. }
  838. final InputStream openInputStream(HttpConnection conn)
  839. throws IOException {
  840. InputStream input = conn.getInputStream();
  841. if (isGzipContent(conn))
  842. input = new GZIPInputStream(input);
  843. return input;
  844. }
  845. IOException wrongContentType(String expType, String actType) {
  846. final String why = MessageFormat.format(JGitText.get().expectedReceivedContentType, expType, actType);
  847. return new TransportException(uri, why);
  848. }
  849. private static NetscapeCookieFile getCookieFileFromConfig(
  850. HttpConfig config) {
  851. if (!StringUtils.isEmptyOrNull(config.getCookieFile())) {
  852. try {
  853. Path cookieFilePath = Paths.get(config.getCookieFile());
  854. return NetscapeCookieFileCache.getInstance(config)
  855. .getEntry(cookieFilePath);
  856. } catch (InvalidPathException e) {
  857. LOG.warn(MessageFormat.format(
  858. JGitText.get().couldNotReadCookieFile,
  859. config.getCookieFile()), e);
  860. }
  861. }
  862. return null;
  863. }
  864. private static Set<HttpCookie> filterCookies(NetscapeCookieFile cookieFile,
  865. URL url) {
  866. if (cookieFile != null) {
  867. return filterCookies(cookieFile.getCookies(true), url);
  868. }
  869. return Collections.emptySet();
  870. }
  871. /**
  872. *
  873. * @param allCookies
  874. * a list of cookies.
  875. * @param url
  876. * the url for which to filter the list of cookies.
  877. * @return only the cookies from {@code allCookies} which are relevant (i.e.
  878. * are not expired, have a matching domain, have a matching path and
  879. * have a matching secure attribute)
  880. */
  881. private static Set<HttpCookie> filterCookies(Set<HttpCookie> allCookies,
  882. URL url) {
  883. Set<HttpCookie> filteredCookies = new HashSet<>();
  884. for (HttpCookie cookie : allCookies) {
  885. if (cookie.hasExpired()) {
  886. continue;
  887. }
  888. if (!matchesCookieDomain(url.getHost(), cookie.getDomain())) {
  889. continue;
  890. }
  891. if (!matchesCookiePath(url.getPath(), cookie.getPath())) {
  892. continue;
  893. }
  894. if (cookie.getSecure() && !"https".equals(url.getProtocol())) { //$NON-NLS-1$
  895. continue;
  896. }
  897. filteredCookies.add(cookie);
  898. }
  899. return filteredCookies;
  900. }
  901. /**
  902. *
  903. * The utility method to check whether a host name is in a cookie's domain
  904. * or not. Similar to {@link HttpCookie#domainMatches(String, String)} but
  905. * implements domain matching rules according to
  906. * <a href="https://tools.ietf.org/html/rfc6265#section-5.1.3">RFC 6265,
  907. * section 5.1.3</a> instead of the rules from
  908. * <a href="https://tools.ietf.org/html/rfc2965#section-3.3">RFC 2965,
  909. * section 3.3.1</a>.
  910. * <p>
  911. * The former rules are also used by libcurl internally.
  912. * <p>
  913. * The rules are as follows
  914. *
  915. * A string matches another domain string if at least one of the following
  916. * conditions holds:
  917. * <ul>
  918. * <li>The domain string and the string are identical. (Note that both the
  919. * domain string and the string will have been canonicalized to lower case
  920. * at this point.)</li>
  921. * <li>All of the following conditions hold
  922. * <ul>
  923. * <li>The domain string is a suffix of the string.</li>
  924. * <li>The last character of the string that is not included in the domain
  925. * string is a %x2E (".") character.</li>
  926. * <li>The string is a host name (i.e., not an IP address).</li>
  927. * </ul>
  928. * </li>
  929. * </ul>
  930. *
  931. * @param host
  932. * the host to compare against the cookieDomain
  933. * @param cookieDomain
  934. * the domain to compare against
  935. * @return {@code true} if they domain-match; {@code false} if not
  936. *
  937. * @see <a href= "https://tools.ietf.org/html/rfc6265#section-5.1.3">RFC
  938. * 6265, section 5.1.3 (Domain Matching)</a>
  939. * @see <a href=
  940. * "https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8206092">JDK-8206092
  941. * : HttpCookie.domainMatches() does not match to sub-sub-domain</a>
  942. */
  943. static boolean matchesCookieDomain(String host, String cookieDomain) {
  944. cookieDomain = cookieDomain.toLowerCase(Locale.ROOT);
  945. host = host.toLowerCase(Locale.ROOT);
  946. if (host.equals(cookieDomain)) {
  947. return true;
  948. }
  949. if (!host.endsWith(cookieDomain)) {
  950. return false;
  951. }
  952. return host.charAt(host.length() - cookieDomain.length() - 1) == '.';
  953. }
  954. /**
  955. * The utility method to check whether a path is matching a cookie path
  956. * domain or not. The rules are defined by
  957. * <a href="https://tools.ietf.org/html/rfc6265#section-5.1.4">RFC 6265,
  958. * section 5.1.4</a>:
  959. *
  960. * A request-path path-matches a given cookie-path if at least one of the
  961. * following conditions holds:
  962. * <ul>
  963. * <li>The cookie-path and the request-path are identical.</li>
  964. * <li>The cookie-path is a prefix of the request-path, and the last
  965. * character of the cookie-path is %x2F ("/").</li>
  966. * <li>The cookie-path is a prefix of the request-path, and the first
  967. * character of the request-path that is not included in the cookie- path is
  968. * a %x2F ("/") character.</li>
  969. * </ul>
  970. * @param path
  971. * the path to check
  972. * @param cookiePath
  973. * the cookie's path
  974. *
  975. * @return {@code true} if they path-match; {@code false} if not
  976. */
  977. static boolean matchesCookiePath(String path, String cookiePath) {
  978. if (cookiePath.equals(path)) {
  979. return true;
  980. }
  981. if (!cookiePath.endsWith("/")) { //$NON-NLS-1$
  982. cookiePath += "/"; //$NON-NLS-1$
  983. }
  984. return path.startsWith(cookiePath);
  985. }
  986. private boolean isSmartHttp(HttpConnection c, String service) {
  987. final String expType = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$
  988. final String actType = c.getContentType();
  989. return expType.equals(actType);
  990. }
  991. private boolean isGzipContent(HttpConnection c) {
  992. return ENCODING_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING))
  993. || ENCODING_X_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING));
  994. }
  995. private void readSmartHeaders(InputStream in, String service)
  996. throws IOException {
  997. // A smart reply will have a '#' after the first 4 bytes, but
  998. // a dumb reply cannot contain a '#' until after byte 41. Do a
  999. // quick check to make sure its a smart reply before we parse
  1000. // as a pkt-line stream.
  1001. //
  1002. final byte[] magic = new byte[5];
  1003. IO.readFully(in, magic, 0, magic.length);
  1004. if (magic[4] != '#') {
  1005. throw new TransportException(uri, MessageFormat.format(
  1006. JGitText.get().expectedPktLineWithService, RawParseUtils.decode(magic)));
  1007. }
  1008. final PacketLineIn pckIn = new PacketLineIn(new UnionInputStream(
  1009. new ByteArrayInputStream(magic), in));
  1010. final String exp = "# service=" + service; //$NON-NLS-1$
  1011. final String act = pckIn.readString();
  1012. if (!exp.equals(act)) {
  1013. throw new TransportException(uri, MessageFormat.format(
  1014. JGitText.get().expectedGot, exp, act));
  1015. }
  1016. while (!PacketLineIn.isEnd(pckIn.readString())) {
  1017. // for now, ignore the remaining header lines
  1018. }
  1019. }
  1020. class HttpObjectDB extends WalkRemoteObjectDatabase {
  1021. private final URL httpObjectsUrl;
  1022. HttpObjectDB(URL b) {
  1023. httpObjectsUrl = b;
  1024. }
  1025. @Override
  1026. URIish getURI() {
  1027. return new URIish(httpObjectsUrl);
  1028. }
  1029. @Override
  1030. Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
  1031. try {
  1032. return readAlternates(INFO_HTTP_ALTERNATES);
  1033. } catch (FileNotFoundException err) {
  1034. // Fall through.
  1035. }
  1036. try {
  1037. return readAlternates(INFO_ALTERNATES);
  1038. } catch (FileNotFoundException err) {
  1039. // Fall through.
  1040. }
  1041. return null;
  1042. }
  1043. @Override
  1044. WalkRemoteObjectDatabase openAlternate(String location)
  1045. throws IOException {
  1046. return new HttpObjectDB(new URL(httpObjectsUrl, location));
  1047. }
  1048. @Override
  1049. BufferedReader openReader(String path) throws IOException {
  1050. // Line oriented readable content is likely to compress well.
  1051. // Request gzip encoding.
  1052. InputStream is = open(path, AcceptEncoding.GZIP).in;
  1053. return new BufferedReader(new InputStreamReader(is, UTF_8));
  1054. }
  1055. @Override
  1056. Collection<String> getPackNames() throws IOException {
  1057. final Collection<String> packs = new ArrayList<>();
  1058. try (BufferedReader br = openReader(INFO_PACKS)) {
  1059. for (;;) {
  1060. final String s = br.readLine();
  1061. if (s == null || s.length() == 0)
  1062. break;
  1063. if (!s.startsWith("P pack-") || !s.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$
  1064. throw invalidAdvertisement(s);
  1065. packs.add(s.substring(2));
  1066. }
  1067. return packs;
  1068. } catch (FileNotFoundException err) {
  1069. return packs;
  1070. }
  1071. }
  1072. @Override
  1073. FileStream open(String path) throws IOException {
  1074. return open(path, AcceptEncoding.UNSPECIFIED);
  1075. }
  1076. FileStream open(String path, AcceptEncoding acceptEncoding)
  1077. throws IOException {
  1078. final URL base = httpObjectsUrl;
  1079. final URL u = new URL(base, path);
  1080. final HttpConnection c = httpOpen(METHOD_GET, u, acceptEncoding);
  1081. switch (HttpSupport.response(c)) {
  1082. case HttpConnection.HTTP_OK:
  1083. final InputStream in = openInputStream(c);
  1084. // If content is being gzipped and then transferred, the content
  1085. // length in the header is the zipped content length, not the
  1086. // actual content length.
  1087. if (!isGzipContent(c)) {
  1088. final int len = c.getContentLength();
  1089. return new FileStream(in, len);
  1090. }
  1091. return new FileStream(in);
  1092. case HttpConnection.HTTP_NOT_FOUND:
  1093. throw new FileNotFoundException(u.toString());
  1094. default:
  1095. throw new IOException(u.toString() + ": " //$NON-NLS-1$
  1096. + HttpSupport.response(c) + " " //$NON-NLS-1$
  1097. + c.getResponseMessage());
  1098. }
  1099. }
  1100. Map<String, Ref> readAdvertisedImpl(final BufferedReader br)
  1101. throws IOException, PackProtocolException {
  1102. final TreeMap<String, Ref> avail = new TreeMap<>();
  1103. for (;;) {
  1104. String line = br.readLine();
  1105. if (line == null)
  1106. break;
  1107. final int tab = line.indexOf('\t');
  1108. if (tab < 0)
  1109. throw invalidAdvertisement(line);
  1110. String name;
  1111. final ObjectId id;
  1112. name = line.substring(tab + 1);
  1113. id = ObjectId.fromString(line.substring(0, tab));
  1114. if (name.endsWith("^{}")) { //$NON-NLS-1$
  1115. name = name.substring(0, name.length() - 3);
  1116. final Ref prior = avail.get(name);
  1117. if (prior == null)
  1118. throw outOfOrderAdvertisement(name);
  1119. if (prior.getPeeledObjectId() != null)
  1120. throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$
  1121. avail.put(name, new ObjectIdRef.PeeledTag(
  1122. Ref.Storage.NETWORK, name,
  1123. prior.getObjectId(), id));
  1124. } else {
  1125. Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag(
  1126. Ref.Storage.NETWORK, name, id));
  1127. if (prior != null)
  1128. throw duplicateAdvertisement(name);
  1129. }
  1130. }
  1131. return avail;
  1132. }
  1133. private PackProtocolException outOfOrderAdvertisement(String n) {
  1134. return new PackProtocolException(MessageFormat.format(JGitText.get().advertisementOfCameBefore, n, n));
  1135. }
  1136. private PackProtocolException invalidAdvertisement(String n) {
  1137. return new PackProtocolException(MessageFormat.format(JGitText.get().invalidAdvertisementOf, n));
  1138. }
  1139. private PackProtocolException duplicateAdvertisement(String n) {
  1140. return new PackProtocolException(MessageFormat.format(JGitText.get().duplicateAdvertisementsOf, n));
  1141. }
  1142. @Override
  1143. void close() {
  1144. // We do not maintain persistent connections.
  1145. }
  1146. }
  1147. class SmartHttpFetchConnection extends BasePackFetchConnection {
  1148. private MultiRequestService svc;
  1149. SmartHttpFetchConnection(InputStream advertisement)
  1150. throws TransportException {
  1151. super(TransportHttp.this);
  1152. statelessRPC = true;
  1153. init(advertisement, DisabledOutputStream.INSTANCE);
  1154. outNeedsEnd = false;
  1155. readAdvertisedRefs();
  1156. }
  1157. @Override
  1158. protected void doFetch(final ProgressMonitor monitor,
  1159. final Collection<Ref> want, final Set<ObjectId> have,
  1160. final OutputStream outputStream) throws TransportException {
  1161. try {
  1162. svc = new MultiRequestService(SVC_UPLOAD_PACK);
  1163. init(svc.getInputStream(), svc.getOutputStream());
  1164. super.doFetch(monitor, want, have, outputStream);
  1165. } finally {
  1166. svc = null;
  1167. }
  1168. }
  1169. @Override
  1170. protected void onReceivePack() {
  1171. svc.finalRequest = true;
  1172. }
  1173. }
  1174. class SmartHttpPushConnection extends BasePackPushConnection {
  1175. SmartHttpPushConnection(InputStream advertisement)
  1176. throws TransportException {
  1177. super(TransportHttp.this);
  1178. statelessRPC = true;
  1179. init(advertisement, DisabledOutputStream.INSTANCE);
  1180. outNeedsEnd = false;
  1181. readAdvertisedRefs();
  1182. }
  1183. @Override
  1184. protected void doPush(final ProgressMonitor monitor,
  1185. final Map<String, RemoteRefUpdate> refUpdates,
  1186. OutputStream outputStream) throws TransportException {
  1187. final Service svc = new MultiRequestService(SVC_RECEIVE_PACK);
  1188. init(svc.getInputStream(), svc.getOutputStream());
  1189. super.doPush(monitor, refUpdates, outputStream);
  1190. }
  1191. }
  1192. /** Basic service for sending and receiving HTTP requests. */
  1193. abstract class Service {
  1194. protected final String serviceName;
  1195. protected final String requestType;
  1196. protected final String responseType;
  1197. protected HttpConnection conn;
  1198. protected HttpOutputStream out;
  1199. protected final HttpExecuteStream execute;
  1200. final UnionInputStream in;
  1201. Service(String serviceName) {
  1202. this.serviceName = serviceName;
  1203. this.requestType = "application/x-" + serviceName + "-request"; //$NON-NLS-1$ //$NON-NLS-2$
  1204. this.responseType = "application/x-" + serviceName + "-result"; //$NON-NLS-1$ //$NON-NLS-2$
  1205. this.out = new HttpOutputStream();
  1206. this.execute = new HttpExecuteStream();
  1207. this.in = new UnionInputStream(execute);
  1208. }
  1209. void openStream() throws IOException {
  1210. conn = httpOpen(METHOD_POST, new URL(baseUrl, serviceName),
  1211. AcceptEncoding.GZIP);
  1212. conn.setInstanceFollowRedirects(false);
  1213. conn.setDoOutput(true);
  1214. conn.setRequestProperty(HDR_CONTENT_TYPE, requestType);
  1215. conn.setRequestProperty(HDR_ACCEPT, responseType);
  1216. }
  1217. void sendRequest() throws IOException {
  1218. // Try to compress the content, but only if that is smaller.
  1219. TemporaryBuffer buf = new TemporaryBuffer.Heap(
  1220. http.getPostBuffer());
  1221. try (GZIPOutputStream gzip = new GZIPOutputStream(buf)) {
  1222. out.writeTo(gzip, null);
  1223. if (out.length() < buf.length())
  1224. buf = out;
  1225. } catch (IOException err) {
  1226. // Most likely caused by overflowing the buffer, meaning
  1227. // its larger if it were compressed. Don't compress.
  1228. buf = out;
  1229. }
  1230. HttpAuthMethod authenticator = null;
  1231. Collection<Type> ignoreTypes = EnumSet.noneOf(Type.class);
  1232. // Counts number of repeated authentication attempts using the same
  1233. // authentication scheme
  1234. int authAttempts = 1;
  1235. int redirects = 0;
  1236. for (;;) {
  1237. try {
  1238. // The very first time we will try with the authentication
  1239. // method used on the initial GET request. This is a hint
  1240. // only; it may fail. If so, we'll then re-try with proper
  1241. // 401 handling, going through the available authentication
  1242. // schemes.
  1243. openStream();
  1244. if (buf != out) {
  1245. conn.setRequestProperty(HDR_CONTENT_ENCODING,
  1246. ENCODING_GZIP);
  1247. }
  1248. conn.setFixedLengthStreamingMode((int) buf.length());
  1249. try (OutputStream httpOut = conn.getOutputStream()) {
  1250. buf.writeTo(httpOut, null);
  1251. }
  1252. final int status = HttpSupport.response(conn);
  1253. switch (status) {
  1254. case HttpConnection.HTTP_OK:
  1255. // We're done.
  1256. return;
  1257. case HttpConnection.HTTP_NOT_FOUND:
  1258. throw createNotFoundException(uri, conn.getURL(),
  1259. conn.getResponseMessage());
  1260. case HttpConnection.HTTP_FORBIDDEN:
  1261. throw new TransportException(uri,
  1262. MessageFormat.format(
  1263. JGitText.get().serviceNotPermitted,
  1264. baseUrl, serviceName));
  1265. case HttpConnection.HTTP_MOVED_PERM:
  1266. case HttpConnection.HTTP_MOVED_TEMP:
  1267. case HttpConnection.HTTP_11_MOVED_PERM:
  1268. case HttpConnection.HTTP_11_MOVED_TEMP:
  1269. // SEE_OTHER after a POST doesn't make sense for a git
  1270. // server, so we don't handle it here and thus we'll
  1271. // report an error in openResponse() later on.
  1272. if (http.getFollowRedirects() != HttpRedirectMode.TRUE) {
  1273. // Let openResponse() issue an error
  1274. return;
  1275. }
  1276. currentUri = redirect(conn.getURL(),
  1277. conn.getHeaderField(HDR_LOCATION),
  1278. '/' + serviceName, redirects++);
  1279. try {
  1280. baseUrl = toURL(currentUri);
  1281. } catch (MalformedURLException e) {
  1282. throw new TransportException(uri,
  1283. MessageFormat.format(
  1284. JGitText.get().invalidRedirectLocation,
  1285. baseUrl, currentUri),
  1286. e);
  1287. }
  1288. continue;
  1289. case HttpConnection.HTTP_UNAUTHORIZED:
  1290. HttpAuthMethod nextMethod = HttpAuthMethod
  1291. .scanResponse(conn, ignoreTypes);
  1292. switch (nextMethod.getType()) {
  1293. case NONE:
  1294. throw new TransportException(uri,
  1295. MessageFormat.format(
  1296. JGitText.get().authenticationNotSupported,
  1297. conn.getURL()));
  1298. case NEGOTIATE:
  1299. // RFC 4559 states "When using the SPNEGO [...] with
  1300. // [...] POST, the authentication should be complete
  1301. // [...] before sending the user data." So in theory
  1302. // the initial GET should have been authenticated
  1303. // already. (Unless there was a redirect?)
  1304. //
  1305. // We try this only once:
  1306. ignoreTypes.add(HttpAuthMethod.Type.NEGOTIATE);
  1307. if (authenticator != null) {
  1308. ignoreTypes.add(authenticator.getType());
  1309. }
  1310. authAttempts = 1;
  1311. // We only do the Kerberos part of SPNEGO, which
  1312. // requires only one round.
  1313. break;
  1314. default:
  1315. // DIGEST or BASIC. Let's be sure we ignore
  1316. // NEGOTIATE; if it was available, we have tried it
  1317. // before.
  1318. ignoreTypes.add(HttpAuthMethod.Type.NEGOTIATE);
  1319. if (authenticator == null || authenticator
  1320. .getType() != nextMethod.getType()) {
  1321. if (authenticator != null) {
  1322. ignoreTypes.add(authenticator.getType());
  1323. }
  1324. authAttempts = 1;
  1325. }
  1326. break;
  1327. }
  1328. authMethod = nextMethod;
  1329. authenticator = nextMethod;
  1330. CredentialsProvider credentialsProvider = getCredentialsProvider();
  1331. if (credentialsProvider == null) {
  1332. throw new TransportException(uri,
  1333. JGitText.get().noCredentialsProvider);
  1334. }
  1335. if (authAttempts > 1) {
  1336. credentialsProvider.reset(currentUri);
  1337. }
  1338. if (3 < authAttempts || !authMethod
  1339. .authorize(currentUri, credentialsProvider)) {
  1340. throw new TransportException(uri,
  1341. JGitText.get().notAuthorized);
  1342. }
  1343. authAttempts++;
  1344. continue;
  1345. default:
  1346. // Just return here; openResponse() will report an
  1347. // appropriate error.
  1348. return;
  1349. }
  1350. } catch (SSLHandshakeException e) {
  1351. handleSslFailure(e);
  1352. continue; // Re-try
  1353. } catch (IOException e) {
  1354. if (authenticator == null || authMethod
  1355. .getType() != HttpAuthMethod.Type.NONE) {
  1356. // Can happen for instance if the server advertises
  1357. // Negotiate, but the client isn't configured for
  1358. // Kerberos. The first time (authenticator == null) we
  1359. // must re-try even if the authMethod was NONE: this may
  1360. // occur if the server advertised NTLM on the GET
  1361. // and the HttpConnection managed to successfully
  1362. // authenticate under the hood with NTLM. We might not
  1363. // have picked this up on the GET's 200 response.
  1364. if (authMethod.getType() != HttpAuthMethod.Type.NONE) {
  1365. ignoreTypes.add(authMethod.getType());
  1366. }
  1367. // Start over with the remaining available methods.
  1368. authMethod = HttpAuthMethod.Type.NONE.method(null);
  1369. authenticator = authMethod;
  1370. authAttempts = 1;
  1371. continue;
  1372. }
  1373. throw e;
  1374. }
  1375. }
  1376. }
  1377. void openResponse() throws IOException {
  1378. final int status = HttpSupport.response(conn);
  1379. if (status != HttpConnection.HTTP_OK) {
  1380. throw new TransportException(uri, status + " " //$NON-NLS-1$
  1381. + conn.getResponseMessage());
  1382. }
  1383. final String contentType = conn.getContentType();
  1384. if (!responseType.equals(contentType)) {
  1385. conn.getInputStream().close();
  1386. throw wrongContentType(responseType, contentType);
  1387. }
  1388. }
  1389. HttpOutputStream getOutputStream() {
  1390. return out;
  1391. }
  1392. InputStream getInputStream() {
  1393. return in;
  1394. }
  1395. abstract void execute() throws IOException;
  1396. class HttpExecuteStream extends InputStream {
  1397. @Override
  1398. public int read() throws IOException {
  1399. execute();
  1400. return -1;
  1401. }
  1402. @Override
  1403. public int read(byte[] b, int off, int len) throws IOException {
  1404. execute();
  1405. return -1;
  1406. }
  1407. @Override
  1408. public long skip(long n) throws IOException {
  1409. execute();
  1410. return 0;
  1411. }
  1412. }
  1413. class HttpOutputStream extends TemporaryBuffer {
  1414. HttpOutputStream() {
  1415. super(http.getPostBuffer());
  1416. }
  1417. @Override
  1418. protected OutputStream overflow() throws IOException {
  1419. openStream();
  1420. conn.setChunkedStreamingMode(0);
  1421. return conn.getOutputStream();
  1422. }
  1423. }
  1424. }
  1425. /**
  1426. * State required to speak multiple HTTP requests with the remote.
  1427. * <p>
  1428. * A service wrapper provides a normal looking InputStream and OutputStream
  1429. * pair which are connected via HTTP to the named remote service. Writing to
  1430. * the OutputStream is buffered until either the buffer overflows, or
  1431. * reading from the InputStream occurs. If overflow occurs HTTP/1.1 and its
  1432. * chunked transfer encoding is used to stream the request data to the
  1433. * remote service. If the entire request fits in the memory buffer, the
  1434. * older HTTP/1.0 standard and a fixed content length is used instead.
  1435. * <p>
  1436. * It is an error to attempt to read without there being outstanding data
  1437. * ready for transmission on the OutputStream.
  1438. * <p>
  1439. * No state is preserved between write-read request pairs. The caller is
  1440. * responsible for replaying state vector information as part of the request
  1441. * data written to the OutputStream. Any session HTTP cookies may or may not
  1442. * be preserved between requests, it is left up to the JVM's implementation
  1443. * of the HTTP client.
  1444. */
  1445. class MultiRequestService extends Service {
  1446. boolean finalRequest;
  1447. MultiRequestService(String serviceName) {
  1448. super(serviceName);
  1449. }
  1450. /** Keep opening send-receive pairs to the given URI. */
  1451. @Override
  1452. void execute() throws IOException {
  1453. out.close();
  1454. if (conn == null) {
  1455. if (out.length() == 0) {
  1456. // Request output hasn't started yet, but more data is being
  1457. // requested. If there is no request data buffered and the
  1458. // final request was already sent, do nothing to ensure the
  1459. // caller is shown EOF on the InputStream; otherwise an
  1460. // programming error has occurred within this module.
  1461. if (finalRequest)
  1462. return;
  1463. throw new TransportException(uri,
  1464. JGitText.get().startingReadStageWithoutWrittenRequestDataPendingIsNotSupported);
  1465. }
  1466. sendRequest();
  1467. }
  1468. out.reset();
  1469. openResponse();
  1470. in.add(openInputStream(conn));
  1471. if (!finalRequest)
  1472. in.add(execute);
  1473. conn = null;
  1474. }
  1475. }
  1476. /** Service for maintaining a single long-poll connection. */
  1477. class LongPollService extends Service {
  1478. /**
  1479. * @param serviceName
  1480. */
  1481. LongPollService(String serviceName) {
  1482. super(serviceName);
  1483. }
  1484. /** Only open one send-receive request. */
  1485. @Override
  1486. void execute() throws IOException {
  1487. out.close();
  1488. if (conn == null)
  1489. sendRequest();
  1490. openResponse();
  1491. in.add(openInputStream(conn));
  1492. }
  1493. }
  1494. }