You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

AmazonS3.java 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755
  1. /*
  2. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
  3. *
  4. * This program and the accompanying materials are made available under the
  5. * terms of the Eclipse Distribution License v. 1.0 which is available at
  6. * https://www.eclipse.org/org/documents/edl-v10.php.
  7. *
  8. * SPDX-License-Identifier: BSD-3-Clause
  9. */
  10. package org.eclipse.jgit.transport;
  11. import static java.nio.charset.StandardCharsets.UTF_8;
  12. import java.io.ByteArrayOutputStream;
  13. import java.io.File;
  14. import java.io.FileInputStream;
  15. import java.io.FileNotFoundException;
  16. import java.io.IOException;
  17. import java.io.InputStream;
  18. import java.io.OutputStream;
  19. import java.net.HttpURLConnection;
  20. import java.net.Proxy;
  21. import java.net.ProxySelector;
  22. import java.net.URL;
  23. import java.net.URLConnection;
  24. import java.security.DigestOutputStream;
  25. import java.security.GeneralSecurityException;
  26. import java.security.InvalidKeyException;
  27. import java.security.MessageDigest;
  28. import java.security.NoSuchAlgorithmException;
  29. import java.text.MessageFormat;
  30. import java.text.SimpleDateFormat;
  31. import java.util.ArrayList;
  32. import java.util.Collections;
  33. import java.util.Comparator;
  34. import java.util.Date;
  35. import java.util.HashSet;
  36. import java.util.Iterator;
  37. import java.util.List;
  38. import java.util.Locale;
  39. import java.util.Map;
  40. import java.util.Properties;
  41. import java.util.Set;
  42. import java.util.SortedMap;
  43. import java.util.TimeZone;
  44. import java.util.TreeMap;
  45. import java.util.stream.Collectors;
  46. import java.time.Instant;
  47. import javax.crypto.Mac;
  48. import javax.crypto.spec.SecretKeySpec;
  49. import org.eclipse.jgit.internal.JGitText;
  50. import org.eclipse.jgit.lib.Constants;
  51. import org.eclipse.jgit.lib.NullProgressMonitor;
  52. import org.eclipse.jgit.lib.ProgressMonitor;
  53. import org.eclipse.jgit.util.Base64;
  54. import org.eclipse.jgit.util.HttpSupport;
  55. import org.eclipse.jgit.util.StringUtils;
  56. import org.eclipse.jgit.util.TemporaryBuffer;
  57. import org.xml.sax.Attributes;
  58. import org.xml.sax.InputSource;
  59. import org.xml.sax.SAXException;
  60. import org.xml.sax.XMLReader;
  61. import org.xml.sax.helpers.DefaultHandler;
  62. import org.xml.sax.helpers.XMLReaderFactory;
  63. /**
  64. * A simple HTTP REST client for the Amazon S3 service.
  65. * <p>
  66. * This client uses the REST API to communicate with the Amazon S3 servers and
  67. * read or write content through a bucket that the user has access to. It is a
  68. * very lightweight implementation of the S3 API and therefore does not have all
  69. * of the bells and whistles of popular client implementations.
  70. * <p>
  71. * Authentication is always performed using the user's AWSAccessKeyId and their
  72. * private AWSSecretAccessKey.
  73. * <p>
  74. * Optional client-side encryption may be enabled if requested. The format is
  75. * compatible with <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a>,
  76. * a popular Java based Amazon S3 client library. Enabling encryption can hide
  77. * sensitive data from the operators of the S3 service.
  78. */
  79. public class AmazonS3 {
  80. private static final Set<String> SIGNED_HEADERS;
  81. private static final String HMAC = "HmacSHA1"; //$NON-NLS-1$
  82. private static final String X_AMZ_ACL = "x-amz-acl"; //$NON-NLS-1$
  83. private static final String X_AMZ_META = "x-amz-meta-"; //$NON-NLS-1$
  84. static {
  85. SIGNED_HEADERS = new HashSet<>();
  86. SIGNED_HEADERS.add("content-type"); //$NON-NLS-1$
  87. SIGNED_HEADERS.add("content-md5"); //$NON-NLS-1$
  88. SIGNED_HEADERS.add("date"); //$NON-NLS-1$
  89. }
  90. private static boolean isSignedHeader(String name) {
  91. final String nameLC = StringUtils.toLowerCase(name);
  92. return SIGNED_HEADERS.contains(nameLC) || nameLC.startsWith("x-amz-"); //$NON-NLS-1$
  93. }
  94. private static String toCleanString(List<String> list) {
  95. final StringBuilder s = new StringBuilder();
  96. for (String v : list) {
  97. if (s.length() > 0)
  98. s.append(',');
  99. s.append(v.replaceAll("\n", "").trim()); //$NON-NLS-1$ //$NON-NLS-2$
  100. }
  101. return s.toString();
  102. }
  103. private static String remove(Map<String, String> m, String k) {
  104. final String r = m.remove(k);
  105. return r != null ? r : ""; //$NON-NLS-1$
  106. }
  107. private static String httpNow() {
  108. final String tz = "GMT"; //$NON-NLS-1$
  109. final SimpleDateFormat fmt;
  110. fmt = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss", Locale.US); //$NON-NLS-1$
  111. fmt.setTimeZone(TimeZone.getTimeZone(tz));
  112. return fmt.format(new Date()) + " " + tz; //$NON-NLS-1$
  113. }
  114. private static MessageDigest newMD5() {
  115. try {
  116. return MessageDigest.getInstance("MD5"); //$NON-NLS-1$
  117. } catch (NoSuchAlgorithmException e) {
  118. throw new RuntimeException(JGitText.get().JRELacksMD5Implementation, e);
  119. }
  120. }
  121. /** AWSAccessKeyId, public string that identifies the user's account. */
  122. private final String publicKey;
  123. /** Decoded form of the private AWSSecretAccessKey, to sign requests. */
  124. private final SecretKeySpec privateKey;
  125. /** Our HTTP proxy support, in case we are behind a firewall. */
  126. private final ProxySelector proxySelector;
  127. /** ACL to apply to created objects. */
  128. private final String acl;
  129. /** Maximum number of times to try an operation. */
  130. final int maxAttempts;
  131. /** Encryption algorithm, may be a null instance that provides pass-through. */
  132. private final WalkEncryption encryption;
  133. /** Directory for locally buffered content. */
  134. private final File tmpDir;
  135. /** S3 Bucket Domain. */
  136. private final String domain;
  137. /** Property names used in amazon connection configuration file. */
  138. interface Keys {
  139. String ACCESS_KEY = "accesskey"; //$NON-NLS-1$
  140. String SECRET_KEY = "secretkey"; //$NON-NLS-1$
  141. String PASSWORD = "password"; //$NON-NLS-1$
  142. String CRYPTO_ALG = "crypto.algorithm"; //$NON-NLS-1$
  143. String CRYPTO_VER = "crypto.version"; //$NON-NLS-1$
  144. String ACL = "acl"; //$NON-NLS-1$
  145. String DOMAIN = "domain"; //$NON-NLS-1$
  146. String HTTP_RETRY = "httpclient.retry-max"; //$NON-NLS-1$
  147. String TMP_DIR = "tmpdir"; //$NON-NLS-1$
  148. }
  149. /**
  150. * Create a new S3 client for the supplied user information.
  151. * <p>
  152. * The connection properties are a subset of those supported by the popular
  153. * <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a> library.
  154. * For example:
  155. *
  156. * <pre>
  157. * # AWS Access and Secret Keys (required)
  158. * accesskey: &lt;YourAWSAccessKey&gt;
  159. * secretkey: &lt;YourAWSSecretKey&gt;
  160. *
  161. * # Access Control List setting to apply to uploads, must be one of:
  162. * # PRIVATE, PUBLIC_READ (defaults to PRIVATE).
  163. * acl: PRIVATE
  164. *
  165. * # S3 Domain
  166. * # AWS S3 Region Domain (defaults to s3.amazonaws.com)
  167. * domain: s3.amazonaws.com
  168. *
  169. * # Number of times to retry after internal error from S3.
  170. * httpclient.retry-max: 3
  171. *
  172. * # End-to-end encryption (hides content from S3 owners)
  173. * password: &lt;encryption pass-phrase&gt;
  174. * crypto.algorithm: PBEWithMD5AndDES
  175. * </pre>
  176. *
  177. * @param props
  178. * connection properties.
  179. */
  180. public AmazonS3(final Properties props) {
  181. domain = props.getProperty(Keys.DOMAIN, "s3.amazonaws.com"); //$NON-NLS-1$
  182. publicKey = props.getProperty(Keys.ACCESS_KEY);
  183. if (publicKey == null)
  184. throw new IllegalArgumentException(JGitText.get().missingAccesskey);
  185. final String secret = props.getProperty(Keys.SECRET_KEY);
  186. if (secret == null)
  187. throw new IllegalArgumentException(JGitText.get().missingSecretkey);
  188. privateKey = new SecretKeySpec(Constants.encodeASCII(secret), HMAC);
  189. final String pacl = props.getProperty(Keys.ACL, "PRIVATE"); //$NON-NLS-1$
  190. if (StringUtils.equalsIgnoreCase("PRIVATE", pacl)) //$NON-NLS-1$
  191. acl = "private"; //$NON-NLS-1$
  192. else if (StringUtils.equalsIgnoreCase("PUBLIC", pacl)) //$NON-NLS-1$
  193. acl = "public-read"; //$NON-NLS-1$
  194. else if (StringUtils.equalsIgnoreCase("PUBLIC-READ", pacl)) //$NON-NLS-1$
  195. acl = "public-read"; //$NON-NLS-1$
  196. else if (StringUtils.equalsIgnoreCase("PUBLIC_READ", pacl)) //$NON-NLS-1$
  197. acl = "public-read"; //$NON-NLS-1$
  198. else
  199. throw new IllegalArgumentException("Invalid acl: " + pacl); //$NON-NLS-1$
  200. try {
  201. encryption = WalkEncryption.instance(props);
  202. } catch (GeneralSecurityException e) {
  203. throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
  204. }
  205. maxAttempts = Integer
  206. .parseInt(props.getProperty(Keys.HTTP_RETRY, "3")); //$NON-NLS-1$
  207. proxySelector = ProxySelector.getDefault();
  208. String tmp = props.getProperty(Keys.TMP_DIR);
  209. tmpDir = tmp != null && tmp.length() > 0 ? new File(tmp) : null;
  210. }
  211. /**
  212. * Get the content of a bucket object.
  213. *
  214. * @param bucket
  215. * name of the bucket storing the object.
  216. * @param key
  217. * key of the object within its bucket.
  218. * @return connection to stream the content of the object. The request
  219. * properties of the connection may not be modified by the caller as
  220. * the request parameters have already been signed.
  221. * @throws java.io.IOException
  222. * sending the request was not possible.
  223. */
  224. public URLConnection get(String bucket, String key)
  225. throws IOException {
  226. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  227. final HttpURLConnection c = open("GET", bucket, key); //$NON-NLS-1$
  228. authorize(c);
  229. switch (HttpSupport.response(c)) {
  230. case HttpURLConnection.HTTP_OK:
  231. encryption.validate(c, X_AMZ_META);
  232. return c;
  233. case HttpURLConnection.HTTP_NOT_FOUND:
  234. throw new FileNotFoundException(key);
  235. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  236. continue;
  237. default:
  238. throw error(JGitText.get().s3ActionReading, key, c);
  239. }
  240. }
  241. throw maxAttempts(JGitText.get().s3ActionReading, key);
  242. }
  243. /**
  244. * Decrypt an input stream from {@link #get(String, String)}.
  245. *
  246. * @param u
  247. * connection previously created by {@link #get(String, String)}}.
  248. * @return stream to read plain text from.
  249. * @throws java.io.IOException
  250. * decryption could not be configured.
  251. */
  252. public InputStream decrypt(URLConnection u) throws IOException {
  253. return encryption.decrypt(u.getInputStream());
  254. }
  255. /**
  256. * List the names of keys available within a bucket.
  257. * <p>
  258. * This method is primarily meant for obtaining a "recursive directory
  259. * listing" rooted under the specified bucket and prefix location.
  260. * It returns the keys sorted in reverse order of LastModified time
  261. * (freshest keys first).
  262. *
  263. * @param bucket
  264. * name of the bucket whose objects should be listed.
  265. * @param prefix
  266. * common prefix to filter the results by. Must not be null.
  267. * Supplying the empty string will list all keys in the bucket.
  268. * Supplying a non-empty string will act as though a trailing '/'
  269. * appears in prefix, even if it does not.
  270. * @return list of keys starting with <code>prefix</code>, after removing
  271. * <code>prefix</code> (or <code>prefix + "/"</code>)from all
  272. * of them.
  273. * @throws java.io.IOException
  274. * sending the request was not possible, or the response XML
  275. * document could not be parsed properly.
  276. */
  277. public List<String> list(String bucket, String prefix)
  278. throws IOException {
  279. if (prefix.length() > 0 && !prefix.endsWith("/")) //$NON-NLS-1$
  280. prefix += "/"; //$NON-NLS-1$
  281. final ListParser lp = new ListParser(bucket, prefix);
  282. do {
  283. lp.list();
  284. } while (lp.truncated);
  285. Comparator<KeyInfo> comparator = Comparator.comparingLong(KeyInfo::getLastModifiedSecs);
  286. return lp.entries.stream().sorted(comparator.reversed())
  287. .map(KeyInfo::getName).collect(Collectors.toList());
  288. }
  289. /**
  290. * Delete a single object.
  291. * <p>
  292. * Deletion always succeeds, even if the object does not exist.
  293. *
  294. * @param bucket
  295. * name of the bucket storing the object.
  296. * @param key
  297. * key of the object within its bucket.
  298. * @throws java.io.IOException
  299. * deletion failed due to communications error.
  300. */
  301. public void delete(String bucket, String key)
  302. throws IOException {
  303. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  304. final HttpURLConnection c = open("DELETE", bucket, key); //$NON-NLS-1$
  305. authorize(c);
  306. switch (HttpSupport.response(c)) {
  307. case HttpURLConnection.HTTP_NO_CONTENT:
  308. return;
  309. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  310. continue;
  311. default:
  312. throw error(JGitText.get().s3ActionDeletion, key, c);
  313. }
  314. }
  315. throw maxAttempts(JGitText.get().s3ActionDeletion, key);
  316. }
  317. /**
  318. * Atomically create or replace a single small object.
  319. * <p>
  320. * This form is only suitable for smaller contents, where the caller can
  321. * reasonable fit the entire thing into memory.
  322. * <p>
  323. * End-to-end data integrity is assured by internally computing the MD5
  324. * checksum of the supplied data and transmitting the checksum along with
  325. * the data itself.
  326. *
  327. * @param bucket
  328. * name of the bucket storing the object.
  329. * @param key
  330. * key of the object within its bucket.
  331. * @param data
  332. * new data content for the object. Must not be null. Zero length
  333. * array will create a zero length object.
  334. * @throws java.io.IOException
  335. * creation/updating failed due to communications error.
  336. */
  337. public void put(String bucket, String key, byte[] data)
  338. throws IOException {
  339. if (encryption != WalkEncryption.NONE) {
  340. // We have to copy to produce the cipher text anyway so use
  341. // the large object code path as it supports that behavior.
  342. //
  343. try (OutputStream os = beginPut(bucket, key, null, null)) {
  344. os.write(data);
  345. }
  346. return;
  347. }
  348. final String md5str = Base64.encodeBytes(newMD5().digest(data));
  349. final String lenstr = String.valueOf(data.length);
  350. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  351. final HttpURLConnection c = open("PUT", bucket, key); //$NON-NLS-1$
  352. c.setRequestProperty("Content-Length", lenstr); //$NON-NLS-1$
  353. c.setRequestProperty("Content-MD5", md5str); //$NON-NLS-1$
  354. c.setRequestProperty(X_AMZ_ACL, acl);
  355. authorize(c);
  356. c.setDoOutput(true);
  357. c.setFixedLengthStreamingMode(data.length);
  358. try (OutputStream os = c.getOutputStream()) {
  359. os.write(data);
  360. }
  361. switch (HttpSupport.response(c)) {
  362. case HttpURLConnection.HTTP_OK:
  363. return;
  364. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  365. continue;
  366. default:
  367. throw error(JGitText.get().s3ActionWriting, key, c);
  368. }
  369. }
  370. throw maxAttempts(JGitText.get().s3ActionWriting, key);
  371. }
  372. /**
  373. * Atomically create or replace a single large object.
  374. * <p>
  375. * Initially the returned output stream buffers data into memory, but if the
  376. * total number of written bytes starts to exceed an internal limit the data
  377. * is spooled to a temporary file on the local drive.
  378. * <p>
  379. * Network transmission is attempted only when <code>close()</code> gets
  380. * called at the end of output. Closing the returned stream can therefore
  381. * take significant time, especially if the written content is very large.
  382. * <p>
  383. * End-to-end data integrity is assured by internally computing the MD5
  384. * checksum of the supplied data and transmitting the checksum along with
  385. * the data itself.
  386. *
  387. * @param bucket
  388. * name of the bucket storing the object.
  389. * @param key
  390. * key of the object within its bucket.
  391. * @param monitor
  392. * (optional) progress monitor to post upload completion to
  393. * during the stream's close method.
  394. * @param monitorTask
  395. * (optional) task name to display during the close method.
  396. * @return a stream which accepts the new data, and transmits once closed.
  397. * @throws java.io.IOException
  398. * if encryption was enabled it could not be configured.
  399. */
  400. public OutputStream beginPut(final String bucket, final String key,
  401. final ProgressMonitor monitor, final String monitorTask)
  402. throws IOException {
  403. final MessageDigest md5 = newMD5();
  404. final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(tmpDir) {
  405. @Override
  406. public void close() throws IOException {
  407. super.close();
  408. try {
  409. putImpl(bucket, key, md5.digest(), this, monitor,
  410. monitorTask);
  411. } finally {
  412. destroy();
  413. }
  414. }
  415. };
  416. return encryption.encrypt(new DigestOutputStream(buffer, md5));
  417. }
  418. void putImpl(final String bucket, final String key,
  419. final byte[] csum, final TemporaryBuffer buf,
  420. ProgressMonitor monitor, String monitorTask) throws IOException {
  421. if (monitor == null)
  422. monitor = NullProgressMonitor.INSTANCE;
  423. if (monitorTask == null)
  424. monitorTask = MessageFormat.format(JGitText.get().progressMonUploading, key);
  425. final String md5str = Base64.encodeBytes(csum);
  426. final long len = buf.length();
  427. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  428. final HttpURLConnection c = open("PUT", bucket, key); //$NON-NLS-1$
  429. c.setFixedLengthStreamingMode(len);
  430. c.setRequestProperty("Content-MD5", md5str); //$NON-NLS-1$
  431. c.setRequestProperty(X_AMZ_ACL, acl);
  432. encryption.request(c, X_AMZ_META);
  433. authorize(c);
  434. c.setDoOutput(true);
  435. monitor.beginTask(monitorTask, (int) (len / 1024));
  436. try (OutputStream os = c.getOutputStream()) {
  437. buf.writeTo(os, monitor);
  438. } finally {
  439. monitor.endTask();
  440. }
  441. switch (HttpSupport.response(c)) {
  442. case HttpURLConnection.HTTP_OK:
  443. return;
  444. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  445. continue;
  446. default:
  447. throw error(JGitText.get().s3ActionWriting, key, c);
  448. }
  449. }
  450. throw maxAttempts(JGitText.get().s3ActionWriting, key);
  451. }
  452. IOException error(final String action, final String key,
  453. final HttpURLConnection c) throws IOException {
  454. final IOException err = new IOException(MessageFormat.format(
  455. JGitText.get().amazonS3ActionFailed, action, key,
  456. Integer.valueOf(HttpSupport.response(c)),
  457. c.getResponseMessage()));
  458. if (c.getErrorStream() == null) {
  459. return err;
  460. }
  461. try (InputStream errorStream = c.getErrorStream()) {
  462. final ByteArrayOutputStream b = new ByteArrayOutputStream();
  463. byte[] buf = new byte[2048];
  464. for (;;) {
  465. final int n = errorStream.read(buf);
  466. if (n < 0) {
  467. break;
  468. }
  469. if (n > 0) {
  470. b.write(buf, 0, n);
  471. }
  472. }
  473. buf = b.toByteArray();
  474. if (buf.length > 0) {
  475. err.initCause(new IOException("\n" + new String(buf, UTF_8))); //$NON-NLS-1$
  476. }
  477. }
  478. return err;
  479. }
  480. IOException maxAttempts(String action, String key) {
  481. return new IOException(MessageFormat.format(
  482. JGitText.get().amazonS3ActionFailedGivingUp, action, key,
  483. Integer.valueOf(maxAttempts)));
  484. }
  485. private HttpURLConnection open(final String method, final String bucket,
  486. final String key) throws IOException {
  487. final Map<String, String> noArgs = Collections.emptyMap();
  488. return open(method, bucket, key, noArgs);
  489. }
  490. HttpURLConnection open(final String method, final String bucket,
  491. final String key, final Map<String, String> args)
  492. throws IOException {
  493. final StringBuilder urlstr = new StringBuilder();
  494. urlstr.append("http://"); //$NON-NLS-1$
  495. urlstr.append(bucket);
  496. urlstr.append('.');
  497. urlstr.append(domain);
  498. urlstr.append('/');
  499. if (key.length() > 0)
  500. HttpSupport.encode(urlstr, key);
  501. if (!args.isEmpty()) {
  502. final Iterator<Map.Entry<String, String>> i;
  503. urlstr.append('?');
  504. i = args.entrySet().iterator();
  505. while (i.hasNext()) {
  506. final Map.Entry<String, String> e = i.next();
  507. urlstr.append(e.getKey());
  508. urlstr.append('=');
  509. HttpSupport.encode(urlstr, e.getValue());
  510. if (i.hasNext())
  511. urlstr.append('&');
  512. }
  513. }
  514. final URL url = new URL(urlstr.toString());
  515. final Proxy proxy = HttpSupport.proxyFor(proxySelector, url);
  516. final HttpURLConnection c;
  517. c = (HttpURLConnection) url.openConnection(proxy);
  518. c.setRequestMethod(method);
  519. c.setRequestProperty("User-Agent", "jgit/1.0"); //$NON-NLS-1$ //$NON-NLS-2$
  520. c.setRequestProperty("Date", httpNow()); //$NON-NLS-1$
  521. return c;
  522. }
  523. void authorize(HttpURLConnection c) throws IOException {
  524. final Map<String, List<String>> reqHdr = c.getRequestProperties();
  525. final SortedMap<String, String> sigHdr = new TreeMap<>();
  526. for (Map.Entry<String, List<String>> entry : reqHdr.entrySet()) {
  527. final String hdr = entry.getKey();
  528. if (isSignedHeader(hdr))
  529. sigHdr.put(StringUtils.toLowerCase(hdr), toCleanString(entry.getValue()));
  530. }
  531. final StringBuilder s = new StringBuilder();
  532. s.append(c.getRequestMethod());
  533. s.append('\n');
  534. s.append(remove(sigHdr, "content-md5")); //$NON-NLS-1$
  535. s.append('\n');
  536. s.append(remove(sigHdr, "content-type")); //$NON-NLS-1$
  537. s.append('\n');
  538. s.append(remove(sigHdr, "date")); //$NON-NLS-1$
  539. s.append('\n');
  540. for (Map.Entry<String, String> e : sigHdr.entrySet()) {
  541. s.append(e.getKey());
  542. s.append(':');
  543. s.append(e.getValue());
  544. s.append('\n');
  545. }
  546. final String host = c.getURL().getHost();
  547. s.append('/');
  548. s.append(host.substring(0, host.length() - domain.length() - 1));
  549. s.append(c.getURL().getPath());
  550. final String sec;
  551. try {
  552. final Mac m = Mac.getInstance(HMAC);
  553. m.init(privateKey);
  554. sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes(UTF_8)));
  555. } catch (NoSuchAlgorithmException e) {
  556. throw new IOException(MessageFormat.format(JGitText.get().noHMACsupport, HMAC, e.getMessage()));
  557. } catch (InvalidKeyException e) {
  558. throw new IOException(MessageFormat.format(JGitText.get().invalidKey, e.getMessage()));
  559. }
  560. c.setRequestProperty("Authorization", "AWS " + publicKey + ":" + sec); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
  561. }
  562. static Properties properties(File authFile)
  563. throws FileNotFoundException, IOException {
  564. final Properties p = new Properties();
  565. try (FileInputStream in = new FileInputStream(authFile)) {
  566. p.load(in);
  567. }
  568. return p;
  569. }
  570. /**
  571. * KeyInfo enables sorting of keys by lastModified time
  572. */
  573. private static final class KeyInfo {
  574. private final String name;
  575. private final long lastModifiedSecs;
  576. public KeyInfo(String aname, long lsecs) {
  577. name = aname;
  578. lastModifiedSecs = lsecs;
  579. }
  580. public String getName() {
  581. return name;
  582. }
  583. public long getLastModifiedSecs() {
  584. return lastModifiedSecs;
  585. }
  586. }
  587. private final class ListParser extends DefaultHandler {
  588. final List<KeyInfo> entries = new ArrayList<>();
  589. private final String bucket;
  590. private final String prefix;
  591. boolean truncated;
  592. private StringBuilder data;
  593. private String keyName;
  594. private Instant keyLastModified;
  595. ListParser(String bn, String p) {
  596. bucket = bn;
  597. prefix = p;
  598. }
  599. void list() throws IOException {
  600. final Map<String, String> args = new TreeMap<>();
  601. if (prefix.length() > 0)
  602. args.put("prefix", prefix); //$NON-NLS-1$
  603. if (!entries.isEmpty())
  604. args.put("marker", prefix + entries.get(entries.size() - 1).getName()); //$NON-NLS-1$
  605. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  606. final HttpURLConnection c = open("GET", bucket, "", args); //$NON-NLS-1$ //$NON-NLS-2$
  607. authorize(c);
  608. switch (HttpSupport.response(c)) {
  609. case HttpURLConnection.HTTP_OK:
  610. truncated = false;
  611. data = null;
  612. keyName = null;
  613. keyLastModified = null;
  614. final XMLReader xr;
  615. try {
  616. xr = XMLReaderFactory.createXMLReader();
  617. } catch (SAXException e) {
  618. throw new IOException(
  619. JGitText.get().noXMLParserAvailable, e);
  620. }
  621. xr.setContentHandler(this);
  622. try (InputStream in = c.getInputStream()) {
  623. xr.parse(new InputSource(in));
  624. } catch (SAXException parsingError) {
  625. throw new IOException(
  626. MessageFormat.format(
  627. JGitText.get().errorListing, prefix),
  628. parsingError);
  629. }
  630. return;
  631. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  632. continue;
  633. default:
  634. throw AmazonS3.this.error("Listing", prefix, c); //$NON-NLS-1$
  635. }
  636. }
  637. throw maxAttempts("Listing", prefix); //$NON-NLS-1$
  638. }
  639. @Override
  640. public void startElement(final String uri, final String name,
  641. final String qName, final Attributes attributes)
  642. throws SAXException {
  643. if ("Key".equals(name) || "IsTruncated".equals(name) || "LastModified".equals(name)) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
  644. data = new StringBuilder();
  645. }
  646. if ("Contents".equals(name)) { //$NON-NLS-1$
  647. keyName = null;
  648. keyLastModified = null;
  649. }
  650. }
  651. @Override
  652. public void ignorableWhitespace(final char[] ch, final int s,
  653. final int n) throws SAXException {
  654. if (data != null)
  655. data.append(ch, s, n);
  656. }
  657. @Override
  658. public void characters(char[] ch, int s, int n)
  659. throws SAXException {
  660. if (data != null)
  661. data.append(ch, s, n);
  662. }
  663. @Override
  664. public void endElement(final String uri, final String name,
  665. final String qName) throws SAXException {
  666. if ("Key".equals(name)) { //$NON-NLS-1$
  667. keyName = data.toString().substring(prefix.length());
  668. } else if ("IsTruncated".equals(name)) { //$NON-NLS-1$
  669. truncated = StringUtils.equalsIgnoreCase("true", data.toString()); //$NON-NLS-1$
  670. } else if ("LastModified".equals(name)) { //$NON-NLS-1$
  671. keyLastModified = Instant.parse(data.toString());
  672. } else if ("Contents".equals(name)) { //$NON-NLS-1$
  673. entries.add(new KeyInfo(keyName, keyLastModified.getEpochSecond()));
  674. }
  675. data = null;
  676. }
  677. }
  678. }