Du kannst nicht mehr als 25 Themen auswählen Themen müssen mit entweder einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

AmazonS3.java 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734
  1. /*
  2. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  3. * and other copyright owners as documented in the project's IP log.
  4. *
  5. * This program and the accompanying materials are made available
  6. * under the terms of the Eclipse Distribution License v1.0 which
  7. * accompanies this distribution, is reproduced below, and is
  8. * available at http://www.eclipse.org/org/documents/edl-v10.php
  9. *
  10. * All rights reserved.
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above copyright
  17. * notice, this list of conditions and the following disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials provided
  22. * with the distribution.
  23. *
  24. * - Neither the name of the Eclipse Foundation, Inc. nor the
  25. * names of its contributors may be used to endorse or promote
  26. * products derived from this software without specific prior
  27. * written permission.
  28. *
  29. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42. */
  43. package org.eclipse.jgit.transport;
  44. import java.io.ByteArrayOutputStream;
  45. import java.io.File;
  46. import java.io.FileInputStream;
  47. import java.io.FileNotFoundException;
  48. import java.io.IOException;
  49. import java.io.InputStream;
  50. import java.io.OutputStream;
  51. import java.net.HttpURLConnection;
  52. import java.net.Proxy;
  53. import java.net.ProxySelector;
  54. import java.net.URL;
  55. import java.net.URLConnection;
  56. import java.security.DigestOutputStream;
  57. import java.security.InvalidKeyException;
  58. import java.security.MessageDigest;
  59. import java.security.NoSuchAlgorithmException;
  60. import java.security.spec.InvalidKeySpecException;
  61. import java.text.MessageFormat;
  62. import java.text.SimpleDateFormat;
  63. import java.util.ArrayList;
  64. import java.util.Collections;
  65. import java.util.Date;
  66. import java.util.HashSet;
  67. import java.util.Iterator;
  68. import java.util.List;
  69. import java.util.Locale;
  70. import java.util.Map;
  71. import java.util.Properties;
  72. import java.util.Set;
  73. import java.util.SortedMap;
  74. import java.util.TimeZone;
  75. import java.util.TreeMap;
  76. import javax.crypto.Mac;
  77. import javax.crypto.spec.SecretKeySpec;
  78. import org.eclipse.jgit.JGitText;
  79. import org.eclipse.jgit.lib.Constants;
  80. import org.eclipse.jgit.lib.NullProgressMonitor;
  81. import org.eclipse.jgit.lib.ProgressMonitor;
  82. import org.eclipse.jgit.util.Base64;
  83. import org.eclipse.jgit.util.HttpSupport;
  84. import org.eclipse.jgit.util.StringUtils;
  85. import org.eclipse.jgit.util.TemporaryBuffer;
  86. import org.xml.sax.Attributes;
  87. import org.xml.sax.InputSource;
  88. import org.xml.sax.SAXException;
  89. import org.xml.sax.XMLReader;
  90. import org.xml.sax.helpers.DefaultHandler;
  91. import org.xml.sax.helpers.XMLReaderFactory;
  92. /**
  93. * A simple HTTP REST client for the Amazon S3 service.
  94. * <p>
  95. * This client uses the REST API to communicate with the Amazon S3 servers and
  96. * read or write content through a bucket that the user has access to. It is a
  97. * very lightweight implementation of the S3 API and therefore does not have all
  98. * of the bells and whistles of popular client implementations.
  99. * <p>
  100. * Authentication is always performed using the user's AWSAccessKeyId and their
  101. * private AWSSecretAccessKey.
  102. * <p>
  103. * Optional client-side encryption may be enabled if requested. The format is
  104. * compatible with <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a>,
  105. * a popular Java based Amazon S3 client library. Enabling encryption can hide
  106. * sensitive data from the operators of the S3 service.
  107. */
  108. public class AmazonS3 {
  109. private static final Set<String> SIGNED_HEADERS;
  110. private static final String HMAC = "HmacSHA1";
  111. private static final String DOMAIN = "s3.amazonaws.com";
  112. private static final String X_AMZ_ACL = "x-amz-acl";
  113. private static final String X_AMZ_META = "x-amz-meta-";
  114. static {
  115. SIGNED_HEADERS = new HashSet<String>();
  116. SIGNED_HEADERS.add("content-type");
  117. SIGNED_HEADERS.add("content-md5");
  118. SIGNED_HEADERS.add("date");
  119. }
  120. private static boolean isSignedHeader(final String name) {
  121. final String nameLC = StringUtils.toLowerCase(name);
  122. return SIGNED_HEADERS.contains(nameLC) || nameLC.startsWith("x-amz-");
  123. }
  124. private static String toCleanString(final List<String> list) {
  125. final StringBuilder s = new StringBuilder();
  126. for (final String v : list) {
  127. if (s.length() > 0)
  128. s.append(',');
  129. s.append(v.replaceAll("\n", "").trim());
  130. }
  131. return s.toString();
  132. }
  133. private static String remove(final Map<String, String> m, final String k) {
  134. final String r = m.remove(k);
  135. return r != null ? r : "";
  136. }
  137. private static String httpNow() {
  138. final String tz = "GMT";
  139. final SimpleDateFormat fmt;
  140. fmt = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss", Locale.US);
  141. fmt.setTimeZone(TimeZone.getTimeZone(tz));
  142. return fmt.format(new Date()) + " " + tz;
  143. }
  144. private static MessageDigest newMD5() {
  145. try {
  146. return MessageDigest.getInstance("MD5");
  147. } catch (NoSuchAlgorithmException e) {
  148. throw new RuntimeException(JGitText.get().JRELacksMD5Implementation, e);
  149. }
  150. }
  151. /** AWSAccessKeyId, public string that identifies the user's account. */
  152. private final String publicKey;
  153. /** Decoded form of the private AWSSecretAccessKey, to sign requests. */
  154. private final SecretKeySpec privateKey;
  155. /** Our HTTP proxy support, in case we are behind a firewall. */
  156. private final ProxySelector proxySelector;
  157. /** ACL to apply to created objects. */
  158. private final String acl;
  159. /** Maximum number of times to try an operation. */
  160. private final int maxAttempts;
  161. /** Encryption algorithm, may be a null instance that provides pass-through. */
  162. private final WalkEncryption encryption;
  163. /**
  164. * Create a new S3 client for the supplied user information.
  165. * <p>
  166. * The connection properties are a subset of those supported by the popular
  167. * <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a> library.
  168. * For example:
  169. *
  170. * <pre>
  171. * # AWS Access and Secret Keys (required)
  172. * accesskey: &lt;YourAWSAccessKey&gt;
  173. * secretkey: &lt;YourAWSSecretKey&gt;
  174. *
  175. * # Access Control List setting to apply to uploads, must be one of:
  176. * # PRIVATE, PUBLIC_READ (defaults to PRIVATE).
  177. * acl: PRIVATE
  178. *
  179. * # Number of times to retry after internal error from S3.
  180. * httpclient.retry-max: 3
  181. *
  182. * # End-to-end encryption (hides content from S3 owners)
  183. * password: &lt;encryption pass-phrase&gt;
  184. * crypto.algorithm: PBEWithMD5AndDES
  185. * </pre>
  186. *
  187. * @param props
  188. * connection properties.
  189. *
  190. */
  191. public AmazonS3(final Properties props) {
  192. publicKey = props.getProperty("accesskey");
  193. if (publicKey == null)
  194. throw new IllegalArgumentException(JGitText.get().missingAccesskey);
  195. final String secret = props.getProperty("secretkey");
  196. if (secret == null)
  197. throw new IllegalArgumentException(JGitText.get().missingSecretkey);
  198. privateKey = new SecretKeySpec(Constants.encodeASCII(secret), HMAC);
  199. final String pacl = props.getProperty("acl", "PRIVATE");
  200. if (StringUtils.equalsIgnoreCase("PRIVATE", pacl))
  201. acl = "private";
  202. else if (StringUtils.equalsIgnoreCase("PUBLIC", pacl))
  203. acl = "public-read";
  204. else if (StringUtils.equalsIgnoreCase("PUBLIC-READ", pacl))
  205. acl = "public-read";
  206. else if (StringUtils.equalsIgnoreCase("PUBLIC_READ", pacl))
  207. acl = "public-read";
  208. else
  209. throw new IllegalArgumentException("Invalid acl: " + pacl);
  210. try {
  211. final String cPas = props.getProperty("password");
  212. if (cPas != null) {
  213. String cAlg = props.getProperty("crypto.algorithm");
  214. if (cAlg == null)
  215. cAlg = "PBEWithMD5AndDES";
  216. encryption = new WalkEncryption.ObjectEncryptionV2(cAlg, cPas);
  217. } else {
  218. encryption = WalkEncryption.NONE;
  219. }
  220. } catch (InvalidKeySpecException e) {
  221. throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
  222. } catch (NoSuchAlgorithmException e) {
  223. throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
  224. }
  225. maxAttempts = Integer.parseInt(props.getProperty(
  226. "httpclient.retry-max", "3"));
  227. proxySelector = ProxySelector.getDefault();
  228. }
  229. /**
  230. * Get the content of a bucket object.
  231. *
  232. * @param bucket
  233. * name of the bucket storing the object.
  234. * @param key
  235. * key of the object within its bucket.
  236. * @return connection to stream the content of the object. The request
  237. * properties of the connection may not be modified by the caller as
  238. * the request parameters have already been signed.
  239. * @throws IOException
  240. * sending the request was not possible.
  241. */
  242. public URLConnection get(final String bucket, final String key)
  243. throws IOException {
  244. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  245. final HttpURLConnection c = open("GET", bucket, key);
  246. authorize(c);
  247. switch (HttpSupport.response(c)) {
  248. case HttpURLConnection.HTTP_OK:
  249. encryption.validate(c, X_AMZ_META);
  250. return c;
  251. case HttpURLConnection.HTTP_NOT_FOUND:
  252. throw new FileNotFoundException(key);
  253. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  254. continue;
  255. default:
  256. throw error("Reading", key, c);
  257. }
  258. }
  259. throw maxAttempts("Reading", key);
  260. }
  261. /**
  262. * Decrypt an input stream from {@link #get(String, String)}.
  263. *
  264. * @param u
  265. * connection previously created by {@link #get(String, String)}}.
  266. * @return stream to read plain text from.
  267. * @throws IOException
  268. * decryption could not be configured.
  269. */
  270. public InputStream decrypt(final URLConnection u) throws IOException {
  271. return encryption.decrypt(u.getInputStream());
  272. }
  273. /**
  274. * List the names of keys available within a bucket.
  275. * <p>
  276. * This method is primarily meant for obtaining a "recursive directory
  277. * listing" rooted under the specified bucket and prefix location.
  278. *
  279. * @param bucket
  280. * name of the bucket whose objects should be listed.
  281. * @param prefix
  282. * common prefix to filter the results by. Must not be null.
  283. * Supplying the empty string will list all keys in the bucket.
  284. * Supplying a non-empty string will act as though a trailing '/'
  285. * appears in prefix, even if it does not.
  286. * @return list of keys starting with <code>prefix</code>, after removing
  287. * <code>prefix</code> (or <code>prefix + "/"</code>)from all
  288. * of them.
  289. * @throws IOException
  290. * sending the request was not possible, or the response XML
  291. * document could not be parsed properly.
  292. */
  293. public List<String> list(final String bucket, String prefix)
  294. throws IOException {
  295. if (prefix.length() > 0 && !prefix.endsWith("/"))
  296. prefix += "/";
  297. final ListParser lp = new ListParser(bucket, prefix);
  298. do {
  299. lp.list();
  300. } while (lp.truncated);
  301. return lp.entries;
  302. }
  303. /**
  304. * Delete a single object.
  305. * <p>
  306. * Deletion always succeeds, even if the object does not exist.
  307. *
  308. * @param bucket
  309. * name of the bucket storing the object.
  310. * @param key
  311. * key of the object within its bucket.
  312. * @throws IOException
  313. * deletion failed due to communications error.
  314. */
  315. public void delete(final String bucket, final String key)
  316. throws IOException {
  317. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  318. final HttpURLConnection c = open("DELETE", bucket, key);
  319. authorize(c);
  320. switch (HttpSupport.response(c)) {
  321. case HttpURLConnection.HTTP_NO_CONTENT:
  322. return;
  323. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  324. continue;
  325. default:
  326. throw error("Deletion", key, c);
  327. }
  328. }
  329. throw maxAttempts("Deletion", key);
  330. }
  331. /**
  332. * Atomically create or replace a single small object.
  333. * <p>
  334. * This form is only suitable for smaller contents, where the caller can
  335. * reasonable fit the entire thing into memory.
  336. * <p>
  337. * End-to-end data integrity is assured by internally computing the MD5
  338. * checksum of the supplied data and transmitting the checksum along with
  339. * the data itself.
  340. *
  341. * @param bucket
  342. * name of the bucket storing the object.
  343. * @param key
  344. * key of the object within its bucket.
  345. * @param data
  346. * new data content for the object. Must not be null. Zero length
  347. * array will create a zero length object.
  348. * @throws IOException
  349. * creation/updating failed due to communications error.
  350. */
  351. public void put(final String bucket, final String key, final byte[] data)
  352. throws IOException {
  353. if (encryption != WalkEncryption.NONE) {
  354. // We have to copy to produce the cipher text anyway so use
  355. // the large object code path as it supports that behavior.
  356. //
  357. final OutputStream os = beginPut(bucket, key, null, null);
  358. os.write(data);
  359. os.close();
  360. return;
  361. }
  362. final String md5str = Base64.encodeBytes(newMD5().digest(data));
  363. final String lenstr = String.valueOf(data.length);
  364. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  365. final HttpURLConnection c = open("PUT", bucket, key);
  366. c.setRequestProperty("Content-Length", lenstr);
  367. c.setRequestProperty("Content-MD5", md5str);
  368. c.setRequestProperty(X_AMZ_ACL, acl);
  369. authorize(c);
  370. c.setDoOutput(true);
  371. c.setFixedLengthStreamingMode(data.length);
  372. final OutputStream os = c.getOutputStream();
  373. try {
  374. os.write(data);
  375. } finally {
  376. os.close();
  377. }
  378. switch (HttpSupport.response(c)) {
  379. case HttpURLConnection.HTTP_OK:
  380. return;
  381. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  382. continue;
  383. default:
  384. throw error("Writing", key, c);
  385. }
  386. }
  387. throw maxAttempts("Writing", key);
  388. }
  389. /**
  390. * Atomically create or replace a single large object.
  391. * <p>
  392. * Initially the returned output stream buffers data into memory, but if the
  393. * total number of written bytes starts to exceed an internal limit the data
  394. * is spooled to a temporary file on the local drive.
  395. * <p>
  396. * Network transmission is attempted only when <code>close()</code> gets
  397. * called at the end of output. Closing the returned stream can therefore
  398. * take significant time, especially if the written content is very large.
  399. * <p>
  400. * End-to-end data integrity is assured by internally computing the MD5
  401. * checksum of the supplied data and transmitting the checksum along with
  402. * the data itself.
  403. *
  404. * @param bucket
  405. * name of the bucket storing the object.
  406. * @param key
  407. * key of the object within its bucket.
  408. * @param monitor
  409. * (optional) progress monitor to post upload completion to
  410. * during the stream's close method.
  411. * @param monitorTask
  412. * (optional) task name to display during the close method.
  413. * @return a stream which accepts the new data, and transmits once closed.
  414. * @throws IOException
  415. * if encryption was enabled it could not be configured.
  416. */
  417. public OutputStream beginPut(final String bucket, final String key,
  418. final ProgressMonitor monitor, final String monitorTask)
  419. throws IOException {
  420. final MessageDigest md5 = newMD5();
  421. final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile() {
  422. @Override
  423. public void close() throws IOException {
  424. super.close();
  425. try {
  426. putImpl(bucket, key, md5.digest(), this, monitor,
  427. monitorTask);
  428. } finally {
  429. destroy();
  430. }
  431. }
  432. };
  433. return encryption.encrypt(new DigestOutputStream(buffer, md5));
  434. }
  435. private void putImpl(final String bucket, final String key,
  436. final byte[] csum, final TemporaryBuffer buf,
  437. ProgressMonitor monitor, String monitorTask) throws IOException {
  438. if (monitor == null)
  439. monitor = NullProgressMonitor.INSTANCE;
  440. if (monitorTask == null)
  441. monitorTask = MessageFormat.format(JGitText.get().progressMonUploading, key);
  442. final String md5str = Base64.encodeBytes(csum);
  443. final long len = buf.length();
  444. final String lenstr = String.valueOf(len);
  445. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  446. final HttpURLConnection c = open("PUT", bucket, key);
  447. c.setRequestProperty("Content-Length", lenstr);
  448. c.setRequestProperty("Content-MD5", md5str);
  449. c.setRequestProperty(X_AMZ_ACL, acl);
  450. encryption.request(c, X_AMZ_META);
  451. authorize(c);
  452. c.setDoOutput(true);
  453. c.setFixedLengthStreamingMode((int) len);
  454. monitor.beginTask(monitorTask, (int) (len / 1024));
  455. final OutputStream os = c.getOutputStream();
  456. try {
  457. buf.writeTo(os, monitor);
  458. } finally {
  459. monitor.endTask();
  460. os.close();
  461. }
  462. switch (HttpSupport.response(c)) {
  463. case HttpURLConnection.HTTP_OK:
  464. return;
  465. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  466. continue;
  467. default:
  468. throw error("Writing", key, c);
  469. }
  470. }
  471. throw maxAttempts("Writing", key);
  472. }
  473. private IOException error(final String action, final String key,
  474. final HttpURLConnection c) throws IOException {
  475. final IOException err = new IOException(MessageFormat.format(JGitText.get().amazonS3ActionFailed
  476. , action, key, HttpSupport.response(c), c.getResponseMessage()));
  477. final InputStream errorStream = c.getErrorStream();
  478. if (errorStream == null)
  479. return err;
  480. final ByteArrayOutputStream b = new ByteArrayOutputStream();
  481. byte[] buf = new byte[2048];
  482. for (;;) {
  483. final int n = errorStream.read(buf);
  484. if (n < 0)
  485. break;
  486. if (n > 0)
  487. b.write(buf, 0, n);
  488. }
  489. buf = b.toByteArray();
  490. if (buf.length > 0)
  491. err.initCause(new IOException("\n" + new String(buf)));
  492. return err;
  493. }
  494. private IOException maxAttempts(final String action, final String key) {
  495. return new IOException(MessageFormat.format(JGitText.get().amazonS3ActionFailedGivingUp
  496. , action, key, maxAttempts));
  497. }
  498. private HttpURLConnection open(final String method, final String bucket,
  499. final String key) throws IOException {
  500. final Map<String, String> noArgs = Collections.emptyMap();
  501. return open(method, bucket, key, noArgs);
  502. }
  503. private HttpURLConnection open(final String method, final String bucket,
  504. final String key, final Map<String, String> args)
  505. throws IOException {
  506. final StringBuilder urlstr = new StringBuilder();
  507. urlstr.append("http://");
  508. urlstr.append(bucket);
  509. urlstr.append('.');
  510. urlstr.append(DOMAIN);
  511. urlstr.append('/');
  512. if (key.length() > 0)
  513. HttpSupport.encode(urlstr, key);
  514. if (!args.isEmpty()) {
  515. final Iterator<Map.Entry<String, String>> i;
  516. urlstr.append('?');
  517. i = args.entrySet().iterator();
  518. while (i.hasNext()) {
  519. final Map.Entry<String, String> e = i.next();
  520. urlstr.append(e.getKey());
  521. urlstr.append('=');
  522. HttpSupport.encode(urlstr, e.getValue());
  523. if (i.hasNext())
  524. urlstr.append('&');
  525. }
  526. }
  527. final URL url = new URL(urlstr.toString());
  528. final Proxy proxy = HttpSupport.proxyFor(proxySelector, url);
  529. final HttpURLConnection c;
  530. c = (HttpURLConnection) url.openConnection(proxy);
  531. c.setRequestMethod(method);
  532. c.setRequestProperty("User-Agent", "jgit/1.0");
  533. c.setRequestProperty("Date", httpNow());
  534. return c;
  535. }
  536. private void authorize(final HttpURLConnection c) throws IOException {
  537. final Map<String, List<String>> reqHdr = c.getRequestProperties();
  538. final SortedMap<String, String> sigHdr = new TreeMap<String, String>();
  539. for (final Map.Entry<String, List<String>> entry : reqHdr.entrySet()) {
  540. final String hdr = entry.getKey();
  541. if (isSignedHeader(hdr))
  542. sigHdr.put(StringUtils.toLowerCase(hdr), toCleanString(entry.getValue()));
  543. }
  544. final StringBuilder s = new StringBuilder();
  545. s.append(c.getRequestMethod());
  546. s.append('\n');
  547. s.append(remove(sigHdr, "content-md5"));
  548. s.append('\n');
  549. s.append(remove(sigHdr, "content-type"));
  550. s.append('\n');
  551. s.append(remove(sigHdr, "date"));
  552. s.append('\n');
  553. for (final Map.Entry<String, String> e : sigHdr.entrySet()) {
  554. s.append(e.getKey());
  555. s.append(':');
  556. s.append(e.getValue());
  557. s.append('\n');
  558. }
  559. final String host = c.getURL().getHost();
  560. s.append('/');
  561. s.append(host.substring(0, host.length() - DOMAIN.length() - 1));
  562. s.append(c.getURL().getPath());
  563. final String sec;
  564. try {
  565. final Mac m = Mac.getInstance(HMAC);
  566. m.init(privateKey);
  567. sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes("UTF-8")));
  568. } catch (NoSuchAlgorithmException e) {
  569. throw new IOException(MessageFormat.format(JGitText.get().noHMACsupport, HMAC, e.getMessage()));
  570. } catch (InvalidKeyException e) {
  571. throw new IOException(MessageFormat.format(JGitText.get().invalidKey, e.getMessage()));
  572. }
  573. c.setRequestProperty("Authorization", "AWS " + publicKey + ":" + sec);
  574. }
  575. static Properties properties(final File authFile)
  576. throws FileNotFoundException, IOException {
  577. final Properties p = new Properties();
  578. final FileInputStream in = new FileInputStream(authFile);
  579. try {
  580. p.load(in);
  581. } finally {
  582. in.close();
  583. }
  584. return p;
  585. }
  586. private final class ListParser extends DefaultHandler {
  587. final List<String> entries = new ArrayList<String>();
  588. private final String bucket;
  589. private final String prefix;
  590. boolean truncated;
  591. private StringBuilder data;
  592. ListParser(final String bn, final String p) {
  593. bucket = bn;
  594. prefix = p;
  595. }
  596. void list() throws IOException {
  597. final Map<String, String> args = new TreeMap<String, String>();
  598. if (prefix.length() > 0)
  599. args.put("prefix", prefix);
  600. if (!entries.isEmpty())
  601. args.put("marker", prefix + entries.get(entries.size() - 1));
  602. for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
  603. final HttpURLConnection c = open("GET", bucket, "", args);
  604. authorize(c);
  605. switch (HttpSupport.response(c)) {
  606. case HttpURLConnection.HTTP_OK:
  607. truncated = false;
  608. data = null;
  609. final XMLReader xr;
  610. try {
  611. xr = XMLReaderFactory.createXMLReader();
  612. } catch (SAXException e) {
  613. throw new IOException(JGitText.get().noXMLParserAvailable);
  614. }
  615. xr.setContentHandler(this);
  616. final InputStream in = c.getInputStream();
  617. try {
  618. xr.parse(new InputSource(in));
  619. } catch (SAXException parsingError) {
  620. final IOException p;
  621. p = new IOException(MessageFormat.format(JGitText.get().errorListing, prefix));
  622. p.initCause(parsingError);
  623. throw p;
  624. } finally {
  625. in.close();
  626. }
  627. return;
  628. case HttpURLConnection.HTTP_INTERNAL_ERROR:
  629. continue;
  630. default:
  631. throw AmazonS3.this.error("Listing", prefix, c);
  632. }
  633. }
  634. throw maxAttempts("Listing", prefix);
  635. }
  636. @Override
  637. public void startElement(final String uri, final String name,
  638. final String qName, final Attributes attributes)
  639. throws SAXException {
  640. if ("Key".equals(name) || "IsTruncated".equals(name))
  641. data = new StringBuilder();
  642. }
  643. @Override
  644. public void ignorableWhitespace(final char[] ch, final int s,
  645. final int n) throws SAXException {
  646. if (data != null)
  647. data.append(ch, s, n);
  648. }
  649. @Override
  650. public void characters(final char[] ch, final int s, final int n)
  651. throws SAXException {
  652. if (data != null)
  653. data.append(ch, s, n);
  654. }
  655. @Override
  656. public void endElement(final String uri, final String name,
  657. final String qName) throws SAXException {
  658. if ("Key".equals(name))
  659. entries.add(data.toString().substring(prefix.length()));
  660. else if ("IsTruncated".equals(name))
  661. truncated = StringUtils.equalsIgnoreCase("true", data.toString());
  662. data = null;
  663. }
  664. }
  665. }