jgit/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportAmazonS3.java

/*
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Distribution License v. 1.0 which is available at
 * https://www.eclipse.org/org/documents/edl-v10.php.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

package org.eclipse.jgit.transport;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLConnection;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;

import org.eclipse.jgit.errors.NotSupportedException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Ref.Storage;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.SymbolicRef;

/**
 * Transport over the non-Git aware Amazon S3 protocol.
 * <p>
 * This transport communicates with the Amazon S3 servers (a non-free commercial
 * hosting service that users must subscribe to). Some users may find transport
 * to and from S3 to be a useful backup service.
 * <p>
 * The transport does not require any specialized Git support on the remote
 * (server side) repository, as Amazon does not provide any such support.
 * Repository files are retrieved directly through the S3 API, which uses
 * extended HTTP/1.1 semantics. This make it possible to read or write Git data
 * from a remote repository that is stored on S3.
 * <p>
 * Unlike the HTTP variant (see
 * {@link org.eclipse.jgit.transport.TransportHttp}) we rely upon being able to
 * list objects in a bucket, as the S3 API supports this function. By listing
 * the bucket contents we can avoid relying on <code>objects/info/packs</code>
 * or <code>info/refs</code> in the remote repository.
 * <p>
 * Concurrent pushing over this transport is not supported. Multiple concurrent
 * push operations may cause confusion in the repository state.
 *
 * @see WalkFetchConnection
 * @see WalkPushConnection
 */
public class TransportAmazonS3 extends HttpTransport implements WalkTransport {
	static final String S3_SCHEME = "amazon-s3"; //$NON-NLS-1$

	static final TransportProtocol PROTO_S3 = new TransportProtocol() {
		@Override
		public String getName() {
			return "Amazon S3"; //$NON-NLS-1$
		}

		@Override
		public Set<String> getSchemes() {
			return Collections.singleton(S3_SCHEME);
		}

		@Override
		public Set<URIishField> getRequiredFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
					URIishField.HOST, URIishField.PATH));
		}

		@Override
		public Set<URIishField> getOptionalFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.PASS));
		}

		@Override
		public Transport open(URIish uri, Repository local, String remoteName)
				throws NotSupportedException {
			return new TransportAmazonS3(local, uri);
		}
	};

	/** User information necessary to connect to S3. */
	final AmazonS3 s3;

	/** Bucket the remote repository is stored in. */
	final String bucket;

	/**
	 * Key prefix which all objects related to the repository start with.
	 * <p>
	 * The prefix does not start with "/".
	 * <p>
	 * The prefix does not end with "/". The trailing slash is stripped during
	 * the constructor if a trailing slash was supplied in the URIish.
	 * <p>
	 * All files within the remote repository start with
	 * <code>keyPrefix + "/"</code>.
	 */
	private final String keyPrefix;

	TransportAmazonS3(final Repository local, final URIish uri)
			throws NotSupportedException {
		super(local, uri);

		Properties props = loadProperties();
		File directory = local.getDirectory();
		if (!props.containsKey("tmpdir") && directory != null) //$NON-NLS-1$
			props.put("tmpdir", directory.getPath()); //$NON-NLS-1$

		s3 = new AmazonS3(props);
		bucket = uri.getHost();

		String p = uri.getPath();
		if (p.startsWith("/")) //$NON-NLS-1$
			p = p.substring(1);
		if (p.endsWith("/")) //$NON-NLS-1$
			p = p.substring(0, p.length() - 1);
		keyPrefix = p;
	}

	private Properties loadProperties() throws NotSupportedException {
		if (local.getDirectory() != null) {
			File propsFile = new File(local.getDirectory(), uri.getUser());
			if (propsFile.isFile())
				return loadPropertiesFile(propsFile);
		}

		File propsFile = new File(local.getFS().userHome(), uri.getUser());
		if (propsFile.isFile())
			return loadPropertiesFile(propsFile);

		Properties props = new Properties();
		String user = uri.getUser();
		String pass = uri.getPass();
		if (user != null && pass != null) {
		        props.setProperty("accesskey", user); //$NON-NLS-1$
		        props.setProperty("secretkey", pass); //$NON-NLS-1$
		} else
			throw new NotSupportedException(MessageFormat.format(
					JGitText.get().cannotReadFile, propsFile));
		return props;
	}

	private static Properties loadPropertiesFile(File propsFile)
			throws NotSupportedException {
		try {
			return AmazonS3.properties(propsFile);
		} catch (IOException e) {
			throw new NotSupportedException(MessageFormat.format(
					JGitText.get().cannotReadFile, propsFile), e);
		}
	}

	/** {@inheritDoc} */
	@Override
	public FetchConnection openFetch() throws TransportException {
		final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects"); //$NON-NLS-1$
		final WalkFetchConnection r = new WalkFetchConnection(this, c);
		r.available(c.readAdvertisedRefs());
		return r;
	}

	/** {@inheritDoc} */
	@Override
	public PushConnection openPush() throws TransportException {
		final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects"); //$NON-NLS-1$
		final WalkPushConnection r = new WalkPushConnection(this, c);
		r.available(c.readAdvertisedRefs());
		return r;
	}

	/** {@inheritDoc} */
	@Override
	public void close() {
		// No explicit connections are maintained.
	}

	class DatabaseS3 extends WalkRemoteObjectDatabase {
		private final String bucketName;

		private final String objectsKey;

		DatabaseS3(final String b, final String o) {
			bucketName = b;
			objectsKey = o;
		}

		private String resolveKey(String subpath) {
			if (subpath.endsWith("/")) //$NON-NLS-1$
				subpath = subpath.substring(0, subpath.length() - 1);
			String k = objectsKey;
			while (subpath.startsWith(ROOT_DIR)) {
				k = k.substring(0, k.lastIndexOf('/'));
				subpath = subpath.substring(3);
			}
			return k + "/" + subpath; //$NON-NLS-1$
		}

		@Override
		URIish getURI() {
			URIish u = new URIish();
			u = u.setScheme(S3_SCHEME);
			u = u.setHost(bucketName);
			u = u.setPath("/" + objectsKey); //$NON-NLS-1$
			return u;
		}

		@Override
		Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
			try {
				return readAlternates(Constants.INFO_ALTERNATES);
			} catch (FileNotFoundException err) {
				// Fall through.
			}
			return null;
		}

		@Override
		WalkRemoteObjectDatabase openAlternate(String location)
				throws IOException {
			return new DatabaseS3(bucketName, resolveKey(location));
		}

		@Override
		Collection<String> getPackNames() throws IOException {
			// s3.list returns most recently modified packs first.
			// These are the packs most likely to contain missing refs.
			final List<String> packList = s3.list(bucket, resolveKey("pack")); //$NON-NLS-1$
			final HashSet<String> have = new HashSet<>();
			have.addAll(packList);

			final Collection<String> packs = new ArrayList<>();
			for (String n : packList) {
				if (!n.startsWith("pack-") || !n.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$
					continue;

				final String in = n.substring(0, n.length() - 5) + ".idx"; //$NON-NLS-1$
				if (have.contains(in))
					packs.add(n);
			}
			return packs;
		}

		@Override
		FileStream open(String path) throws IOException {
			final URLConnection c = s3.get(bucket, resolveKey(path));
			final InputStream raw = c.getInputStream();
			final InputStream in = s3.decrypt(c);
			final int len = c.getContentLength();
			return new FileStream(in, raw == in ? len : -1);
		}

		@Override
		void deleteFile(String path) throws IOException {
			s3.delete(bucket, resolveKey(path));
		}

		@Override
		OutputStream writeFile(final String path,
				final ProgressMonitor monitor, final String monitorTask)
				throws IOException {
			return s3.beginPut(bucket, resolveKey(path), monitor, monitorTask);
		}

		@Override
		void writeFile(String path, byte[] data) throws IOException {
			s3.put(bucket, resolveKey(path), data);
		}

		Map<String, Ref> readAdvertisedRefs() throws TransportException {
			final TreeMap<String, Ref> avail = new TreeMap<>();
			readPackedRefs(avail);
			readLooseRefs(avail);
			readRef(avail, Constants.HEAD);
			return avail;
		}

		private void readLooseRefs(TreeMap<String, Ref> avail)
				throws TransportException {
			try {
				for (final String n : s3.list(bucket, resolveKey(ROOT_DIR
						+ "refs"))) //$NON-NLS-1$
					readRef(avail, "refs/" + n); //$NON-NLS-1$
			} catch (IOException e) {
				throw new TransportException(getURI(), JGitText.get().cannotListRefs, e);
			}
		}

		private Ref readRef(TreeMap<String, Ref> avail, String rn)
				throws TransportException {
			final String s;
			String ref = ROOT_DIR + rn;
			try {
				try (BufferedReader br = openReader(ref)) {
					s = br.readLine();
				}
			} catch (FileNotFoundException noRef) {
				return null;
			} catch (IOException err) {
				throw new TransportException(getURI(), MessageFormat.format(
						JGitText.get().transportExceptionReadRef, ref), err);
			}

			if (s == null)
				throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionEmptyRef, rn));

			if (s.startsWith("ref: ")) { //$NON-NLS-1$
				final String target = s.substring("ref: ".length()); //$NON-NLS-1$
				Ref r = avail.get(target);
				if (r == null)
					r = readRef(avail, target);
				if (r == null)
					r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
				r = new SymbolicRef(rn, r);
				avail.put(r.getName(), r);
				return r;
			}

			if (ObjectId.isId(s)) {
				final Ref r = new ObjectIdRef.Unpeeled(loose(avail.get(rn)),
						rn, ObjectId.fromString(s));
				avail.put(r.getName(), r);
				return r;
			}

			throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionBadRef, rn, s));
		}

		private Storage loose(Ref r) {
			if (r != null && r.getStorage() == Storage.PACKED)
				return Storage.LOOSE_PACKED;
			return Storage.LOOSE;
		}

		@Override
		void close() {
			// We do not maintain persistent connections.
		}
	}
}