Support Amazon S3 based storage for LFS

Add a storage implementation storing large objects in Amazon S3.
The AmazonS3Repository pre-signs download and upload requests.

AWS access and secret key are expected to be in the
$HOME/.aws/credentials file in the following format:

[default]
  accessKey = ...
  secretKey = ...

Use AWS version 4 request signing [1] because it is more secure and
supported by all regions. The version 3 signing is not supported in
newer regions.

In follow up changes we should:

- implement getVerifyAction() and do actual verification. Subclasses of
S3Repository can implement caching for object meta data (size) in order
to avoid extra roundtrips to S3. Verification should ensure that meta
data store and content of S3 storage are in sync

- HEAD request used in S3Repository.getSize() seems to always return
Content-length 0 in contrast to the documentation [2]. So getSize() does
detect if the object exists in S3 or not but in case the object exists
it always returns size 0

[1] http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
[2] https://forums.aws.amazon.com/thread.jspa?threadID=223616

Change-Id: Ic47f094928a259e5264c92b3aacf6d90210907a8
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Signed-off-by: Sasa Zivkov <sasa.zivkov@sap.com>

2 files changed, 87 insertions, 8 deletions

diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java
index 18398f664e..00bea86c41 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java
@@ -43,12 +43,15 @@
 
 package org.eclipse.jgit.pgm.debug;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.text.MessageFormat;
 
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConfiguration;
@@ -58,12 +61,18 @@ import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.lfs.server.LargeFileRepository;
 import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
 import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
 import org.eclipse.jgit.lfs.server.fs.FileLfsRepository;
+import org.eclipse.jgit.lfs.server.s3.S3Config;
+import org.eclipse.jgit.lfs.server.s3.S3Repository;
 import org.eclipse.jgit.pgm.Command;
 import org.eclipse.jgit.pgm.TextBuiltin;
+import org.eclipse.jgit.pgm.internal.CLIText;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
 import org.kohsuke.args4j.Argument;
 import org.kohsuke.args4j.Option;
 
@@ -153,7 +162,11 @@ class LfsStore extends TextBuiltin {
 	}
 
 	private static enum StoreType {
-		FS;
+		FS, S3;
+	}
+
+	private static enum StorageClass {
+		REDUCED_REDUNDANCY, STANDARD
 	}
 
 	private static final String OBJECTS = "objects/"; //$NON-NLS-1$
@@ -162,8 +175,8 @@ class LfsStore extends TextBuiltin {
 
 	private static final String PROTOCOL_PATH = "/lfs/objects/batch"; //$NON-NLS-1$
 
-	@Option(name = "--port", aliases = {"-p" }, metaVar = "metaVar_port",
-			usage = "usage_LFSPort")
+	@Option(name = "--port", aliases = {"-p" },
+			metaVar = "metaVar_port", usage = "usage_LFSPort")
 	int port;
 
 	@Option(name = "--store", metaVar = "metaVar_lfsStorage", usage = "usage_LFSRunStore")
@@ -173,6 +186,25 @@ class LfsStore extends TextBuiltin {
 			usage = "usage_LFSStoreUrl")
 	String storeUrl;
 
+	@Option(name = "--region", aliases = {"-r" },
+			metaVar = "metaVar_s3Region", usage = "usage_S3Region")
+	String region; // $NON-NLS-1$
+
+	@Option(name = "--bucket", aliases = {"-b" },
+			metaVar = "metaVar_s3Bucket", usage = "usage_S3Bucket")
+	String bucket; // $NON-NLS-1$
+
+	@Option(name = "--storage-class", aliases = {"-c" },
+			metaVar = "metaVar_s3StorageClass", usage = "usage_S3StorageClass")
+	StorageClass storageClass = StorageClass.REDUCED_REDUNDANCY;
+
+	@Option(name = "--expire", aliases = {"-e" },
+			metaVar = "metaVar_seconds", usage = "usage_S3Expiration")
+	int expirationSeconds = 600;
+
+	@Option(name = "--no-ssl-verify", usage = "usage_S3NoSslVerify")
+	boolean disableSslVerify = false;
+
 	@Argument(required = false, metaVar = "metaVar_directory", usage = "usage_LFSDirectory")
 	String directory;
 
@@ -203,9 +235,17 @@ class LfsStore extends TextBuiltin {
 			repository = fsRepo;
 			break;
 
+		case S3:
+			readAWSKeys();
+			checkOptions();
+			S3Config config = new S3Config(region.toString(), bucket,
+					storageClass.toString(), accessKey, secretKey,
+					expirationSeconds, disableSslVerify);
+			repository = new S3Repository(config);
+			break;
 		default:
-			throw new IllegalArgumentException(
-					"Unknown store type: " + storeType); //$NON-NLS-1$
+			throw new IllegalArgumentException(MessageFormat
+					.format(CLIText.get().lfsUnknownStoreType, storeType));
 		}
 
 		LfsProtocolServlet protocol = new LfsProtocolServlet() {
@@ -222,10 +262,38 @@ class LfsStore extends TextBuiltin {
 
 		server.start();
 
-		outw.println("LFS protocol URL: " + getProtocolUrl(baseURI)); //$NON-NLS-1$
+		outw.println(MessageFormat.format(CLIText.get().lfsProtocolUrl,
+				getProtocolUrl(baseURI)));
 		if (storeType == StoreType.FS) {
-			outw.println("LFS objects located in: " + directory); //$NON-NLS-1$
-			outw.println("LFS store URL: " + getStoreUrl(baseURI)); //$NON-NLS-1$
+			outw.println(MessageFormat.format(CLIText.get().lfsStoreDirectory,
+					directory));
+			outw.println(MessageFormat.format(CLIText.get().lfsStoreUrl,
+					getStoreUrl(baseURI)));
+		}
+	}
+
+	private void checkOptions() {
+		if (bucket == null || bucket.length() == 0) {
+			throw die(MessageFormat.format(CLIText.get().s3InvalidBucket,
+					bucket));
+		}
+	}
+
+	private void readAWSKeys() throws IOException, ConfigInvalidException {
+		String credentialsPath = System.getProperty("user.home") //$NON-NLS-1$
+				+ "/.aws/credentials"; //$NON-NLS-1$
+		FileBasedConfig c = new FileBasedConfig(new File(credentialsPath),
+				FS.DETECTED);
+		c.load();
+		accessKey = c.getString("default", null, "accessKey"); //$NON-NLS-1$//$NON-NLS-2$
+		secretKey = c.getString("default", null, "secretKey"); //$NON-NLS-1$ //$NON-NLS-2$
+		if (accessKey == null || accessKey.isEmpty()) {
+			throw die(MessageFormat.format(CLIText.get().lfsNoAccessKey,
+					credentialsPath));
+		}
+		if (secretKey == null || secretKey.isEmpty()) {
+			throw die(MessageFormat.format(CLIText.get().lfsNoSecretKey,
+					credentialsPath));
 		}
 	}
 
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
index 2812137266..e99fe007d9 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java
@@ -150,6 +150,12 @@ public class CLIText extends TranslationBundle {
 	/***/ public String initializedEmptyGitRepositoryIn;
 	/***/ public String invalidHttpProxyOnlyHttpSupported;
 	/***/ public String jgitVersion;
+	/***/ public String lfsNoAccessKey;
+	/***/ public String lfsNoSecretKey;
+	/***/ public String lfsProtocolUrl;
+	/***/ public String lfsStoreDirectory;
+	/***/ public String lfsStoreUrl;
+	/***/ public String lfsUnknownStoreType;
 	/***/ public String lineFormat;
 	/***/ public String listeningOn;
 	/***/ public String mergeCheckoutConflict;
@@ -178,6 +184,7 @@ public class CLIText extends TranslationBundle {
 	/***/ public String metaVar_filepattern;
 	/***/ public String metaVar_gitDir;
 	/***/ public String metaVar_hostName;
+	/***/ public String metaVar_lfsStorage;
 	/***/ public String metaVar_linesOfContext;
 	/***/ public String metaVar_message;
 	/***/ public String metaVar_n;
@@ -192,6 +199,9 @@ public class CLIText extends TranslationBundle {
 	/***/ public String metaVar_refs;
 	/***/ public String metaVar_refspec;
 	/***/ public String metaVar_remoteName;
+	/***/ public String metaVar_s3Bucket;
+	/***/ public String metaVar_s3Region;
+	/***/ public String metaVar_s3StorageClass;
 	/***/ public String metaVar_seconds;
 	/***/ public String metaVar_service;
 	/***/ public String metaVar_treeish;
@@ -237,6 +247,7 @@ public class CLIText extends TranslationBundle {
 	/***/ public String remoteRefObjectChangedIsNotExpectedOne;
 	/***/ public String remoteSideDoesNotSupportDeletingRefs;
 	/***/ public String repaint;
+	/***/ public String s3InvalidBucket;
 	/***/ public String serviceNotSupported;
 	/***/ public String skippingObject;
 	/***/ public String statusFileListFormat;