aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrei Pozolotin <andrei.pozolotin@gmail.com>2015-09-21 22:59:14 +0000
committerAndrei Pozolotin <andrei.pozolotin@gmail.com>2015-10-18 19:14:31 +0000
commit81810aff298ffb3e871b4dbab76be2c8b9a46ea8 (patch)
treec31aaaeceba6a925aa2e2dafe81d09f1bad552bc
parentfd060943daf24873e23a49203be19f7491bd46f7 (diff)
downloadjgit-81810aff298ffb3e871b4dbab76be2c8b9a46ea8.tar.gz
jgit-81810aff298ffb3e871b4dbab76be2c8b9a46ea8.zip
Adding AES Walk Encryption support in http://www.jets3t.org/ mode
See previous attempt: https://git.eclipse.org/r/#/c/16674/ Here we preserve as much of JetS3t mode as possible while allowing to use new Java 8+ PBE algorithms such as PBEWithHmacSHA512AndAES_256 Summary of changes: * change pom.xml to control long tests * add WalkEncryptionTest.launch to run long tests * add AmazonS3.Keys to to normalize use of constants * change WalkEncryption to support AES in JetS3t mode * add WalkEncryptionTest to test remote encryption pipeline * add support for CI configuration for live Amazon S3 testing * add log4j based logging for tests in both Eclipse and Maven build To test locally, check out the review branch, then: * create amazon test configuration file * located your home dir: ${user.home} * named jgit-s3-config.properties * file format follows AmazonS3 connection settings file: accesskey = your-amazon-access-key secretkey = your-amazon-secret-key test.bucket = your-bucket-for-testing * finally: * run in Eclipse: WalkEncryptionTest.launch * or * run in Shell: mvn test --define test=WalkEncryptionTest Change-Id: I6f455fd9fb4eac261ca73d0bec6a4e7dae9f2e91 Signed-off-by: Andrei Pozolotin <andrei.pozolotin@gmail.com>
-rw-r--r--.gitignore1
-rw-r--r--org.eclipse.jgit.test/build.properties1
-rw-r--r--org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest-Proxy.launch20
-rw-r--r--org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest.launch20
-rw-r--r--org.eclipse.jgit.test/pom.xml28
-rw-r--r--org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.disabled.properties48
-rw-r--r--org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.bucket.json20
-rw-r--r--org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.user.json24
-rw-r--r--org.eclipse.jgit.test/tst-rsrc/log4j.properties9
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/WalkEncryptionTest.java1060
-rw-r--r--org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties1
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java1
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java42
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java179
14 files changed, 1388 insertions, 66 deletions
diff --git a/.gitignore b/.gitignore
index ea8c4bf7f3..139e5aee6d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
/target
+/.project
diff --git a/org.eclipse.jgit.test/build.properties b/org.eclipse.jgit.test/build.properties
index afc4855d67..786046c58a 100644
--- a/org.eclipse.jgit.test/build.properties
+++ b/org.eclipse.jgit.test/build.properties
@@ -4,3 +4,4 @@ source.. = tst/,\
bin.includes = META-INF/,\
.,\
plugin.properties
+additional.bundles = org.apache.log4j
diff --git a/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest-Proxy.launch b/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest-Proxy.launch
new file mode 100644
index 0000000000..fe3a013720
--- /dev/null
+++ b/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest-Proxy.launch
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.m2e.Maven2LaunchConfigurationType">
+<booleanAttribute key="M2_DEBUG_OUTPUT" value="false"/>
+<stringAttribute key="M2_GOALS" value="test --define test=WalkEncryptionTest --define http_proxy=http://proxy:3128"/>
+<booleanAttribute key="M2_NON_RECURSIVE" value="false"/>
+<booleanAttribute key="M2_OFFLINE" value="false"/>
+<stringAttribute key="M2_PROFILES" value=""/>
+<listAttribute key="M2_PROPERTIES"/>
+<stringAttribute key="M2_RUNTIME" value="EMBEDDED"/>
+<booleanAttribute key="M2_SKIP_TESTS" value="false"/>
+<intAttribute key="M2_THREADS" value="1"/>
+<booleanAttribute key="M2_UPDATE_SNAPSHOTS" value="false"/>
+<stringAttribute key="M2_USER_SETTINGS" value=""/>
+<booleanAttribute key="M2_WORKSPACE_RESOLUTION" value="false"/>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/java-8-oracle"/>
+<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/org.eclipse.jgit.test}"/>
+</launchConfiguration>
diff --git a/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest.launch b/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest.launch
new file mode 100644
index 0000000000..3b4a5a24e1
--- /dev/null
+++ b/org.eclipse.jgit.test/org.eclipse.jgit.test-WalkEncryptionTest.launch
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.m2e.Maven2LaunchConfigurationType">
+<booleanAttribute key="M2_DEBUG_OUTPUT" value="false"/>
+<stringAttribute key="M2_GOALS" value="test --define test=WalkEncryptionTest --activate-profiles test.long"/>
+<booleanAttribute key="M2_NON_RECURSIVE" value="false"/>
+<booleanAttribute key="M2_OFFLINE" value="false"/>
+<stringAttribute key="M2_PROFILES" value=""/>
+<listAttribute key="M2_PROPERTIES"/>
+<stringAttribute key="M2_RUNTIME" value="EMBEDDED"/>
+<booleanAttribute key="M2_SKIP_TESTS" value="false"/>
+<intAttribute key="M2_THREADS" value="1"/>
+<booleanAttribute key="M2_UPDATE_SNAPSHOTS" value="false"/>
+<stringAttribute key="M2_USER_SETTINGS" value=""/>
+<booleanAttribute key="M2_WORKSPACE_RESOLUTION" value="false"/>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/java-8-oracle"/>
+<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:/org.eclipse.jgit.test}"/>
+</launchConfiguration>
diff --git a/org.eclipse.jgit.test/pom.xml b/org.eclipse.jgit.test/pom.xml
index 51ac8463cc..539caff149 100644
--- a/org.eclipse.jgit.test/pom.xml
+++ b/org.eclipse.jgit.test/pom.xml
@@ -69,6 +69,16 @@
<scope>test</scope>
</dependency>
+ <!-- Optional security provider for encryption tests. -->
+ <!-- See https://dev.eclipse.org/ipzilla/show_bug.cgi?id=9554 -->
+ <!-- See https://bugs.eclipse.org/bugs/show_bug.cgi?id=467064 -->
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>1.52</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
@@ -101,6 +111,24 @@
</dependency>
</dependencies>
+ <profiles>
+ <!-- Profile provides a property which enables long running tests. -->
+ <profile>
+ <id>test.long</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <argLine>-Djgit.test.long=true</argLine>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<build>
<sourceDirectory>src/</sourceDirectory>
<testSourceDirectory>tst/</testSourceDirectory>
diff --git a/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.disabled.properties b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.disabled.properties
new file mode 100644
index 0000000000..d540977e94
--- /dev/null
+++ b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.disabled.properties
@@ -0,0 +1,48 @@
+#
+# See WalkEncryptionTest.java
+#
+# This file is a template for test configuration file used by WalkEncryptionTest.
+# To be active, this file must have the following hard coded name: jgit-s3-config.properties
+# To be active, this file must be discovered by WalkEncryptionTest from one of these locations:
+# * ${user.home}/jgit-s3-config.properties
+# * ${user.dir}/jgit-s3-config.properties
+# * ${user.dir}/tst-rsrc/jgit-s3-config.properties
+# When this file is missing, tests in WalkEncryptionTest will not run, only report a warning.
+#
+
+#
+# WalkEncryptionTest requires amazon s3 test bucket setup.
+#
+# Test bucket setup instructions:
+#
+# Create IAM user:
+# http://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html
+# * user name: jgit.eclipse.org
+#
+# Configure IAM user S3 bucket access
+# http://docs.aws.amazon.com/AmazonS3/latest/dev/example-policies-s3.html
+# * attach S3 user policy to user account: jgit-s3-config.policy.user.json
+#
+# Create S3 bucket:
+# http://docs.aws.amazon.com/AmazonS3/latest/gsg/CreatingABucket.html
+# * bucket name: jgit.eclipse.org
+#
+# Configure S3 bucket source address/mask access:
+# http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html
+# * attach bucket policy to the test bucket: jgit-s3-config.policy.bucket.json
+# * verify that any required source address/mask is included in the bucket policy:
+# * see https://wiki.eclipse.org/Hudson
+# * see http://www.tcpiputils.com/browse/ip-address/198.41.30.200
+# * proxy.eclipse.org 198.41.30.0/24
+# * Andrei Pozolotin 67.175.188.187/32
+#
+# Configure bucket 1 day expiration in object life cycle management:
+# * https://docs.aws.amazon.com/AmazonS3/latest/dev/manage-lifecycle-using-console.html
+#
+
+# Test bucket name
+test.bucket=jgit.eclipse.org
+
+# IAM credentials for user jgit.eclipse.org
+accesskey=AKIAIYWXB4ETREBRMZDQ
+secretkey=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
diff --git a/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.bucket.json b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.bucket.json
new file mode 100644
index 0000000000..3020b09a00
--- /dev/null
+++ b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.bucket.json
@@ -0,0 +1,20 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Sid": "DenyAllButKnownSourceAddressWithMask",
+ "Effect": "Deny",
+ "Principal": "*",
+ "Action": "s3:*",
+ "Resource": "arn:aws:s3:::jgit.eclipse.org/*",
+ "Condition": {
+ "NotIpAddress": {
+ "aws:SourceIp": [
+ "198.41.30.0/24",
+ "67.175.188.187/32"
+ ]
+ }
+ }
+ }
+ ]
+}
diff --git a/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.user.json b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.user.json
new file mode 100644
index 0000000000..830d0888c0
--- /dev/null
+++ b/org.eclipse.jgit.test/tst-rsrc/jgit-s3-config.policy.user.json
@@ -0,0 +1,24 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Sid": "BucketList",
+ "Effect": "Allow",
+ "Action": "s3:ListAllMyBuckets",
+ "Resource": [
+ "arn:aws:s3:::jgit.eclipse.org"
+ ]
+ },
+ {
+ "Sid": "BucketFullControl",
+ "Effect": "Allow",
+ "Action": [
+ "s3:*"
+ ],
+ "Resource": [
+ "arn:aws:s3:::jgit.eclipse.org",
+ "arn:aws:s3:::jgit.eclipse.org/*"
+ ]
+ }
+ ]
+}
diff --git a/org.eclipse.jgit.test/tst-rsrc/log4j.properties b/org.eclipse.jgit.test/tst-rsrc/log4j.properties
new file mode 100644
index 0000000000..14620ffae4
--- /dev/null
+++ b/org.eclipse.jgit.test/tst-rsrc/log4j.properties
@@ -0,0 +1,9 @@
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/WalkEncryptionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/WalkEncryptionTest.java
new file mode 100644
index 0000000000..f2701cb41f
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/WalkEncryptionTest.java
@@ -0,0 +1,1060 @@
+/*
+ * Copyright (C) 2015, Andrei Pozolotin.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.transport;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.security.GeneralSecurityException;
+import java.security.Provider;
+import java.security.Security;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+
+import javax.crypto.SecretKeyFactory;
+
+import org.apache.log4j.Logger;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
+import org.eclipse.jgit.util.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Suite;
+
+import static org.eclipse.jgit.transport.WalkEncryptionTest.Util.*;
+
+/**
+ * Amazon S3 encryption pipeline test.
+ *
+ * See {@link AmazonS3} {@link WalkEncryption}
+ *
+ * Note: CI server must provide amazon credentials (access key, secret key,
+ * bucket name) via one of methods available in {@link Names}.
+ *
+ * Note: long running tests are activated by Maven profile "test.long". There is
+ * also a separate Eclipse m2e launcher for that. See 'pom.xml' and
+ * 'WalkEncryptionTest.launch'.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({ //
+ WalkEncryptionTest.MinimalSet.class, //
+ WalkEncryptionTest.TestablePBE.class, //
+})
+public class WalkEncryptionTest {
+
+ /**
+ * Logger setup: ${project_loc}/tst-rsrc/log4j.properties
+ */
+ static final Logger logger = Logger.getLogger(WalkEncryptionTest.class);
+
+ /**
+ * Property names used in test session.
+ */
+ interface Names {
+
+ // Names of discovered test properties.
+
+ String TEST_BUCKET = "test.bucket";
+
+ // Names of test environment variables for CI.
+
+ String ENV_ACCESS_KEY = "JGIT_S3_ACCESS_KEY";
+
+ String ENV_SECRET_KEY = "JGIT_S3_SECRET_KEY";
+
+ String ENV_BUCKET_NAME = "JGIT_S3_BUCKET_NAME";
+
+ // Name of test environment variable file path for CI.
+
+ String ENV_CONFIG_FILE = "JGIT_S3_CONFIG_FILE";
+
+ // Names of test system properties for CI.
+
+ String SYS_ACCESS_KEY = "jgit.s3.access.key";
+
+ String SYS_SECRET_KEY = "jgit.s3.secret.key";
+
+ String SYS_BUCKET_NAME = "jgit.s3.bucket.name";
+
+ // Name of test system property file path for CI.
+ String SYS_CONFIG_FILE = "jgit.s3.config.file";
+
+ // Hard coded name of test properties file for CI.
+ // File format follows AmazonS3.Keys:
+ // #
+ // # Required entries:
+ // #
+ // accesskey = your-amazon-access-key # default AmazonS3.Keys
+ // secretkey = your-amazon-secret-key # default AmazonS3.Keys
+ // test.bucket = your-bucket-for-testing # custom name, for this test
+ String CONFIG_FILE = "jgit-s3-config.properties";
+
+ // Test properties file in [user home] of CI.
+ String HOME_CONFIG_FILE = System.getProperty("user.home")
+ + File.separator + CONFIG_FILE;
+
+ // Test properties file in [project work directory] of CI.
+ String WORK_CONFIG_FILE = System.getProperty("user.dir")
+ + File.separator + CONFIG_FILE;
+
+ // Test properties file in [project test source directory] of CI.
+ String TEST_CONFIG_FILE = System.getProperty("user.dir")
+ + File.separator + "tst-rsrc" + File.separator + CONFIG_FILE;
+
+ }
+
+ /**
+ * Find test properties from various sources in order of priority.
+ */
+ static class Props implements WalkEncryptionTest.Names, AmazonS3.Keys {
+
+ static boolean haveEnvVar(String name) {
+ return System.getenv(name) != null;
+ }
+
+ static boolean haveEnvVarFile(String name) {
+ return haveEnvVar(name) && new File(name).exists();
+ }
+
+ static boolean haveSysProp(String name) {
+ return System.getProperty(name) != null;
+ }
+
+ static boolean haveSysPropFile(String name) {
+ return haveSysProp(name) && new File(name).exists();
+ }
+
+ static void loadEnvVar(String source, String target, Properties props) {
+ props.put(target, System.getenv(source));
+ }
+
+ static void loadSysProp(String source, String target,
+ Properties props) {
+ props.put(target, System.getProperty(source));
+ }
+
+ static boolean haveProp(String name, Properties props) {
+ return props.containsKey(name);
+ }
+
+ static boolean checkTestProps(Properties props) {
+ return haveProp(ACCESS_KEY, props) && haveProp(SECRET_KEY, props)
+ && haveProp(TEST_BUCKET, props);
+ }
+
+ static Properties fromEnvVars() {
+ if (haveEnvVar(ENV_ACCESS_KEY) && haveEnvVar(ENV_SECRET_KEY)
+ && haveEnvVar(ENV_BUCKET_NAME)) {
+ Properties props = new Properties();
+ loadEnvVar(ENV_ACCESS_KEY, ACCESS_KEY, props);
+ loadEnvVar(ENV_SECRET_KEY, SECRET_KEY, props);
+ loadEnvVar(ENV_BUCKET_NAME, TEST_BUCKET, props);
+ return props;
+ } else {
+ return null;
+ }
+ }
+
+ static Properties fromEnvFile() throws Exception {
+ if (haveEnvVarFile(ENV_CONFIG_FILE)) {
+ Properties props = new Properties();
+ props.load(new FileInputStream(ENV_CONFIG_FILE));
+ if (checkTestProps(props)) {
+ return props;
+ } else {
+ throw new Error("Environment config file is incomplete.");
+ }
+ } else {
+ return null;
+ }
+ }
+
+ static Properties fromSysProps() {
+ if (haveSysProp(SYS_ACCESS_KEY) && haveSysProp(SYS_SECRET_KEY)
+ && haveSysProp(SYS_BUCKET_NAME)) {
+ Properties props = new Properties();
+ loadSysProp(SYS_ACCESS_KEY, ACCESS_KEY, props);
+ loadSysProp(SYS_SECRET_KEY, SECRET_KEY, props);
+ loadSysProp(SYS_BUCKET_NAME, TEST_BUCKET, props);
+ return props;
+ } else {
+ return null;
+ }
+ }
+
+ static Properties fromSysFile() throws Exception {
+ if (haveSysPropFile(SYS_CONFIG_FILE)) {
+ Properties props = new Properties();
+ props.load(new FileInputStream(SYS_CONFIG_FILE));
+ if (checkTestProps(props)) {
+ return props;
+ } else {
+ throw new Error("System props config file is incomplete.");
+ }
+ } else {
+ return null;
+ }
+ }
+
+ static Properties fromConfigFile(String path) throws Exception {
+ File file = new File(path);
+ if (file.exists()) {
+ Properties props = new Properties();
+ props.load(new FileInputStream(file));
+ if (checkTestProps(props)) {
+ return props;
+ } else {
+ throw new Error("Props config file is incomplete: " + path);
+ }
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Find test properties from various sources in order of priority.
+ *
+ * @return result
+ * @throws Exception
+ */
+ static Properties discover() throws Exception {
+ Properties props;
+ if ((props = fromEnvVars()) != null) {
+ logger.debug(
+ "Using test properties from environment variables.");
+ return props;
+ }
+ if ((props = fromEnvFile()) != null) {
+ logger.debug(
+ "Using test properties from environment variable config file.");
+ return props;
+ }
+ if ((props = fromSysProps()) != null) {
+ logger.debug("Using test properties from system properties.");
+ return props;
+ }
+ if ((props = fromSysFile()) != null) {
+ logger.debug(
+ "Using test properties from system property config file.");
+ return props;
+ }
+ if ((props = fromConfigFile(HOME_CONFIG_FILE)) != null) {
+ logger.debug(
+ "Using test properties from hard coded ${user.home} file.");
+ return props;
+ }
+ if ((props = fromConfigFile(WORK_CONFIG_FILE)) != null) {
+ logger.debug(
+ "Using test properties from hard coded ${user.dir} file.");
+ return props;
+ }
+ if ((props = fromConfigFile(TEST_CONFIG_FILE)) != null) {
+ logger.debug(
+ "Using test properties from hard coded ${project.source} file.");
+ return props;
+ }
+ throw new Error("Can not load test properties form any source.");
+ }
+
+ }
+
+ /**
+ * Collection of test utility methods.
+ */
+ static class Util {
+
+ static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ /**
+ * Read UTF-8 encoded text file into string.
+ *
+ * @param file
+ * @return result
+ * @throws Exception
+ */
+ static String textRead(File file) throws Exception {
+ return new String(Files.readAllBytes(file.toPath()), UTF_8);
+ }
+
+ /**
+ * Write string into UTF-8 encoded file.
+ *
+ * @param file
+ * @param text
+ * @throws Exception
+ */
+ static void textWrite(File file, String text) throws Exception {
+ Files.write(file.toPath(), text.getBytes(UTF_8));
+ }
+
+ static void verifyFileContent(File fileOne, File fileTwo)
+ throws Exception {
+ assertTrue(fileOne.length() > 0);
+ assertTrue(fileTwo.length() > 0);
+ String textOne = textRead(fileOne);
+ String textTwo = textRead(fileTwo);
+ assertEquals(textOne, textTwo);
+ }
+
+ /**
+ * Create local folder.
+ *
+ * @param folder
+ * @throws Exception
+ */
+ static void folderCreate(String folder) throws Exception {
+ File path = new File(folder);
+ assertTrue(path.mkdirs());
+ }
+
+ /**
+ * Delete local folder.
+ *
+ * @param folder
+ * @throws Exception
+ */
+ static void folderDelete(String folder) throws Exception {
+ File path = new File(folder);
+ FileUtils.delete(path,
+ FileUtils.RECURSIVE | FileUtils.SKIP_MISSING);
+ }
+
+ /**
+ * Discover public address of CI server.
+ *
+ * @return result
+ * @throws Exception
+ */
+ static String publicAddress() throws Exception {
+ String service = "http://checkip.amazonaws.com";
+ URL url = new URL(service);
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(url.openStream()));
+ try {
+ return reader.readLine();
+ } finally {
+ reader.close();
+ }
+ }
+
+ /**
+ * Discover Password-Based Encryption (PBE) engines providing both
+ * [SecretKeyFactory] and [AlgorithmParameters].
+ *
+ * @return result
+ */
+ // https://www.bouncycastle.org/specifications.html
+ // https://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html
+ static List<String> cryptoCipherListPBE() {
+ return cryptoCipherList("(PBE).*(WITH).+(AND).+");
+ }
+
+ static String securityProviderName(String algorithm) throws Exception {
+ return SecretKeyFactory.getInstance(algorithm).getProvider()
+ .getName();
+ }
+
+ static List<String> cryptoCipherList(String regex) {
+ Set<String> source = Security.getAlgorithms("Cipher");
+ Set<String> target = new TreeSet<String>();
+ for (String algo : source) {
+ algo = algo.toUpperCase();
+ if (algo.matches(regex)) {
+ target.add(algo);
+ }
+ }
+ return new ArrayList<String>(target);
+ }
+
+ /**
+ * Verify if any security provider published the algorithm.
+ *
+ * @param algorithm
+ * @return result
+ */
+ static boolean isAlgorithmPresent(String algorithm) {
+ Set<String> cipherSet = Security.getAlgorithms("Cipher");
+ for (String source : cipherSet) {
+ // Standard names are not case-sensitive.
+ // http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html
+ String target = algorithm.toUpperCase();
+ if (source.equalsIgnoreCase(target)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Stream copy.
+ *
+ * @param from
+ * @param into
+ * @return count
+ * @throws IOException
+ */
+ static long transferStream(InputStream from, OutputStream into)
+ throws IOException {
+ byte[] array = new byte[1 * 1024];
+ long total = 0;
+ while (true) {
+ int count = from.read(array);
+ if (count == -1) {
+ break;
+ }
+ into.write(array, 0, count);
+ total += count;
+ }
+ return total;
+ }
+
+ /**
+ * Setup proxy during CI build.
+ *
+ * @throws Exception
+ */
+ // https://wiki.eclipse.org/Hudson#Accessing_the_Internet_using_Proxy
+ // http://docs.oracle.com/javase/7/docs/api/java/net/doc-files/net-properties.html
+ static void proxySetup() throws Exception {
+ String keyNoProxy = "no_proxy";
+ String keyHttpProxy = "http_proxy";
+ String keyHttpsProxy = "https_proxy";
+
+ String no_proxy = System.getProperty(keyNoProxy,
+ System.getenv(keyNoProxy));
+ if (no_proxy != null) {
+ System.setProperty("http.nonProxyHosts", no_proxy);
+ logger.info("Proxy NOT: " + no_proxy);
+ }
+
+ String http_proxy = System.getProperty(keyHttpProxy,
+ System.getenv(keyHttpProxy));
+ if (http_proxy != null) {
+ URL url = new URL(http_proxy);
+ System.setProperty("http.proxyHost", url.getHost());
+ System.setProperty("http.proxyPort", "" + url.getPort());
+ logger.info("Proxy HTTP: " + http_proxy);
+ }
+
+ String https_proxy = System.getProperty(keyHttpsProxy,
+ System.getenv(keyHttpsProxy));
+ if (https_proxy != null) {
+ URL url = new URL(https_proxy);
+ System.setProperty("https.proxyHost", url.getHost());
+ System.setProperty("https.proxyPort", "" + url.getPort());
+ logger.info("Proxy HTTPS: " + https_proxy);
+ }
+
+ if (no_proxy == null && http_proxy == null && https_proxy == null) {
+ logger.info("Proxy not used.");
+ }
+
+ }
+
+ /**
+ * Permit long tests on CI or with manual activation.
+ *
+ * @return result
+ */
+ static boolean permitLongTests() {
+ return isBuildCI() || isProfileActive();
+ }
+
+ /**
+ * Using Maven profile activation, see pom.xml
+ *
+ * @return result
+ */
+ static boolean isProfileActive() {
+ return Boolean.parseBoolean(System.getProperty("jgit.test.long"));
+ }
+
+ /**
+ * Detect if build is running on CI.
+ *
+ * @return result
+ */
+ static boolean isBuildCI() {
+ return System.getenv("HUDSON_HOME") != null;
+ }
+
+ }
+
+ /**
+ * Common base for encryption tests.
+ */
+ @FixMethodOrder(MethodSorters.NAME_ASCENDING)
+ public abstract static class Base extends SampleDataRepositoryTestCase {
+
+ /**
+ * S3 URI user used by JGIT to discover connection configuration file.
+ */
+ static final String JGIT_USER = "tester-" + System.currentTimeMillis();
+
+ /**
+ * S3 content encoding password used for this test session.
+ */
+ static final String JGIT_PASS = "secret-" + System.currentTimeMillis();
+
+ /**
+ * S3 repository configuration file expected by {@link AmazonS3}.
+ */
+ static final String JGIT_CONF_FILE = System.getProperty("user.home")
+ + "/" + JGIT_USER;
+
+ /**
+ * Name representing remote or local JGIT repository.
+ */
+ static final String JGIT_REPO_DIR = JGIT_USER + ".jgit";
+
+ /**
+ * Local JGIT repository for this test session.
+ */
+ static final String JGIT_LOCAL_DIR = System.getProperty("user.dir")
+ + "/target/" + JGIT_REPO_DIR;
+
+ /**
+ * Remote JGIT repository for this test session.
+ */
+ static final String JGIT_REMOTE_DIR = JGIT_REPO_DIR;
+
+ /**
+ * Generate JGIT S3 connection configuration file.
+ *
+ * @param algorithm
+ * @throws Exception
+ */
+ static void configCreate(String algorithm) throws Exception {
+ Properties props = Props.discover();
+ props.put(AmazonS3.Keys.PASSWORD, JGIT_PASS);
+ props.put(AmazonS3.Keys.CRYPTO_ALG, algorithm);
+ PrintWriter writer = new PrintWriter(JGIT_CONF_FILE);
+ props.store(writer, "JGIT S3 connection configuration file.");
+ writer.close();
+ }
+
+ /**
+ * Remove JGIT connection configuration file.
+ *
+ * @throws Exception
+ */
+ static void configDelete() throws Exception {
+ File path = new File(JGIT_CONF_FILE);
+ FileUtils.delete(path, FileUtils.SKIP_MISSING);
+ }
+
+ /**
+ * Generate remote URI for the test session.
+ *
+ * @return result
+ * @throws Exception
+ */
+ static String amazonURI() throws Exception {
+ Properties props = Props.discover();
+ String bucket = props.getProperty(Names.TEST_BUCKET);
+ assertNotNull(bucket);
+ return TransportAmazonS3.S3_SCHEME + "://" + JGIT_USER + "@"
+ + bucket + "/" + JGIT_REPO_DIR;
+ }
+
+ /**
+ * Create S3 repository folder.
+ *
+ * @throws Exception
+ */
+ static void remoteCreate() throws Exception {
+ Properties props = Props.discover();
+ props.remove(AmazonS3.Keys.PASSWORD); // Disable encryption.
+ String bucket = props.getProperty(Names.TEST_BUCKET);
+ AmazonS3 s3 = new AmazonS3(props);
+ String path = JGIT_REMOTE_DIR + "/";
+ s3.put(bucket, path, new byte[0]);
+ logger.debug("remote create: " + JGIT_REMOTE_DIR);
+ }
+
+ /**
+ * Delete S3 repository folder.
+ *
+ * @throws Exception
+ */
+ static void remoteDelete() throws Exception {
+ Properties props = Props.discover();
+ props.remove(AmazonS3.Keys.PASSWORD); // Disable encryption.
+ String bucket = props.getProperty(Names.TEST_BUCKET);
+ AmazonS3 s3 = new AmazonS3(props);
+ List<String> list = s3.list(bucket, JGIT_REMOTE_DIR);
+ for (String path : list) {
+ path = JGIT_REMOTE_DIR + "/" + path;
+ s3.delete(bucket, path);
+ }
+ logger.debug("remote delete: " + JGIT_REMOTE_DIR);
+ }
+
+ /**
+ * Verify if we can create/delete remote file.
+ *
+ * @throws Exception
+ */
+ static void remoteVerify() throws Exception {
+ Properties props = Props.discover();
+ String bucket = props.getProperty(Names.TEST_BUCKET);
+ AmazonS3 s3 = new AmazonS3(props);
+ String file = JGIT_USER + "-" + UUID.randomUUID().toString();
+ String path = JGIT_REMOTE_DIR + "/" + file;
+ s3.put(bucket, path, file.getBytes(UTF_8));
+ s3.delete(bucket, path);
+ }
+
+ /**
+ * Verify if JRE security policy allows the algorithm.
+ *
+ * @param algorithm
+ * @return result
+ */
+ static boolean isAlgorithmAllowed(String algorithm) {
+ try {
+ WalkEncryption crypto = new WalkEncryption.ObjectEncryptionJetS3tV2(
+ algorithm, JGIT_PASS);
+ verifyCrypto(crypto);
+ return true;
+ } catch (IOException e) {
+ return false; // Encryption failure.
+ } catch (GeneralSecurityException e) {
+ throw new Error(e); // Construction failure.
+ }
+ }
+
+ /**
+ * Verify round trip encryption.
+ *
+ * @param crypto
+ * @throws IOException
+ */
+ static void verifyCrypto(WalkEncryption crypto) throws IOException {
+ String charset = "UTF-8";
+ String sourceText = "secret-message Свобода 老子";
+ String targetText;
+ byte[] cipherText;
+ {
+ byte[] origin = sourceText.getBytes(charset);
+ ByteArrayOutputStream target = new ByteArrayOutputStream();
+ OutputStream source = crypto.encrypt(target);
+ source.write(origin);
+ source.flush();
+ source.close();
+ cipherText = target.toByteArray();
+ }
+ {
+ InputStream source = new ByteArrayInputStream(cipherText);
+ InputStream target = crypto.decrypt(source);
+ ByteArrayOutputStream result = new ByteArrayOutputStream();
+ transferStream(target, result);
+ targetText = result.toString(charset);
+ }
+ assertEquals(sourceText, targetText);
+ }
+
+ /**
+ * Algorithm is testable when it is present and allowed by policy.
+ *
+ * @param algorithm
+ * @return result
+ */
+ static boolean isAlgorithmTestable(String algorithm) {
+ return isAlgorithmPresent(algorithm)
+ && isAlgorithmAllowed(algorithm);
+ }
+
+ /**
+ * Log algorithm, provider, testability.
+ *
+ * @param algorithm
+ * @throws Exception
+ */
+ static void reportAlgorithmStatus(String algorithm) throws Exception {
+ final boolean present = isAlgorithmPresent(algorithm);
+ final boolean allowed = present && isAlgorithmAllowed(algorithm);
+ final String provider = present ? securityProviderName(algorithm)
+ : "N/A";
+ String status = "Algorithm: " + algorithm + " @ " + provider + "; "
+ + "present/allowed : " + present + "/" + allowed;
+ if (allowed) {
+ logger.info("Testing " + status);
+ } else {
+ logger.warn("Missing " + status);
+ }
+ }
+
+ /**
+ * Verify if we can perform remote tests.
+ *
+ * @return result
+ */
+ static boolean isTestConfigPresent() {
+ try {
+ Props.discover();
+ return true;
+ } catch (Throwable e) {
+ return false;
+ }
+ }
+
+ static void reportTestConfigPresent() {
+ if (isTestConfigPresent()) {
+ logger.info("Amazon S3 test configuration is present.");
+ } else {
+ logger.error(
+ "Amazon S3 test configuration is missing, tests will not run.");
+ }
+ }
+
+ /**
+ * Log public address of CI.
+ *
+ * @throws Exception
+ */
+ static void reportPublicAddress() throws Exception {
+ logger.info("Public address: " + publicAddress());
+ }
+
+ /**
+ * BouncyCastle provider class.
+ *
+ * Needs extra dependency, see pom.xml
+ */
+ // http://search.maven.org/#artifactdetails%7Corg.bouncycastle%7Cbcprov-jdk15on%7C1.52%7Cjar
+ static final String PROVIDER_BC = "org.bouncycastle.jce.provider.BouncyCastleProvider";
+
+ /**
+ * Load BouncyCastle provider if present.
+ */
+ static void loadBouncyCastle() {
+ try {
+ Class<?> provider = Class.forName(PROVIDER_BC);
+ Provider instance = (Provider) provider
+ .getConstructor(new Class[] {})
+ .newInstance(new Object[] {});
+ Security.addProvider(instance);
+ logger.info("Loaded " + PROVIDER_BC);
+ } catch (Throwable e) {
+ logger.warn("Failed to load " + PROVIDER_BC);
+ }
+ }
+
+ static void reportLongTests() {
+ if (permitLongTests()) {
+ logger.info("Long running tests are enabled.");
+ } else {
+ logger.warn("Long running tests are disabled.");
+ }
+ }
+
+ /**
+ * Non-PBE algorithm, for error check.
+ */
+ static final String ALGO_ERROR = "PBKDF2WithHmacSHA1";
+
+ /**
+ * Default JetS3t algorithm present in most JRE.
+ */
+ static final String ALGO_JETS3T = "PBEWithMD5AndDES";
+
+ /**
+ * Minimal strength AES based algorithm present in most JRE.
+ */
+ static final String ALGO_MINIMAL_AES = "PBEWithHmacSHA1AndAES_128";
+
+ /**
+ * Selected non-AES algorithm present in BouncyCastle provider.
+ */
+ static final String ALGO_BOUNCY_CASTLE_CBC = "PBEWithSHAAndTwofish-CBC";
+
+ //////////////////////////////////////////////////
+
+ @BeforeClass
+ public static void initialize() throws Exception {
+ Transport.register(TransportAmazonS3.PROTO_S3);
+ proxySetup();
+ reportLongTests();
+ reportPublicAddress();
+ reportTestConfigPresent();
+ loadBouncyCastle();
+ if (isTestConfigPresent()) {
+ remoteCreate();
+ }
+ }
+
+ @AfterClass
+ public static void terminate() throws Exception {
+ configDelete();
+ folderDelete(JGIT_LOCAL_DIR);
+ if (isTestConfigPresent()) {
+ remoteDelete();
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /**
+ * Optional encrypted amazon remote JGIT life cycle test.
+ *
+ * @param algorithm
+ * @throws Exception
+ */
+ void cryptoTestIfCan(String algorithm) throws Exception {
+ reportAlgorithmStatus(algorithm);
+ assumeTrue(isTestConfigPresent());
+ assumeTrue(isAlgorithmTestable(algorithm));
+ cryptoTest(algorithm);
+ }
+
+ /**
+ * Required encrypted amazon remote JGIT life cycle test.
+ *
+ * @param algorithm
+ * @throws Exception
+ */
+ void cryptoTest(String algorithm) throws Exception {
+
+ remoteDelete();
+ configCreate(algorithm);
+ folderDelete(JGIT_LOCAL_DIR);
+
+ String uri = amazonURI();
+
+ // Local repositories.
+ File dirOne = db.getWorkTree(); // Provided by setup.
+ File dirTwo = new File(JGIT_LOCAL_DIR);
+
+ // Local verification files.
+ String nameStatic = "master.txt"; // Provided by setup.
+ String nameDynamic = JGIT_USER + "-" + UUID.randomUUID().toString();
+
+ String remote = "remote";
+ RefSpec specs = new RefSpec("refs/heads/master:refs/heads/master");
+
+ { // Push into remote from local one.
+
+ StoredConfig config = db.getConfig();
+ RemoteConfig remoteConfig = new RemoteConfig(config, remote);
+ remoteConfig.addURI(new URIish(uri));
+ remoteConfig.update(config);
+ config.save();
+
+ Git git = Git.open(dirOne);
+ git.checkout().setName("master").call();
+ git.push().setRemote(remote).setRefSpecs(specs).call();
+ git.close();
+
+ File fileStatic = new File(dirOne, nameStatic);
+ assertTrue("Provided by setup", fileStatic.exists());
+
+ }
+
+ { // Clone from remote into local two.
+
+ File fileStatic = new File(dirTwo, nameStatic);
+ assertFalse("Not Provided by setup", fileStatic.exists());
+
+ Git git = Git.cloneRepository().setURI(uri).setDirectory(dirTwo)
+ .call();
+ git.close();
+
+ assertTrue("Provided by clone", fileStatic.exists());
+ }
+
+ { // Verify static file content.
+ File fileOne = new File(dirOne, nameStatic);
+ File fileTwo = new File(dirTwo, nameStatic);
+ verifyFileContent(fileOne, fileTwo);
+ }
+
+ { // Verify new file commit and push from local one.
+
+ File fileDynamic = new File(dirOne, nameDynamic);
+ assertFalse("Not Provided by setup", fileDynamic.exists());
+ FileUtils.createNewFile(fileDynamic);
+ textWrite(fileDynamic, nameDynamic);
+ assertTrue("Provided by create", fileDynamic.exists());
+ assertTrue("Need content to encrypt", fileDynamic.length() > 0);
+
+ Git git = Git.open(dirOne);
+ git.add().addFilepattern(nameDynamic).call();
+ git.commit().setMessage(nameDynamic).call();
+ git.push().setRemote(remote).setRefSpecs(specs).call();
+ git.close();
+
+ }
+
+ { // Verify new file pull from remote into local two.
+
+ File fileDynamic = new File(dirTwo, nameDynamic);
+ assertFalse("Not Provided by setup", fileDynamic.exists());
+
+ Git git = Git.open(dirTwo);
+ git.pull().call();
+ git.close();
+
+ assertTrue("Provided by pull", fileDynamic.exists());
+ }
+
+ { // Verify dynamic file content.
+ File fileOne = new File(dirOne, nameDynamic);
+ File fileTwo = new File(dirTwo, nameDynamic);
+ verifyFileContent(fileOne, fileTwo);
+ }
+
+ }
+
+ }
+
+ /**
+ * Test minimal set of algorithms.
+ */
+ @FixMethodOrder(MethodSorters.NAME_ASCENDING)
+ public static class MinimalSet extends Base {
+
+ @Test
+ public void test_A1_ValidURI() throws Exception {
+ assumeTrue(isTestConfigPresent());
+ URIish uri = new URIish(amazonURI());
+ assertTrue("uri=" + uri, TransportAmazonS3.PROTO_S3.canHandle(uri));
+ }
+
+ @Test(expected = Exception.class)
+ public void test_A2_CryptoError() throws Exception {
+ assumeTrue(isTestConfigPresent());
+ cryptoTest(ALGO_ERROR);
+ }
+
+ @Test
+ public void test_A3_CryptoJetS3tDefault() throws Exception {
+ cryptoTestIfCan(ALGO_JETS3T);
+ }
+
+ @Test
+ public void test_A4_CryptoMinimalAES() throws Exception {
+ cryptoTestIfCan(ALGO_MINIMAL_AES);
+ }
+
+ @Test
+ public void test_A5_CryptoBouncyCastleCBC() throws Exception {
+ cryptoTestIfCan(ALGO_BOUNCY_CASTLE_CBC);
+ }
+
+ }
+
+ /**
+ * Test all present and allowed PBE algorithms.
+ */
+ // https://github.com/junit-team/junit/wiki/Parameterized-tests
+ @RunWith(Parameterized.class)
+ @FixMethodOrder(MethodSorters.NAME_ASCENDING)
+ public static class TestablePBE extends Base {
+
+ @Parameters(name = "Algorithm: {0}")
+ public static Collection algorimthmList() {
+ List<String> source = cryptoCipherListPBE();
+ List<Object[]> target = new ArrayList<Object[]>();
+ for (String name : source) {
+ target.add(new Object[] { name });
+ }
+ return target;
+ }
+
+ final String algorithm;
+
+ public TestablePBE(String algorithm) {
+ this.algorithm = algorithm;
+ }
+
+ @Test // Can take long time, needs activation.
+ public void test_B1_Crypto() throws Exception {
+ assumeTrue(permitLongTests());
+ cryptoTestIfCan(algorithm);
+ }
+
+ }
+
+}
diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index 34bbb415ba..51e44fd778 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -228,6 +228,7 @@ emptyCommit=No changes
emptyPathNotPermitted=Empty path not permitted.
emptyRef=Empty ref: {0}
encryptionError=Encryption error: {0}
+encryptionOnlyPBE=Encryption error: only password-based encryption (PBE) algorithms are supported.
endOfFileInEscape=End of file in escape
entryNotFoundByPath=Entry not found by path: {0}
enumValueNotSupported2=Invalid value: {0}.{1}={2}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 9067e82954..e39469bd8c 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -287,6 +287,7 @@ public class JGitText extends TranslationBundle {
/***/ public String emptyPathNotPermitted;
/***/ public String emptyRef;
/***/ public String encryptionError;
+ /***/ public String encryptionOnlyPBE;
/***/ public String endOfFileInEscape;
/***/ public String entryNotFoundByPath;
/***/ public String enumValueNotSupported2;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
index d3cdba5bf3..e55066a8bb 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
@@ -56,10 +56,10 @@ import java.net.ProxySelector;
import java.net.URL;
import java.net.URLConnection;
import java.security.DigestOutputStream;
+import java.security.GeneralSecurityException;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.security.spec.InvalidKeySpecException;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -186,6 +186,19 @@ public class AmazonS3 {
/** S3 Bucket Domain. */
private final String domain;
+ /** Property names used in amazon connection configuration file. */
+ interface Keys {
+ String ACCESS_KEY = "accesskey"; //$NON-NLS-1$
+ String SECRET_KEY = "secretkey"; //$NON-NLS-1$
+ String PASSWORD = "password"; //$NON-NLS-1$
+ String CRYPTO_ALG = "crypto.algorithm"; //$NON-NLS-1$
+ String CRYPTO_VER = "crypto.version"; //$NON-NLS-1$
+ String ACL = "acl"; //$NON-NLS-1$
+ String DOMAIN = "domain"; //$NON-NLS-1$
+ String HTTP_RETRY = "httpclient.retry-max"; //$NON-NLS-1$
+ String TMP_DIR = "tmpdir"; //$NON-NLS-1$
+ }
+
/**
* Create a new S3 client for the supplied user information.
* <p>
@@ -219,17 +232,18 @@ public class AmazonS3 {
*
*/
public AmazonS3(final Properties props) {
- domain = props.getProperty("domain", "s3.amazonaws.com"); //$NON-NLS-1$ //$NON-NLS-2$
- publicKey = props.getProperty("accesskey"); //$NON-NLS-1$
+ domain = props.getProperty(Keys.DOMAIN, "s3.amazonaws.com"); //$NON-NLS-1$
+
+ publicKey = props.getProperty(Keys.ACCESS_KEY);
if (publicKey == null)
throw new IllegalArgumentException(JGitText.get().missingAccesskey);
- final String secret = props.getProperty("secretkey"); //$NON-NLS-1$
+ final String secret = props.getProperty(Keys.SECRET_KEY);
if (secret == null)
throw new IllegalArgumentException(JGitText.get().missingSecretkey);
privateKey = new SecretKeySpec(Constants.encodeASCII(secret), HMAC);
- final String pacl = props.getProperty("acl", "PRIVATE"); //$NON-NLS-1$ //$NON-NLS-2$
+ final String pacl = props.getProperty(Keys.ACL, "PRIVATE"); //$NON-NLS-1$
if (StringUtils.equalsIgnoreCase("PRIVATE", pacl)) //$NON-NLS-1$
acl = "private"; //$NON-NLS-1$
else if (StringUtils.equalsIgnoreCase("PUBLIC", pacl)) //$NON-NLS-1$
@@ -242,26 +256,24 @@ public class AmazonS3 {
throw new IllegalArgumentException("Invalid acl: " + pacl); //$NON-NLS-1$
try {
- final String cPas = props.getProperty("password"); //$NON-NLS-1$
+ final String cPas = props.getProperty(Keys.PASSWORD);
if (cPas != null) {
- String cAlg = props.getProperty("crypto.algorithm"); //$NON-NLS-1$
+ String cAlg = props.getProperty(Keys.CRYPTO_ALG);
if (cAlg == null)
- cAlg = "PBEWithMD5AndDES"; //$NON-NLS-1$
- encryption = new WalkEncryption.ObjectEncryptionV2(cAlg, cPas);
+ cAlg = WalkEncryption.ObjectEncryptionJetS3tV2.JETS3T_ALGORITHM;
+ encryption = new WalkEncryption.ObjectEncryptionJetS3tV2(cAlg, cPas);
} else {
encryption = WalkEncryption.NONE;
}
- } catch (InvalidKeySpecException e) {
- throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
- } catch (NoSuchAlgorithmException e) {
+ } catch (GeneralSecurityException e) {
throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
}
- maxAttempts = Integer.parseInt(props.getProperty(
- "httpclient.retry-max", "3")); //$NON-NLS-1$ //$NON-NLS-2$
+ maxAttempts = Integer
+ .parseInt(props.getProperty(Keys.HTTP_RETRY, "3")); //$NON-NLS-1$
proxySelector = ProxySelector.getDefault();
- String tmp = props.getProperty("tmpdir"); //$NON-NLS-1$
+ String tmp = props.getProperty(Keys.TMP_DIR);
tmpDir = tmp != null && tmp.length() > 0 ? new File(tmp) : null;
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java
index e55b984380..e93a2af3ea 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java
@@ -47,18 +47,16 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.security.spec.InvalidKeySpecException;
+import java.security.GeneralSecurityException;
+import java.security.spec.AlgorithmParameterSpec;
import java.text.MessageFormat;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
-import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.PBEParameterSpec;
@@ -77,23 +75,29 @@ abstract class WalkEncryption {
abstract void request(HttpURLConnection u, String prefix);
- abstract void validate(HttpURLConnection u, String p) throws IOException;
+ abstract void validate(HttpURLConnection u, String prefix) throws IOException;
- protected void validateImpl(final HttpURLConnection u, final String p,
+ // TODO mixed ciphers
+ // consider permitting mixed ciphers to facilitate algorithm migration
+ // i.e. user keeps the password, but changes the algorithm
+ // then existing remote entries will still be readable
+ protected void validateImpl(final HttpURLConnection u, final String prefix,
final String version, final String name) throws IOException {
String v;
- v = u.getHeaderField(p + JETS3T_CRYPTO_VER);
+ v = u.getHeaderField(prefix + JETS3T_CRYPTO_VER);
if (v == null)
v = ""; //$NON-NLS-1$
if (!version.equals(v))
throw new IOException(MessageFormat.format(JGitText.get().unsupportedEncryptionVersion, v));
- v = u.getHeaderField(p + JETS3T_CRYPTO_ALG);
+ v = u.getHeaderField(prefix + JETS3T_CRYPTO_ALG);
if (v == null)
v = ""; //$NON-NLS-1$
- if (!name.equals(v))
- throw new IOException(JGitText.get().unsupportedEncryptionAlgorithm + v);
+ // Standard names are not case-sensitive.
+ // http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html
+ if (!name.equalsIgnoreCase(v))
+ throw new IOException(MessageFormat.format(JGitText.get().unsupportedEncryptionAlgorithm, v));
}
IOException error(final Throwable why) {
@@ -110,9 +114,9 @@ abstract class WalkEncryption {
}
@Override
- void validate(final HttpURLConnection u, final String p)
+ void validate(final HttpURLConnection u, final String prefix)
throws IOException {
- validateImpl(u, p, "", ""); //$NON-NLS-1$ //$NON-NLS-2$
+ validateImpl(u, prefix, "", ""); //$NON-NLS-1$ //$NON-NLS-2$
}
@Override
@@ -126,53 +130,132 @@ abstract class WalkEncryption {
}
}
- static class ObjectEncryptionV2 extends WalkEncryption {
- private static int ITERATION_COUNT = 5000;
+ // PBEParameterSpec factory for Java (version <= 7).
+ // Does not support AlgorithmParameterSpec.
+ static PBEParameterSpec java7PBEParameterSpec(byte[] salt,
+ int iterationCount) {
+ return new PBEParameterSpec(salt, iterationCount);
+ }
+
+ // PBEParameterSpec factory for Java (version >= 8).
+ // Adds support for AlgorithmParameterSpec.
+ static PBEParameterSpec java8PBEParameterSpec(byte[] salt,
+ int iterationCount, AlgorithmParameterSpec paramSpec) {
+ try {
+ @SuppressWarnings("boxing")
+ PBEParameterSpec instance = PBEParameterSpec.class
+ .getConstructor(byte[].class, int.class,
+ AlgorithmParameterSpec.class)
+ .newInstance(salt, iterationCount, paramSpec);
+ return instance;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Current runtime version.
+ // https://docs.oracle.com/javase/7/docs/technotes/guides/versioning/spec/versioning2.html
+ static double javaVersion() {
+ return Double.parseDouble(System.getProperty("java.specification.version")); //$NON-NLS-1$
+ }
+
+ /**
+ * JetS3t compatibility reference: <a href=
+ * "https://bitbucket.org/jmurty/jets3t/src/156c00eb160598c2e9937fd6873f00d3190e28ca/src/org/jets3t/service/security/EncryptionUtil.java">
+ * EncryptionUtil.java</a>
+ * <p>
+ * Note: EncryptionUtil is inadequate:
+ * <li>EncryptionUtil.isCipherAvailableForUse checks encryption only which
+ * "always works", but in JetS3t both encryption and decryption use non-IV
+ * aware algorithm parameters for all PBE specs, which breaks in case of AES
+ * <li>that means that only non-IV algorithms will work round trip in
+ * JetS3t, such as PBEWithMD5AndDES and PBEWithSHAAndTwofish-CBC
+ * <li>any AES based algorithms such as "PBE...With...And...AES" will not
+ * work, since they need proper IV setup
+ */
+ static class ObjectEncryptionJetS3tV2 extends WalkEncryption {
+
+ static final String JETS3T_VERSION = "2"; //$NON-NLS-1$
+
+ static final String JETS3T_ALGORITHM = "PBEWithMD5AndDES"; //$NON-NLS-1$
+
+ static final int JETS3T_ITERATIONS = 5000;
+
+ static final int JETS3T_KEY_SIZE = 32;
+
+ static final byte[] JETS3T_SALT = { //
+ (byte) 0xA4, (byte) 0x0B, (byte) 0xC8, (byte) 0x34, //
+ (byte) 0xD6, (byte) 0x95, (byte) 0xF3, (byte) 0x13 //
+ };
+
+ // Size 16, see com.sun.crypto.provider.AESConstants.AES_BLOCK_SIZE
+ static final byte[] ZERO_AES_IV = new byte[16];
- private static byte[] salt = { (byte) 0xA4, (byte) 0x0B, (byte) 0xC8,
- (byte) 0x34, (byte) 0xD6, (byte) 0x95, (byte) 0xF3, (byte) 0x13 };
+ private final String cryptoVer = JETS3T_VERSION;
- private final String algorithmName;
+ private final String cryptoAlg;
- private final SecretKey skey;
+ private final SecretKey secretKey;
- private final PBEParameterSpec aspec;
+ private final AlgorithmParameterSpec paramSpec;
- ObjectEncryptionV2(final String algo, final String key)
- throws InvalidKeySpecException, NoSuchAlgorithmException {
- algorithmName = algo;
+ ObjectEncryptionJetS3tV2(final String algo, final String key)
+ throws GeneralSecurityException {
+ cryptoAlg = algo;
+
+ // Standard names are not case-sensitive.
+ // http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html
+ String cryptoName = cryptoAlg.toUpperCase();
+
+ if (!cryptoName.startsWith("PBE")) //$NON-NLS-1$
+ throw new GeneralSecurityException(JGitText.get().encryptionOnlyPBE);
+
+ PBEKeySpec keySpec = new PBEKeySpec(key.toCharArray(), JETS3T_SALT, JETS3T_ITERATIONS, JETS3T_KEY_SIZE);
+ secretKey = SecretKeyFactory.getInstance(algo).generateSecret(keySpec);
+
+ // Detect algorithms which require initialization vector.
+ boolean useIV = cryptoName.contains("AES"); //$NON-NLS-1$
+
+ // PBEParameterSpec algorithm parameters are supported from Java 8.
+ boolean isJava8 = javaVersion() >= 1.8;
+
+ if (useIV && isJava8) {
+ // Support IV where possible:
+ // * since JCE provider uses random IV for PBE/AES
+ // * and there is no place to store dynamic IV in JetS3t V2
+ // * we use static IV, and tolerate increased security risk
+ // TODO back port this change to JetS3t V2
+ // See:
+ // https://bitbucket.org/jmurty/jets3t/raw/156c00eb160598c2e9937fd6873f00d3190e28ca/src/org/jets3t/service/security/EncryptionUtil.java
+ // http://cr.openjdk.java.net/~mullan/webrevs/ascarpin/webrev.00/raw_files/new/src/share/classes/com/sun/crypto/provider/PBES2Core.java
+ IvParameterSpec paramIV = new IvParameterSpec(ZERO_AES_IV);
+ paramSpec = java8PBEParameterSpec(JETS3T_SALT, JETS3T_ITERATIONS, paramIV);
+ } else {
+ // Strict legacy JetS3t V2 compatibility, with no IV support.
+ paramSpec = java7PBEParameterSpec(JETS3T_SALT, JETS3T_ITERATIONS);
+ }
- final PBEKeySpec s;
- s = new PBEKeySpec(key.toCharArray(), salt, ITERATION_COUNT, 32);
- skey = SecretKeyFactory.getInstance(algo).generateSecret(s);
- aspec = new PBEParameterSpec(salt, ITERATION_COUNT);
}
@Override
void request(final HttpURLConnection u, final String prefix) {
- u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, "2"); //$NON-NLS-1$
- u.setRequestProperty(prefix + JETS3T_CRYPTO_ALG, algorithmName);
+ u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, cryptoVer);
+ u.setRequestProperty(prefix + JETS3T_CRYPTO_ALG, cryptoAlg);
}
@Override
- void validate(final HttpURLConnection u, final String p)
+ void validate(final HttpURLConnection u, final String prefix)
throws IOException {
- validateImpl(u, p, "2", algorithmName); //$NON-NLS-1$
+ validateImpl(u, prefix, cryptoVer, cryptoAlg);
}
@Override
OutputStream encrypt(final OutputStream os) throws IOException {
try {
- final Cipher c = Cipher.getInstance(algorithmName);
- c.init(Cipher.ENCRYPT_MODE, skey, aspec);
- return new CipherOutputStream(os, c);
- } catch (NoSuchAlgorithmException e) {
- throw error(e);
- } catch (NoSuchPaddingException e) {
- throw error(e);
- } catch (InvalidKeyException e) {
- throw error(e);
- } catch (InvalidAlgorithmParameterException e) {
+ final Cipher cipher = Cipher.getInstance(cryptoAlg);
+ cipher.init(Cipher.ENCRYPT_MODE, secretKey, paramSpec);
+ return new CipherOutputStream(os, cipher);
+ } catch (GeneralSecurityException e) {
throw error(e);
}
}
@@ -180,16 +263,10 @@ abstract class WalkEncryption {
@Override
InputStream decrypt(final InputStream in) throws IOException {
try {
- final Cipher c = Cipher.getInstance(algorithmName);
- c.init(Cipher.DECRYPT_MODE, skey, aspec);
- return new CipherInputStream(in, c);
- } catch (NoSuchAlgorithmException e) {
- throw error(e);
- } catch (NoSuchPaddingException e) {
- throw error(e);
- } catch (InvalidKeyException e) {
- throw error(e);
- } catch (InvalidAlgorithmParameterException e) {
+ final Cipher cipher = Cipher.getInstance(cryptoAlg);
+ cipher.init(Cipher.DECRYPT_MODE, secretKey, paramSpec);
+ return new CipherInputStream(in, cipher);
+ } catch (GeneralSecurityException e) {
throw error(e);
}
}