Merge pull request #4410 from nextcloud/update-aws-sdk

Update aws sdk + s3 improvements

5 files changed, 96 insertions, 82 deletions

diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 425eeb6ce7b..e88471f9461 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -608,6 +608,7 @@ return array(
     'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
     'OC\\Files\\ObjectStore\\S3' => $baseDir . '/lib/private/Files/ObjectStore/S3.php',
     'OC\\Files\\ObjectStore\\S3ConnectionTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',
+    'OC\\Files\\ObjectStore\\S3ObjectTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ObjectTrait.php',
     'OC\\Files\\ObjectStore\\StorageObjectStore' => $baseDir . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
     'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
     'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 68262fafb0d..17c9c8129da 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -638,6 +638,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
         'OC\\Files\\ObjectStore\\S3' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3.php',
         'OC\\Files\\ObjectStore\\S3ConnectionTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',
+        'OC\\Files\\ObjectStore\\S3ObjectTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ObjectTrait.php',
         'OC\\Files\\ObjectStore\\StorageObjectStore' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
         'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
         'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
index 6f37492271c..e4a7b068b65 100644
--- a/lib/private/Files/ObjectStore/S3.php
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -23,13 +23,9 @@ namespace OC\Files\ObjectStore;
 
 use OCP\Files\ObjectStore\IObjectStore;
 
-// TODO: proper composer
-set_include_path(get_include_path() . PATH_SEPARATOR .
-	\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
-require_once 'aws-autoloader.php';
-
 class S3 implements IObjectStore {
 	use S3ConnectionTrait;
+	use S3ObjectTrait;
 
 	public function __construct($parameters) {
 		$this->parseParams($parameters);
@@ -42,66 +38,4 @@ class S3 implements IObjectStore {
 	public function getStorageId() {
 		return $this->id;
 	}
-
-	/**
-	 * @param string $urn the unified resource name used to identify the object
-	 * @return resource stream with the read data
-	 * @throws \Exception when something goes wrong, message will be logged
-	 * @since 7.0.0
-	 */
-	public function readObject($urn) {
-		// Create the command and serialize the request
-		$request = $this->getConnection()->getCommand('GetObject', [
-			'Bucket' => $this->bucket,
-			'Key' => $urn
-		])->prepare();
-
-		$request->dispatch('request.before_send', array(
-			'request' => $request
-		));
-
-		$headers = $request->getHeaderLines();
-		$headers[] = 'Connection: close';
-
-		$opts = [
-			'http' => [
-				'method' => "GET",
-				'header' => $headers
-			],
-			'ssl' => [
-				'verify_peer' => true
-			]
-		];
-
-		$context = stream_context_create($opts);
-		return fopen($request->getUrl(), 'r', false, $context);
-	}
-
-	/**
-	 * @param string $urn the unified resource name used to identify the object
-	 * @param resource $stream stream with the data to write
-	 * @throws \Exception when something goes wrong, message will be logged
-	 * @since 7.0.0
-	 */
-	public function writeObject($urn, $stream) {
-		$this->getConnection()->putObject([
-			'Bucket' => $this->bucket,
-			'Key' => $urn,
-			'Body' => $stream
-		]);
-	}
-
-	/**
-	 * @param string $urn the unified resource name used to identify the object
-	 * @return void
-	 * @throws \Exception when something goes wrong, message will be logged
-	 * @since 7.0.0
-	 */
-	public function deleteObject($urn) {
-		$this->getConnection()->deleteObject([
-			'Bucket' => $this->bucket,
-			'Key' => $urn
-		]);
-	}
-
 }
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
index a8b57cb18d3..fdda19ff700 100644
--- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -53,7 +53,7 @@ trait S3ConnectionTrait {
 		$this->bucket = $params['bucket'];
 		$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
 		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
-		$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
+		$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
 		if (!isset($params['port']) || $params['port'] === '') {
 			$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
 		}
@@ -76,20 +76,21 @@ trait S3ConnectionTrait {
 		$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
 
 		$options = [
-			'key' => $this->params['key'],
-			'secret' => $this->params['secret'],
-			'base_url' => $base_url,
+			'version' => isset($this->params['version']) ? $this->params['version'] : 'latest',
+			'credentials' => [
+				'key' => $this->params['key'],
+				'secret' => $this->params['secret'],
+			],
+			'endpoint' => $base_url,
 			'region' => $this->params['region'],
-			S3Client::COMMAND_PARAMS => [
-				'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
-			]
+			'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false
 		];
 		if (isset($this->params['proxy'])) {
-			$options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']];
+			$options['request.options'] = ['proxy' => $this->params['proxy']];
 		}
-		$this->connection = S3Client::factory($options);
+		$this->connection = new S3Client($options);
 
-		if (!$this->connection->isValidBucketName($this->bucket)) {
+		if (!S3Client::isBucketDnsCompatible($this->bucket)) {
 			throw new \Exception("The configured bucket name is invalid.");
 		}
 
@@ -98,11 +99,6 @@ trait S3ConnectionTrait {
 				$this->connection->createBucket(array(
 					'Bucket' => $this->bucket
 				));
-				$this->connection->waitUntilBucketExists(array(
-					'Bucket' => $this->bucket,
-					'waiter.interval' => 1,
-					'waiter.max_attempts' => 15
-				));
 				$this->testTimeout();
 			} catch (S3Exception $e) {
 				\OCP\Util::logException('files_external', $e);
diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
new file mode 100644
index 00000000000..3ba4da92b98
--- /dev/null
+++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -0,0 +1,82 @@
+<?php
+/**
+ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+use Aws\S3\S3Client;
+use Psr\Http\Message\StreamInterface;
+
+trait S3ObjectTrait {
+	/**
+	 * Returns the connection
+	 *
+	 * @return S3Client connected client
+	 * @throws \Exception if connection could not be made
+	 */
+	abstract protected function getConnection();
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return resource stream with the read data
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function readObject($urn) {
+		$client = $this->getConnection();
+		$command = $client->getCommand('GetObject', [
+			'Bucket' => $this->bucket,
+			'Key' => $urn
+		]);
+		$command['@http']['stream'] = true;
+		$result = $client->execute($command);
+		/** @var StreamInterface $body */
+		$body = $result['Body'];
+
+		return $body->detach();
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @param resource $stream stream with the data to write
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function writeObject($urn, $stream) {
+		$this->getConnection()->putObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn,
+			'Body' => $stream
+		]);
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return void
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function deleteObject($urn) {
+		$this->getConnection()->deleteObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn
+		]);
+	}
+}
\ No newline at end of file