Merge pull request #768 from nextcloud/s3-objectstore

Add S3 objectstore backend

8 files changed, 338 insertions, 7 deletions

diff --git a/lib/private/AppFramework/Http/Output.php b/lib/private/AppFramework/Http/Output.php
index 85f0e6f8feb..1d77350b1a2 100644
--- a/lib/private/AppFramework/Http/Output.php
+++ b/lib/private/AppFramework/Http/Output.php
@@ -48,12 +48,17 @@ class Output implements IOutput {
 	}
 
 	/**
-	 * @param string $path
+	 * @param string|resource $path or file handle
 	 *
 	 * @return bool false if an error occurred
 	 */
 	public function setReadfile($path) {
-		return @readfile($path);
+		if (is_resource($path)) {
+			$output = fopen('php://output', 'w');
+			return stream_copy_to_stream($path, $output) > 0;
+		} else {
+			return @readfile($path);
+		}
 	}
 
 	/**
diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php
index 14a32ba8f76..3ad2932e597 100644
--- a/lib/private/Files/FileInfo.php
+++ b/lib/private/Files/FileInfo.php
@@ -33,6 +33,7 @@ namespace OC\Files;
 use OCP\Files\Cache\ICacheEntry;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\Storage\IStorage;
+use OCP\Files\IHomeStorage;
 use OCP\IUser;
 
 class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
@@ -305,7 +306,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	}
 
 	public function isMounted() {
-		$sid = $this->getStorage()->getId();
+		$storage = $this->getStorage();
+		if ($storage->instanceOfStorage('\OCP\Files\IHomeStorage')) {
+			return false;
+		}
+		$sid = $storage->getId();
 		if (!is_null($sid)) {
 			$sid = explode(':', $sid);
 			return ($sid[0] !== 'home' and $sid[0] !== 'shared');
diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php
index 87878562a42..440a8bc4608 100644
--- a/lib/private/Files/Mount/ObjectHomeMountProvider.php
+++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php
@@ -51,7 +51,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider {
 	 *
 	 * @param IUser $user
 	 * @param IStorageFactory $loader
-	 * @return \OCP\Files\Mount\IMountPoint[]
+	 * @return \OCP\Files\Mount\IMountPoint
 	 */
 	public function getHomeMountForUser(IUser $user, IStorageFactory $loader) {
 
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
new file mode 100644
index 00000000000..5251b473bdf
--- /dev/null
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -0,0 +1,107 @@
+<?php
+/**
+ * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+use OCP\Files\ObjectStore\IObjectStore;
+
+// TODO: proper composer
+set_include_path(get_include_path() . PATH_SEPARATOR .
+	\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
+require_once 'aws-autoloader.php';
+
+class S3 implements IObjectStore {
+	use S3ConnectionTrait;
+
+	public function __construct($parameters) {
+		$this->parseParams($parameters);
+	}
+
+	/**
+	 * @return string the container or bucket name where objects are stored
+	 * @since 7.0.0
+	 */
+	function getStorageId() {
+		return $this->id;
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return resource stream with the read data
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function readObject($urn) {
+		// Create the command and serialize the request
+		$request = $this->getConnection()->getCommand('GetObject', [
+			'Bucket' => $this->bucket,
+			'Key' => $urn
+		])->prepare();
+
+		$request->dispatch('request.before_send', array(
+			'request' => $request
+		));
+
+		$headers = $request->getHeaderLines();
+		$headers[] = 'Connection: close';
+
+		$opts = [
+			'http' => [
+				'method' => "GET",
+				'header' => $headers
+			],
+			'ssl' => [
+				'verify_peer' => true
+			]
+		];
+
+		$context = stream_context_create($opts);
+		return fopen($request->getUrl(), 'r', false, $context);
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @param resource $stream stream with the data to write
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function writeObject($urn, $stream) {
+		$this->getConnection()->putObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn,
+			'Body' => $stream
+		]);
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return void
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function deleteObject($urn) {
+		$this->getConnection()->deleteObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn
+		]);
+	}
+
+}
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
new file mode 100644
index 00000000000..a8b57cb18d3
--- /dev/null
+++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -0,0 +1,124 @@
+<?php
+/**
+ * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+use Aws\S3\Exception\S3Exception;
+use Aws\S3\S3Client;
+
+trait S3ConnectionTrait {
+	/** @var array */
+	protected $params;
+
+	/** @var S3Client */
+	protected $connection;
+
+	/** @var string */
+	protected $id;
+
+	/** @var string */
+	protected $bucket;
+
+	/** @var int */
+	protected $timeout;
+
+	protected $test;
+
+	protected function parseParams($params) {
+		if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
+			throw new \Exception("Access Key, Secret and Bucket have to be configured.");
+		}
+
+		$this->id = 'amazon::' . $params['bucket'];
+
+		$this->test = isset($params['test']);
+		$this->bucket = $params['bucket'];
+		$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
+		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
+		$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
+		if (!isset($params['port']) || $params['port'] === '') {
+			$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
+		}
+		$this->params = $params;
+	}
+
+
+	/**
+	 * Returns the connection
+	 *
+	 * @return S3Client connected client
+	 * @throws \Exception if connection could not be made
+	 */
+	protected function getConnection() {
+		if (!is_null($this->connection)) {
+			return $this->connection;
+		}
+
+		$scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
+		$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
+
+		$options = [
+			'key' => $this->params['key'],
+			'secret' => $this->params['secret'],
+			'base_url' => $base_url,
+			'region' => $this->params['region'],
+			S3Client::COMMAND_PARAMS => [
+				'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
+			]
+		];
+		if (isset($this->params['proxy'])) {
+			$options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']];
+		}
+		$this->connection = S3Client::factory($options);
+
+		if (!$this->connection->isValidBucketName($this->bucket)) {
+			throw new \Exception("The configured bucket name is invalid.");
+		}
+
+		if (!$this->connection->doesBucketExist($this->bucket)) {
+			try {
+				$this->connection->createBucket(array(
+					'Bucket' => $this->bucket
+				));
+				$this->connection->waitUntilBucketExists(array(
+					'Bucket' => $this->bucket,
+					'waiter.interval' => 1,
+					'waiter.max_attempts' => 15
+				));
+				$this->testTimeout();
+			} catch (S3Exception $e) {
+				\OCP\Util::logException('files_external', $e);
+				throw new \Exception('Creation of bucket failed. ' . $e->getMessage());
+			}
+		}
+
+		return $this->connection;
+	}
+
+	/**
+	 * when running the tests wait to let the buckets catch up
+	 */
+	private function testTimeout() {
+		if ($this->test) {
+			sleep($this->timeout);
+		}
+	}
+}
diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php
new file mode 100644
index 00000000000..044243c1ee6
--- /dev/null
+++ b/lib/private/Files/ObjectStore/StorageObjectStore.php
@@ -0,0 +1,90 @@
+<?php
+/**
+ * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+use OCP\Files\ObjectStore\IObjectStore;
+use OCP\Files\Storage\IStorage;
+
+/**
+ * Object store that wraps a storage backend, mostly for testing purposes
+ */
+class StorageObjectStore implements IObjectStore {
+	/** @var IStorage */
+	private $storage;
+
+	/**
+	 * @param IStorage $storage
+	 */
+	public function __construct(IStorage $storage) {
+		$this->storage = $storage;
+	}
+
+	/**
+	 * @return string the container or bucket name where objects are stored
+	 * @since 7.0.0
+	 */
+	function getStorageId() {
+		$this->storage->getId();
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return resource stream with the read data
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function readObject($urn) {
+		$handle = $this->storage->fopen($urn, 'r');
+		if ($handle) {
+			return $handle;
+		} else {
+			throw new \Exception();
+		}
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @param resource $stream stream with the data to write
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function writeObject($urn, $stream) {
+		$handle = $this->storage->fopen($urn, 'w');
+		if ($handle) {
+			stream_copy_to_stream($stream, $handle);
+			fclose($handle);
+		} else {
+			throw new \Exception();
+		}
+	}
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @return void
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	function deleteObject($urn) {
+		$this->storage->unlink($urn);
+	}
+
+}
diff --git a/lib/public/AppFramework/Http/IOutput.php b/lib/public/AppFramework/Http/IOutput.php
index 1544c7d6375..642bcf4170f 100644
--- a/lib/public/AppFramework/Http/IOutput.php
+++ b/lib/public/AppFramework/Http/IOutput.php
@@ -39,7 +39,7 @@ interface IOutput {
 	public function setOutput($out);
 
 	/**
-	 * @param string $path
+	 * @param string|resource $path or file handle
 	 *
 	 * @return bool false if an error occurred
 	 * @since 8.1.0
diff --git a/lib/public/AppFramework/Http/StreamResponse.php b/lib/public/AppFramework/Http/StreamResponse.php
index b5852fb5620..e124bb4ccbf 100644
--- a/lib/public/AppFramework/Http/StreamResponse.php
+++ b/lib/public/AppFramework/Http/StreamResponse.php
@@ -37,7 +37,7 @@ class StreamResponse extends Response implements ICallbackResponse {
 	private $filePath;
 
 	/**
-	 * @param string $filePath the path to the file which should be streamed
+	 * @param string|resource $filePath the path to the file or a file handle which should be streamed
 	 * @since 8.1.0
 	 */
 	public function __construct ($filePath) {
@@ -54,7 +54,7 @@ class StreamResponse extends Response implements ICallbackResponse {
 	public function callback (IOutput $output) {
 		// handle caching
 		if ($output->getHttpResponseCode() !== Http::STATUS_NOT_MODIFIED) {
-			if (!file_exists($this->filePath)) {
+			if (!(file_exists($this->filePath) || is_resource($this->filePath))) {
 				$output->setHttpResponseCode(Http::STATUS_NOT_FOUND);
 			} elseif ($output->setReadfile($this->filePath) === false) {
 				$output->setHttpResponseCode(Http::STATUS_BAD_REQUEST);