Merge pull request #768 from nextcloud/s3-objectstore

Add S3 objectstore backend

4 files changed, 55 insertions, 131 deletions

diff --git a/apps/files/js/filelist.js b/apps/files/js/filelist.js
index d32c3ba7c9e..fb0439114ed 100644
--- a/apps/files/js/filelist.js
+++ b/apps/files/js/filelist.js
@@ -2162,7 +2162,7 @@
 
 			self.filesClient.putFileContents(
 					targetPath,
-					'',
+					' ', // dont create empty files which fails on some storage backends
 					{
 						contentType: 'text/plain',
 						overwrite: true
diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php
index 25b8e74b820..e6e26e3547a 100644
--- a/apps/files_external/lib/Lib/Storage/AmazonS3.php
+++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php
@@ -38,45 +38,30 @@ namespace OCA\Files_External\Lib\Storage;
 
 set_include_path(get_include_path() . PATH_SEPARATOR .
 	\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
-require 'aws-autoloader.php';
+require_once 'aws-autoloader.php';
 
 use Aws\S3\S3Client;
 use Aws\S3\Exception\S3Exception;
 use Icewind\Streams\IteratorDirectory;
+use OC\Files\ObjectStore\S3ConnectionTrait;
 
 class AmazonS3 extends \OC\Files\Storage\Common {
+	use S3ConnectionTrait;
 
 	/**
-	 * @var \Aws\S3\S3Client
-	 */
-	private $connection;
-	/**
-	 * @var string
-	 */
-	private $bucket;
-	/**
 	 * @var array
 	 */
 	private static $tmpFiles = array();
-	/**
-	 * @var array
-	 */
-	private $params;
-	/**
-	 * @var bool
-	 */
-	private $test = false;
-	/**
-	 * @var int
-	 */
-	private $timeout = 15;
+
 	/**
 	 * @var int in seconds
 	 */
 	private $rescanDelay = 10;
 
-	/** @var string  */
-	private $id;
+	public function __construct($parameters) {
+		parent::__construct($parameters);
+		$this->parseParams($parameters);
+	}
 
 	/**
 	 * @param string $path
@@ -92,15 +77,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $path;
 	}
 
-	/**
-	 * when running the tests wait to let the buckets catch up
-	 */
-	private function testTimeout() {
-		if ($this->test) {
-			sleep($this->timeout);
-		}
-	}
-
 	private function isRoot($path) {
 		return $path === '.';
 	}
@@ -112,26 +88,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $path;
 	}
 
-	public function __construct($params) {
-		if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
-			throw new \Exception("Access Key, Secret and Bucket have to be configured.");
-		}
-
-		$this->id = 'amazon::' . $params['bucket'];
-		$this->updateLegacyId($params);
-
-		$this->bucket = $params['bucket'];
-		$this->test = isset($params['test']);
-		$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
-		$this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay'];
-		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
-		$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
-		if (!isset($params['port']) || $params['port'] === '') {
-			$params['port'] = ($params['use_ssl'] === false) ? 80 : 443;
-		}
-		$this->params = $params;
-	}
-
 	/**
 	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
 	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
@@ -558,54 +514,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $this->id;
 	}
 
-	/**
-	 * Returns the connection
-	 *
-	 * @return S3Client connected client
-	 * @throws \Exception if connection could not be made
-	 */
-	public function getConnection() {
-		if (!is_null($this->connection)) {
-			return $this->connection;
-		}
-
-		$scheme = ($this->params['use_ssl'] === false) ? 'http' : 'https';
-		$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
-
-		$this->connection = S3Client::factory(array(
-			'key' => $this->params['key'],
-			'secret' => $this->params['secret'],
-			'base_url' => $base_url,
-			'region' => $this->params['region'],
-			S3Client::COMMAND_PARAMS => [
-				'PathStyle' => $this->params['use_path_style'],
-			],
-		));
-
-		if (!$this->connection->isValidBucketName($this->bucket)) {
-			throw new \Exception("The configured bucket name is invalid.");
-		}
-
-		if (!$this->connection->doesBucketExist($this->bucket)) {
-			try {
-				$this->connection->createBucket(array(
-					'Bucket' => $this->bucket
-				));
-				$this->connection->waitUntilBucketExists(array(
-					'Bucket' => $this->bucket,
-					'waiter.interval' => 1,
-					'waiter.max_attempts' => 15
-				));
-				$this->testTimeout();
-			} catch (S3Exception $e) {
-				\OCP\Util::logException('files_external', $e);
-				throw new \Exception('Creation of bucket failed. '.$e->getMessage());
-			}
-		}
-
-		return $this->connection;
-	}
-
 	public function writeBack($tmpFile) {
 		if (!isset(self::$tmpFiles[$tmpFile])) {
 			return false;
diff --git a/apps/theming/lib/Controller/ThemingController.php b/apps/theming/lib/Controller/ThemingController.php
index 09b4a14f2b0..1eefb2e3dd7 100644
--- a/apps/theming/lib/Controller/ThemingController.php
+++ b/apps/theming/lib/Controller/ThemingController.php
@@ -32,9 +32,12 @@ use OCP\AppFramework\Controller;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\DataDownloadResponse;
 use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\Http\NotFoundResponse;
 use OCP\AppFramework\Http\StreamResponse;
 use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\Files\File;
 use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
 use OCP\IConfig;
 use OCP\IL10N;
 use OCP\IRequest;
@@ -255,15 +258,17 @@ class ThemingController extends Controller {
 	 * @PublicPage
 	 * @NoCSRFRequired
 	 *
-	 * @return StreamResponse|DataResponse
+	 * @return StreamResponse|NotFoundResponse
 	 */
 	public function getLogo() {
-		$pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedinstancelogo';
-		if(!file_exists($pathToLogo)) {
-			return new DataResponse();
+		try {
+			/** @var File $file */
+			$file = $this->rootFolder->get('themedinstancelogo');
+		} catch (NotFoundException $e) {
+			return new NotFoundResponse();
 		}
 
-		$response = new Http\StreamResponse($pathToLogo);
+		$response = new Http\StreamResponse($file->fopen('r'));
 		$response->cacheFor(3600);
 		$response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime()));
 		$response->addHeader('Content-Disposition', 'attachment');
@@ -276,15 +281,17 @@ class ThemingController extends Controller {
 	 * @PublicPage
 	 * @NoCSRFRequired
 	 *
-	 * @return StreamResponse|DataResponse
+	 * @return StreamResponse|NotFoundResponse
 	 */
 	public function getLoginBackground() {
-		$pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedbackgroundlogo';
-		if(!file_exists($pathToLogo)) {
-			return new DataResponse();
+		try {
+			/** @var File $file */
+			$file = $this->rootFolder->get('themedbackgroundlogo');
+		} catch (NotFoundException $e) {
+			return new NotFoundResponse();
 		}
 
-		$response = new StreamResponse($pathToLogo);
+		$response = new StreamResponse($file->fopen('r'));
 		$response->cacheFor(3600);
 		$response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime()));
 		$response->addHeader('Content-Disposition', 'attachment');
diff --git a/apps/theming/tests/Controller/ThemingControllerTest.php b/apps/theming/tests/Controller/ThemingControllerTest.php
index d9d5005e25f..c9f6ff8a885 100644
--- a/apps/theming/tests/Controller/ThemingControllerTest.php
+++ b/apps/theming/tests/Controller/ThemingControllerTest.php
@@ -28,7 +28,9 @@ use OCA\Theming\Controller\ThemingController;
 use OCA\Theming\Util;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\DataResponse;
+use OCP\Files\File;
 use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
 use OCP\IConfig;
 use OCP\IL10N;
 use OCP\IRequest;
@@ -338,26 +340,30 @@ class ThemingControllerTest extends TestCase {
 	}
 
 	public function testGetLogoNotExistent() {
-		$expected = new DataResponse();
+		$this->rootFolder->method('get')
+			->with($this->equalTo('themedinstancelogo'))
+			->willThrowException(new NotFoundException());
+
+		$expected = new Http\NotFoundResponse();
 		$this->assertEquals($expected, $this->themingController->getLogo());
 	}
 
 	public function testGetLogo() {
-		$dataFolder = \OC::$server->getTempManager()->getTemporaryFolder();
-		$tmpLogo = $dataFolder . '/themedinstancelogo';
-		touch($tmpLogo);
-		$this->config
-			->expects($this->once())
-			->method('getSystemValue')
-			->with('datadirectory', \OC::$SERVERROOT . '/data/')
-			->willReturn($dataFolder);
+		$file = $this->createMock(File::class);
+		$this->rootFolder->method('get')
+			->with('themedinstancelogo')
+			->willReturn($file);
+		$file->method('fopen')
+			->with('r')
+			->willReturn('mypath');
+
 		$this->config
 			->expects($this->once())
 			->method('getAppValue')
 			->with('theming', 'logoMime', '')
 			->willReturn('text/svg');
 
-		@$expected = new Http\StreamResponse($tmpLogo);
+		@$expected = new Http\StreamResponse('mypath');
 		$expected->cacheFor(3600);
 		$expected->addHeader('Expires', date(\DateTime::RFC2822, 123));
 		$expected->addHeader('Content-Disposition', 'attachment');
@@ -368,26 +374,29 @@ class ThemingControllerTest extends TestCase {
 
 
 	public function testGetLoginBackgroundNotExistent() {
-		$expected = new DataResponse();
+		$this->rootFolder->method('get')
+			->with('themedbackgroundlogo')
+			->willThrowException(new NotFoundException());
+		$expected = new Http\NotFoundResponse();
 		$this->assertEquals($expected, $this->themingController->getLoginBackground());
 	}
 
 	public function testGetLoginBackground() {
-		$dataFolder = \OC::$server->getTempManager()->getTemporaryFolder();
-		$tmpLogo = $dataFolder . '/themedbackgroundlogo';
-		touch($tmpLogo);
-		$this->config
-			->expects($this->once())
-			->method('getSystemValue')
-			->with('datadirectory', \OC::$SERVERROOT . '/data/')
-			->willReturn($dataFolder);
+		$file = $this->createMock(File::class);
+		$this->rootFolder->method('get')
+			->with('themedbackgroundlogo')
+			->willReturn($file);
+		$file->method('fopen')
+			->with('r')
+			->willReturn('mypath');
+
 		$this->config
 			->expects($this->once())
 			->method('getAppValue')
 			->with('theming', 'backgroundMime', '')
 			->willReturn('image/png');
 
-		@$expected = new Http\StreamResponse($tmpLogo);
+		@$expected = new Http\StreamResponse('mypath');
 		$expected->cacheFor(3600);
 		$expected->addHeader('Expires', date(\DateTime::RFC2822, 123));
 		$expected->addHeader('Content-Disposition', 'attachment');