diff options
author | Robin Appelman <robin@icewind.nl> | 2016-11-18 14:55:07 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-11-18 14:55:07 +0100 |
commit | 8b9ad46ba3ef876f00dc6bdadd7b51d7b1fa1c78 (patch) | |
tree | bc9c6a21ddc68656d8fd93c67f11b754a358afdb | |
parent | 6c11c54434b29bb9f7f07ba9a8070ed8308bc5a4 (diff) | |
parent | 4ac5fdcf11b0ca7dd985d50a91393a1c185821ff (diff) | |
download | nextcloud-server-8b9ad46ba3ef876f00dc6bdadd7b51d7b1fa1c78.tar.gz nextcloud-server-8b9ad46ba3ef876f00dc6bdadd7b51d7b1fa1c78.zip |
Merge pull request #768 from nextcloud/s3-objectstore
Add S3 objectstore backend
19 files changed, 827 insertions, 315 deletions
diff --git a/apps/files/js/filelist.js b/apps/files/js/filelist.js index d32c3ba7c9e..fb0439114ed 100644 --- a/apps/files/js/filelist.js +++ b/apps/files/js/filelist.js @@ -2162,7 +2162,7 @@ self.filesClient.putFileContents( targetPath, - '', + ' ', // dont create empty files which fails on some storage backends { contentType: 'text/plain', overwrite: true diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index 25b8e74b820..e6e26e3547a 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -38,45 +38,30 @@ namespace OCA\Files_External\Lib\Storage; set_include_path(get_include_path() . PATH_SEPARATOR . \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php'); -require 'aws-autoloader.php'; +require_once 'aws-autoloader.php'; use Aws\S3\S3Client; use Aws\S3\Exception\S3Exception; use Icewind\Streams\IteratorDirectory; +use OC\Files\ObjectStore\S3ConnectionTrait; class AmazonS3 extends \OC\Files\Storage\Common { + use S3ConnectionTrait; /** - * @var \Aws\S3\S3Client - */ - private $connection; - /** - * @var string - */ - private $bucket; - /** * @var array */ private static $tmpFiles = array(); - /** - * @var array - */ - private $params; - /** - * @var bool - */ - private $test = false; - /** - * @var int - */ - private $timeout = 15; + /** * @var int in seconds */ private $rescanDelay = 10; - /** @var string */ - private $id; + public function __construct($parameters) { + parent::__construct($parameters); + $this->parseParams($parameters); + } /** * @param string $path @@ -92,15 +77,6 @@ class AmazonS3 extends \OC\Files\Storage\Common { return $path; } - /** - * when running the tests wait to let the buckets catch up - */ - private function testTimeout() { - if ($this->test) { - sleep($this->timeout); - } - } - private function isRoot($path) { return $path === '.'; } @@ -112,26 +88,6 @@ class AmazonS3 extends \OC\Files\Storage\Common { return $path; } - public function __construct($params) { - if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) { - throw new \Exception("Access Key, Secret and Bucket have to be configured."); - } - - $this->id = 'amazon::' . $params['bucket']; - $this->updateLegacyId($params); - - $this->bucket = $params['bucket']; - $this->test = isset($params['test']); - $this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout']; - $this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay']; - $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; - $params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname']; - if (!isset($params['port']) || $params['port'] === '') { - $params['port'] = ($params['use_ssl'] === false) ? 80 : 443; - } - $this->params = $params; - } - /** * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name. * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home @@ -558,54 +514,6 @@ class AmazonS3 extends \OC\Files\Storage\Common { return $this->id; } - /** - * Returns the connection - * - * @return S3Client connected client - * @throws \Exception if connection could not be made - */ - public function getConnection() { - if (!is_null($this->connection)) { - return $this->connection; - } - - $scheme = ($this->params['use_ssl'] === false) ? 'http' : 'https'; - $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; - - $this->connection = S3Client::factory(array( - 'key' => $this->params['key'], - 'secret' => $this->params['secret'], - 'base_url' => $base_url, - 'region' => $this->params['region'], - S3Client::COMMAND_PARAMS => [ - 'PathStyle' => $this->params['use_path_style'], - ], - )); - - if (!$this->connection->isValidBucketName($this->bucket)) { - throw new \Exception("The configured bucket name is invalid."); - } - - if (!$this->connection->doesBucketExist($this->bucket)) { - try { - $this->connection->createBucket(array( - 'Bucket' => $this->bucket - )); - $this->connection->waitUntilBucketExists(array( - 'Bucket' => $this->bucket, - 'waiter.interval' => 1, - 'waiter.max_attempts' => 15 - )); - $this->testTimeout(); - } catch (S3Exception $e) { - \OCP\Util::logException('files_external', $e); - throw new \Exception('Creation of bucket failed. '.$e->getMessage()); - } - } - - return $this->connection; - } - public function writeBack($tmpFile) { if (!isset(self::$tmpFiles[$tmpFile])) { return false; diff --git a/apps/theming/lib/Controller/ThemingController.php b/apps/theming/lib/Controller/ThemingController.php index 09b4a14f2b0..1eefb2e3dd7 100644 --- a/apps/theming/lib/Controller/ThemingController.php +++ b/apps/theming/lib/Controller/ThemingController.php @@ -32,9 +32,12 @@ use OCP\AppFramework\Controller; use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataDownloadResponse; use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\Http\NotFoundResponse; use OCP\AppFramework\Http\StreamResponse; use OCP\AppFramework\Utility\ITimeFactory; +use OCP\Files\File; use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IConfig; use OCP\IL10N; use OCP\IRequest; @@ -255,15 +258,17 @@ class ThemingController extends Controller { * @PublicPage * @NoCSRFRequired * - * @return StreamResponse|DataResponse + * @return StreamResponse|NotFoundResponse */ public function getLogo() { - $pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedinstancelogo'; - if(!file_exists($pathToLogo)) { - return new DataResponse(); + try { + /** @var File $file */ + $file = $this->rootFolder->get('themedinstancelogo'); + } catch (NotFoundException $e) { + return new NotFoundResponse(); } - $response = new Http\StreamResponse($pathToLogo); + $response = new Http\StreamResponse($file->fopen('r')); $response->cacheFor(3600); $response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime())); $response->addHeader('Content-Disposition', 'attachment'); @@ -276,15 +281,17 @@ class ThemingController extends Controller { * @PublicPage * @NoCSRFRequired * - * @return StreamResponse|DataResponse + * @return StreamResponse|NotFoundResponse */ public function getLoginBackground() { - $pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedbackgroundlogo'; - if(!file_exists($pathToLogo)) { - return new DataResponse(); + try { + /** @var File $file */ + $file = $this->rootFolder->get('themedbackgroundlogo'); + } catch (NotFoundException $e) { + return new NotFoundResponse(); } - $response = new StreamResponse($pathToLogo); + $response = new StreamResponse($file->fopen('r')); $response->cacheFor(3600); $response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime())); $response->addHeader('Content-Disposition', 'attachment'); diff --git a/apps/theming/tests/Controller/ThemingControllerTest.php b/apps/theming/tests/Controller/ThemingControllerTest.php index d9d5005e25f..c9f6ff8a885 100644 --- a/apps/theming/tests/Controller/ThemingControllerTest.php +++ b/apps/theming/tests/Controller/ThemingControllerTest.php @@ -28,7 +28,9 @@ use OCA\Theming\Controller\ThemingController; use OCA\Theming\Util; use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataResponse; +use OCP\Files\File; use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IConfig; use OCP\IL10N; use OCP\IRequest; @@ -338,26 +340,30 @@ class ThemingControllerTest extends TestCase { } public function testGetLogoNotExistent() { - $expected = new DataResponse(); + $this->rootFolder->method('get') + ->with($this->equalTo('themedinstancelogo')) + ->willThrowException(new NotFoundException()); + + $expected = new Http\NotFoundResponse(); $this->assertEquals($expected, $this->themingController->getLogo()); } public function testGetLogo() { - $dataFolder = \OC::$server->getTempManager()->getTemporaryFolder(); - $tmpLogo = $dataFolder . '/themedinstancelogo'; - touch($tmpLogo); - $this->config - ->expects($this->once()) - ->method('getSystemValue') - ->with('datadirectory', \OC::$SERVERROOT . '/data/') - ->willReturn($dataFolder); + $file = $this->createMock(File::class); + $this->rootFolder->method('get') + ->with('themedinstancelogo') + ->willReturn($file); + $file->method('fopen') + ->with('r') + ->willReturn('mypath'); + $this->config ->expects($this->once()) ->method('getAppValue') ->with('theming', 'logoMime', '') ->willReturn('text/svg'); - @$expected = new Http\StreamResponse($tmpLogo); + @$expected = new Http\StreamResponse('mypath'); $expected->cacheFor(3600); $expected->addHeader('Expires', date(\DateTime::RFC2822, 123)); $expected->addHeader('Content-Disposition', 'attachment'); @@ -368,26 +374,29 @@ class ThemingControllerTest extends TestCase { public function testGetLoginBackgroundNotExistent() { - $expected = new DataResponse(); + $this->rootFolder->method('get') + ->with('themedbackgroundlogo') + ->willThrowException(new NotFoundException()); + $expected = new Http\NotFoundResponse(); $this->assertEquals($expected, $this->themingController->getLoginBackground()); } public function testGetLoginBackground() { - $dataFolder = \OC::$server->getTempManager()->getTemporaryFolder(); - $tmpLogo = $dataFolder . '/themedbackgroundlogo'; - touch($tmpLogo); - $this->config - ->expects($this->once()) - ->method('getSystemValue') - ->with('datadirectory', \OC::$SERVERROOT . '/data/') - ->willReturn($dataFolder); + $file = $this->createMock(File::class); + $this->rootFolder->method('get') + ->with('themedbackgroundlogo') + ->willReturn($file); + $file->method('fopen') + ->with('r') + ->willReturn('mypath'); + $this->config ->expects($this->once()) ->method('getAppValue') ->with('theming', 'backgroundMime', '') ->willReturn('image/png'); - @$expected = new Http\StreamResponse($tmpLogo); + @$expected = new Http\StreamResponse('mypath'); $expected->cacheFor(3600); $expected->addHeader('Expires', date(\DateTime::RFC2822, 123)); $expected->addHeader('Content-Disposition', 'attachment'); diff --git a/lib/private/AppFramework/Http/Output.php b/lib/private/AppFramework/Http/Output.php index 85f0e6f8feb..1d77350b1a2 100644 --- a/lib/private/AppFramework/Http/Output.php +++ b/lib/private/AppFramework/Http/Output.php @@ -48,12 +48,17 @@ class Output implements IOutput { } /** - * @param string $path + * @param string|resource $path or file handle * * @return bool false if an error occurred */ public function setReadfile($path) { - return @readfile($path); + if (is_resource($path)) { + $output = fopen('php://output', 'w'); + return stream_copy_to_stream($path, $output) > 0; + } else { + return @readfile($path); + } } /** diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 14a32ba8f76..3ad2932e597 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -33,6 +33,7 @@ namespace OC\Files; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage\IStorage; +use OCP\Files\IHomeStorage; use OCP\IUser; class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { @@ -305,7 +306,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { } public function isMounted() { - $sid = $this->getStorage()->getId(); + $storage = $this->getStorage(); + if ($storage->instanceOfStorage('\OCP\Files\IHomeStorage')) { + return false; + } + $sid = $storage->getId(); if (!is_null($sid)) { $sid = explode(':', $sid); return ($sid[0] !== 'home' and $sid[0] !== 'shared'); diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 87878562a42..440a8bc4608 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -51,7 +51,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { * * @param IUser $user * @param IStorageFactory $loader - * @return \OCP\Files\Mount\IMountPoint[] + * @return \OCP\Files\Mount\IMountPoint */ public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php new file mode 100644 index 00000000000..5251b473bdf --- /dev/null +++ b/lib/private/Files/ObjectStore/S3.php @@ -0,0 +1,107 @@ +<?php +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\ObjectStore; + +use OCP\Files\ObjectStore\IObjectStore; + +// TODO: proper composer +set_include_path(get_include_path() . PATH_SEPARATOR . + \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php'); +require_once 'aws-autoloader.php'; + +class S3 implements IObjectStore { + use S3ConnectionTrait; + + public function __construct($parameters) { + $this->parseParams($parameters); + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + function getStorageId() { + return $this->id; + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + // Create the command and serialize the request + $request = $this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn + ])->prepare(); + + $request->dispatch('request.before_send', array( + 'request' => $request + )); + + $headers = $request->getHeaderLines(); + $headers[] = 'Connection: close'; + + $opts = [ + 'http' => [ + 'method' => "GET", + 'header' => $headers + ], + 'ssl' => [ + 'verify_peer' => true + ] + ]; + + $context = stream_context_create($opts); + return fopen($request->getUrl(), 'r', false, $context); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $this->getConnection()->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream + ]); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->getConnection()->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ]); + } + +} diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php new file mode 100644 index 00000000000..a8b57cb18d3 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -0,0 +1,124 @@ +<?php +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\ObjectStore; + +use Aws\S3\Exception\S3Exception; +use Aws\S3\S3Client; + +trait S3ConnectionTrait { + /** @var array */ + protected $params; + + /** @var S3Client */ + protected $connection; + + /** @var string */ + protected $id; + + /** @var string */ + protected $bucket; + + /** @var int */ + protected $timeout; + + protected $test; + + protected function parseParams($params) { + if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) { + throw new \Exception("Access Key, Secret and Bucket have to be configured."); + } + + $this->id = 'amazon::' . $params['bucket']; + + $this->test = isset($params['test']); + $this->bucket = $params['bucket']; + $this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout']; + $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; + $params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname']; + if (!isset($params['port']) || $params['port'] === '') { + $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; + } + $this->params = $params; + } + + + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ + protected function getConnection() { + if (!is_null($this->connection)) { + return $this->connection; + } + + $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; + $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; + + $options = [ + 'key' => $this->params['key'], + 'secret' => $this->params['secret'], + 'base_url' => $base_url, + 'region' => $this->params['region'], + S3Client::COMMAND_PARAMS => [ + 'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, + ] + ]; + if (isset($this->params['proxy'])) { + $options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']]; + } + $this->connection = S3Client::factory($options); + + if (!$this->connection->isValidBucketName($this->bucket)) { + throw new \Exception("The configured bucket name is invalid."); + } + + if (!$this->connection->doesBucketExist($this->bucket)) { + try { + $this->connection->createBucket(array( + 'Bucket' => $this->bucket + )); + $this->connection->waitUntilBucketExists(array( + 'Bucket' => $this->bucket, + 'waiter.interval' => 1, + 'waiter.max_attempts' => 15 + )); + $this->testTimeout(); + } catch (S3Exception $e) { + \OCP\Util::logException('files_external', $e); + throw new \Exception('Creation of bucket failed. ' . $e->getMessage()); + } + } + + return $this->connection; + } + + /** + * when running the tests wait to let the buckets catch up + */ + private function testTimeout() { + if ($this->test) { + sleep($this->timeout); + } + } +} diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php new file mode 100644 index 00000000000..044243c1ee6 --- /dev/null +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -0,0 +1,90 @@ +<?php +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\ObjectStore; + +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\Storage\IStorage; + +/** + * Object store that wraps a storage backend, mostly for testing purposes + */ +class StorageObjectStore implements IObjectStore { + /** @var IStorage */ + private $storage; + + /** + * @param IStorage $storage + */ + public function __construct(IStorage $storage) { + $this->storage = $storage; + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + function getStorageId() { + $this->storage->getId(); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + $handle = $this->storage->fopen($urn, 'r'); + if ($handle) { + return $handle; + } else { + throw new \Exception(); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $handle = $this->storage->fopen($urn, 'w'); + if ($handle) { + stream_copy_to_stream($stream, $handle); + fclose($handle); + } else { + throw new \Exception(); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->storage->unlink($urn); + } + +} diff --git a/lib/public/AppFramework/Http/IOutput.php b/lib/public/AppFramework/Http/IOutput.php index 1544c7d6375..642bcf4170f 100644 --- a/lib/public/AppFramework/Http/IOutput.php +++ b/lib/public/AppFramework/Http/IOutput.php @@ -39,7 +39,7 @@ interface IOutput { public function setOutput($out); /** - * @param string $path + * @param string|resource $path or file handle * * @return bool false if an error occurred * @since 8.1.0 diff --git a/lib/public/AppFramework/Http/StreamResponse.php b/lib/public/AppFramework/Http/StreamResponse.php index b5852fb5620..e124bb4ccbf 100644 --- a/lib/public/AppFramework/Http/StreamResponse.php +++ b/lib/public/AppFramework/Http/StreamResponse.php @@ -37,7 +37,7 @@ class StreamResponse extends Response implements ICallbackResponse { private $filePath; /** - * @param string $filePath the path to the file which should be streamed + * @param string|resource $filePath the path to the file or a file handle which should be streamed * @since 8.1.0 */ public function __construct ($filePath) { @@ -54,7 +54,7 @@ class StreamResponse extends Response implements ICallbackResponse { public function callback (IOutput $output) { // handle caching if ($output->getHttpResponseCode() !== Http::STATUS_NOT_MODIFIED) { - if (!file_exists($this->filePath)) { + if (!(file_exists($this->filePath) || is_resource($this->filePath))) { $output->setHttpResponseCode(Http::STATUS_NOT_FOUND); } elseif ($output->setReadfile($this->filePath) === false) { $output->setHttpResponseCode(Http::STATUS_BAD_REQUEST); diff --git a/tests/lib/AppFramework/Http/OutputTest.php b/tests/lib/AppFramework/Http/OutputTest.php new file mode 100644 index 00000000000..5fe35d24bde --- /dev/null +++ b/tests/lib/AppFramework/Http/OutputTest.php @@ -0,0 +1,31 @@ +<?php +/** + * Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\AppFramework\Http; + +use OC\AppFramework\Http\Output; + +class OutputTest extends \Test\TestCase { + public function testSetOutput() { + $this->expectOutputString('foo'); + $output = new Output(''); + $output->setOutput('foo'); + } + + public function testSetReadfile() { + $this->expectOutputString(file_get_contents(__FILE__)); + $output = new Output(''); + $output->setReadfile(__FILE__); + } + + public function testSetReadfileStream() { + $this->expectOutputString(file_get_contents(__FILE__)); + $output = new Output(''); + $output->setReadfile(fopen(__FILE__, 'r')); + } +} diff --git a/tests/lib/Files/FileInfoTest.php b/tests/lib/Files/FileInfoTest.php new file mode 100644 index 00000000000..ee7a10ccec4 --- /dev/null +++ b/tests/lib/Files/FileInfoTest.php @@ -0,0 +1,46 @@ +<?php +/** + * Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\Files; + +use OC\AllConfig; +use OC\Files\FileInfo; +use OC\Files\Storage\Home; +use OC\Files\Storage\Temporary; +use OC\User\User; +use OCP\IConfig; +use Test\TestCase; +use Test\Traits\UserTrait; + +class FileInfoTest extends TestCase { + use UserTrait; + + private $config; + + public function setUp() { + parent::setUp(); + $this->createUser('foo', 'foo'); + $this->config = $this->getMockBuilder(IConfig::class)->getMock(); + } + + public function testIsMountedHomeStorage() { + $fileInfo = new FileInfo( + '', + new Home(['user' => new User('foo', $this->userBackend, null, $this->config)]), + '', [], null); + $this->assertFalse($fileInfo->isMounted()); + } + + public function testIsMountedNonHomeStorage() { + $fileInfo = new FileInfo( + '', + new Temporary(), + '', [], null); + $this->assertTrue($fileInfo->isMounted()); + } +} diff --git a/tests/lib/Files/ObjectStore/LocalTest.php b/tests/lib/Files/ObjectStore/LocalTest.php new file mode 100644 index 00000000000..99aa23e065d --- /dev/null +++ b/tests/lib/Files/ObjectStore/LocalTest.php @@ -0,0 +1,36 @@ +<?php +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\StorageObjectStore; +use OC\Files\Storage\Temporary; + +class LocalTest extends ObjectStoreTest { + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + protected function getInstance() { + $storage = new Temporary(); + return new StorageObjectStore($storage); + } + +} diff --git a/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php b/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php new file mode 100644 index 00000000000..c9d6c1bd922 --- /dev/null +++ b/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php @@ -0,0 +1,169 @@ +<?php +/** + * @author Jörn Friedrich Dreyer + * @copyright (c) 2014 Jörn Friedrich Dreyer <jfd@owncloud.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE + * License as published by the Free Software Foundation; either + * version 3 of the License, or any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU AFFERO GENERAL PUBLIC LICENSE for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with this library. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\ObjectStore\StorageObjectStore; +use OC\Files\Storage\Temporary; +use OCP\Files\ObjectStore\IObjectStore; +use Test\Files\Storage\Storage; + +/** + * @group DB + */ +class ObjectStoreStorageTest extends Storage { + + /** + * @var IObjectStore + */ + private $objectStorage; + + protected function setUp() { + parent::setUp(); + + $baseStorage = new Temporary(); + $this->objectStorage = new StorageObjectStore($baseStorage); + $config['objectstore'] = $this->objectStorage; + $this->instance = new ObjectStoreStorage($config); + } + + protected function tearDown() { + if (is_null($this->instance)) { + return; + } + $this->instance->getCache()->clear(); + + parent::tearDown(); + } + + public function testStat() { + + $textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt'; + $ctimeStart = time(); + $this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile)); + $this->assertTrue($this->instance->isReadable('/lorem.txt')); + $ctimeEnd = time(); + $mTime = $this->instance->filemtime('/lorem.txt'); + + // check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1) + $this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime); + $this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime); + $this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt')); + + $stat = $this->instance->stat('/lorem.txt'); + //only size and mtime are required in the result + $this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt')); + $this->assertEquals($stat['mtime'], $mTime); + + if ($this->instance->touch('/lorem.txt', 100) !== false) { + $mTime = $this->instance->filemtime('/lorem.txt'); + $this->assertEquals($mTime, 100); + } + } + + public function testCheckUpdate() { + $this->markTestSkipped('Detecting external changes is not supported on object storages'); + } + + /** + * @dataProvider copyAndMoveProvider + */ + public function testMove($source, $target) { + $this->initSourceAndTarget($source); + $sourceId = $this->instance->getCache()->getId(ltrim('/',$source)); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->rename($source, $target); + + $this->assertTrue($this->instance->file_exists($target), $target.' was not created'); + $this->assertFalse($this->instance->file_exists($source), $source.' still exists'); + $this->assertSameAsLorem($target); + + $targetId = $this->instance->getCache()->getId(ltrim('/',$target)); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameDirectory() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $this->instance->file_put_contents('source/test2.txt', 'qwerty'); + $this->instance->mkdir('source/subfolder'); + $this->instance->file_put_contents('source/subfolder/test.txt', 'bar'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertFalse($this->instance->file_exists('source/test2.txt')); + $this->assertFalse($this->instance->file_exists('source/subfolder')); + $this->assertFalse($this->instance->file_exists('source/subfolder/test.txt')); + + $this->assertTrue($this->instance->file_exists('target')); + $this->assertTrue($this->instance->file_exists('target/test1.txt')); + $this->assertTrue($this->instance->file_exists('target/test2.txt')); + $this->assertTrue($this->instance->file_exists('target/subfolder')); + $this->assertTrue($this->instance->file_exists('target/subfolder/test.txt')); + + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt')); + $this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameOverWriteDirectory() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->mkdir('target'); + $this->instance->file_put_contents('target/test1.txt', 'bar'); + $this->instance->file_put_contents('target/test2.txt', 'bar'); + + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertFalse($this->instance->file_exists('target/test2.txt')); + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameOverWriteDirectoryOverFile() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->file_put_contents('target', 'bar'); + + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } +} diff --git a/tests/lib/Files/ObjectStore/ObjectStoreTest.php b/tests/lib/Files/ObjectStore/ObjectStoreTest.php new file mode 100644 index 00000000000..2116306053e --- /dev/null +++ b/tests/lib/Files/ObjectStore/ObjectStoreTest.php @@ -0,0 +1,97 @@ +<?php + +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace Test\Files\ObjectStore; + +use Test\TestCase; + +abstract class ObjectStoreTest extends TestCase { + + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + abstract protected function getInstance(); + + private function stringToStream($data) { + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $data); + rewind($stream); + return $stream; + } + + public function testWriteRead() { + $stream = $this->stringToStream('foobar'); + + $instance = $this->getInstance(); + + $instance->writeObject('1', $stream); + + $result = $instance->readObject('1'); + $instance->deleteObject('1'); + + $this->assertEquals('foobar', stream_get_contents($result)); + + } + + public function testDelete() { + $stream = $this->stringToStream('foobar'); + + $instance = $this->getInstance(); + + $instance->writeObject('2', $stream); + + $instance->deleteObject('2'); + + try { + // to to read to verify that the object no longer exists + $instance->readObject('2'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } + + public function testReadNonExisting() { + $instance = $this->getInstance(); + + try { + $instance->readObject('non-existing'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } + + public function testDeleteNonExisting() { + $instance = $this->getInstance(); + + try { + $instance->deleteObject('non-existing'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } +} diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php new file mode 100644 index 00000000000..10afb9a7aa8 --- /dev/null +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -0,0 +1,38 @@ +<?php +/** + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\S3; + +class S3Test extends ObjectStoreTest { + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + protected function getInstance() { + $config = \OC::$server->getConfig()->getSystemValue('objectstore'); + if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') { + $this->markTestSkipped('objectstore not configured for s3'); + } + + return new S3($config['arguments']); + } +} diff --git a/tests/lib/Files/ObjectStore/SwiftTest.php b/tests/lib/Files/ObjectStore/SwiftTest.php index 9f69e040f87..f2aeace769c 100644 --- a/tests/lib/Files/ObjectStore/SwiftTest.php +++ b/tests/lib/Files/ObjectStore/SwiftTest.php @@ -1,198 +1,38 @@ <?php /** - * @author Jörn Friedrich Dreyer - * @copyright (c) 2014 Jörn Friedrich Dreyer <jfd@owncloud.com> + * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE - * License as published by the Free Software Foundation; either - * version 3 of the License, or any later version. + * @license GNU AGPL version 3 or any later version * - * This library is distributed in the hope that it will be useful, + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU AFFERO GENERAL PUBLIC LICENSE for more details. + * GNU Affero General Public License for more details. * - * You should have received a copy of the GNU Affero General Public - * License along with this library. If not, see <http://www.gnu.org/licenses/>. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. * */ namespace Test\Files\ObjectStore; -use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\ObjectStore\Swift; -/** - * Class SwiftTest - * - * @group DB - * - * @package Test\Files\Cache\ObjectStore - */ -class SwiftTest extends \Test\Files\Storage\Storage { - +class SwiftTest extends ObjectStoreTest { /** - * @var Swift + * @return \OCP\Files\ObjectStore\IObjectStore */ - private $objectStorage; - - protected function setUp() { - parent::setUp(); - - if (!getenv('RUN_OBJECTSTORE_TESTS')) { - $this->markTestSkipped('objectstore tests are unreliable in some environments'); - } - - // reset backend - \OC_User::clearBackends(); - \OC_User::useBackend('database'); - - // create users - $users = array('test'); - foreach($users as $userName) { - $user = \OC::$server->getUserManager()->get($userName); - if ($user !== null) { $user->delete(); } - \OC::$server->getUserManager()->createUser($userName, $userName); - } - - // main test user - \OC_Util::tearDownFS(); - \OC_User::setUserId(''); - \OC\Files\Filesystem::tearDown(); - \OC_User::setUserId('test'); - + protected function getInstance() { $config = \OC::$server->getConfig()->getSystemValue('objectstore'); - $this->objectStorage = new Swift($config['arguments']); - $config['objectstore'] = $this->objectStorage; - $this->instance = new ObjectStoreStorage($config); - } - - protected function tearDown() { - if (is_null($this->instance)) { - return; - } - $this->objectStorage->deleteContainer(true); - $this->instance->getCache()->clear(); - - $users = array('test'); - foreach($users as $userName) { - $user = \OC::$server->getUserManager()->get($userName); - if ($user !== null) { $user->delete(); } - } - parent::tearDown(); - } - - public function testStat() { - - $textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt'; - $ctimeStart = time(); - $this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile)); - $this->assertTrue($this->instance->isReadable('/lorem.txt')); - $ctimeEnd = time(); - $mTime = $this->instance->filemtime('/lorem.txt'); - - // check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1) - $this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime); - $this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime); - $this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt')); - - $stat = $this->instance->stat('/lorem.txt'); - //only size and mtime are required in the result - $this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt')); - $this->assertEquals($stat['mtime'], $mTime); - - if ($this->instance->touch('/lorem.txt', 100) !== false) { - $mTime = $this->instance->filemtime('/lorem.txt'); - $this->assertEquals($mTime, 100); + if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\Swift') { + $this->markTestSkipped('objectstore not configured for swift'); } - } - - public function testCheckUpdate() { - $this->markTestSkipped('Detecting external changes is not supported on object storages'); - } - - /** - * @dataProvider copyAndMoveProvider - */ - public function testMove($source, $target) { - $this->initSourceAndTarget($source); - $sourceId = $this->instance->getCache()->getId(ltrim('/',$source)); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->rename($source, $target); - - $this->assertTrue($this->instance->file_exists($target), $target.' was not created'); - $this->assertFalse($this->instance->file_exists($source), $source.' still exists'); - $this->assertSameAsLorem($target); - - $targetId = $this->instance->getCache()->getId(ltrim('/',$target)); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameDirectory() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $this->instance->file_put_contents('source/test2.txt', 'qwerty'); - $this->instance->mkdir('source/subfolder'); - $this->instance->file_put_contents('source/subfolder/test.txt', 'bar'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - $this->instance->rename('source', 'target'); - - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertFalse($this->instance->file_exists('source/test2.txt')); - $this->assertFalse($this->instance->file_exists('source/subfolder')); - $this->assertFalse($this->instance->file_exists('source/subfolder/test.txt')); - - $this->assertTrue($this->instance->file_exists('target')); - $this->assertTrue($this->instance->file_exists('target/test1.txt')); - $this->assertTrue($this->instance->file_exists('target/test2.txt')); - $this->assertTrue($this->instance->file_exists('target/subfolder')); - $this->assertTrue($this->instance->file_exists('target/subfolder/test.txt')); - - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt')); - $this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameOverWriteDirectory() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->mkdir('target'); - $this->instance->file_put_contents('target/test1.txt', 'bar'); - $this->instance->file_put_contents('target/test2.txt', 'bar'); - - $this->instance->rename('source', 'target'); - - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertFalse($this->instance->file_exists('target/test2.txt')); - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameOverWriteDirectoryOverFile() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->file_put_contents('target', 'bar'); - - $this->instance->rename('source', 'target'); - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + return new Swift($config['arguments']); } } |