From 0a246f9852ba87583a3cc36c227e68353bb8346b Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 8 Aug 2016 13:54:46 +0200 Subject: split off s3 connection logic Signed-off-by: Robin Appelman --- .../Files/ObjectStore/S3ConnectionTrait.php | 124 +++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 lib/private/Files/ObjectStore/S3ConnectionTrait.php (limited to 'lib/private/Files') diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php new file mode 100644 index 00000000000..a8b57cb18d3 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -0,0 +1,124 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +use Aws\S3\Exception\S3Exception; +use Aws\S3\S3Client; + +trait S3ConnectionTrait { + /** @var array */ + protected $params; + + /** @var S3Client */ + protected $connection; + + /** @var string */ + protected $id; + + /** @var string */ + protected $bucket; + + /** @var int */ + protected $timeout; + + protected $test; + + protected function parseParams($params) { + if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) { + throw new \Exception("Access Key, Secret and Bucket have to be configured."); + } + + $this->id = 'amazon::' . $params['bucket']; + + $this->test = isset($params['test']); + $this->bucket = $params['bucket']; + $this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout']; + $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; + $params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname']; + if (!isset($params['port']) || $params['port'] === '') { + $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; + } + $this->params = $params; + } + + + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ + protected function getConnection() { + if (!is_null($this->connection)) { + return $this->connection; + } + + $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; + $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; + + $options = [ + 'key' => $this->params['key'], + 'secret' => $this->params['secret'], + 'base_url' => $base_url, + 'region' => $this->params['region'], + S3Client::COMMAND_PARAMS => [ + 'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, + ] + ]; + if (isset($this->params['proxy'])) { + $options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']]; + } + $this->connection = S3Client::factory($options); + + if (!$this->connection->isValidBucketName($this->bucket)) { + throw new \Exception("The configured bucket name is invalid."); + } + + if (!$this->connection->doesBucketExist($this->bucket)) { + try { + $this->connection->createBucket(array( + 'Bucket' => $this->bucket + )); + $this->connection->waitUntilBucketExists(array( + 'Bucket' => $this->bucket, + 'waiter.interval' => 1, + 'waiter.max_attempts' => 15 + )); + $this->testTimeout(); + } catch (S3Exception $e) { + \OCP\Util::logException('files_external', $e); + throw new \Exception('Creation of bucket failed. ' . $e->getMessage()); + } + } + + return $this->connection; + } + + /** + * when running the tests wait to let the buckets catch up + */ + private function testTimeout() { + if ($this->test) { + sleep($this->timeout); + } + } +} -- cgit v1.2.3 From 64e896cc0d9624718ec4e8753b6b58b242a7a2d5 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 8 Aug 2016 15:13:44 +0200 Subject: split testing of objectstoragestorage and objectstore implementations Signed-off-by: Robin Appelman --- .../Files/ObjectStore/StorageObjectStore.php | 90 ++++++++++ tests/lib/Files/ObjectStore/LocalTest.php | 36 ++++ .../Files/ObjectStore/ObjectStoreStorageTest.php | 169 ++++++++++++++++++ tests/lib/Files/ObjectStore/ObjectStoreTest.php | 97 +++++++++++ tests/lib/Files/ObjectStore/SwiftTest.php | 194 ++------------------- 5 files changed, 409 insertions(+), 177 deletions(-) create mode 100644 lib/private/Files/ObjectStore/StorageObjectStore.php create mode 100644 tests/lib/Files/ObjectStore/LocalTest.php create mode 100644 tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php create mode 100644 tests/lib/Files/ObjectStore/ObjectStoreTest.php (limited to 'lib/private/Files') diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php new file mode 100644 index 00000000000..044243c1ee6 --- /dev/null +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -0,0 +1,90 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\Storage\IStorage; + +/** + * Object store that wraps a storage backend, mostly for testing purposes + */ +class StorageObjectStore implements IObjectStore { + /** @var IStorage */ + private $storage; + + /** + * @param IStorage $storage + */ + public function __construct(IStorage $storage) { + $this->storage = $storage; + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + function getStorageId() { + $this->storage->getId(); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + $handle = $this->storage->fopen($urn, 'r'); + if ($handle) { + return $handle; + } else { + throw new \Exception(); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $handle = $this->storage->fopen($urn, 'w'); + if ($handle) { + stream_copy_to_stream($stream, $handle); + fclose($handle); + } else { + throw new \Exception(); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->storage->unlink($urn); + } + +} diff --git a/tests/lib/Files/ObjectStore/LocalTest.php b/tests/lib/Files/ObjectStore/LocalTest.php new file mode 100644 index 00000000000..99aa23e065d --- /dev/null +++ b/tests/lib/Files/ObjectStore/LocalTest.php @@ -0,0 +1,36 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\StorageObjectStore; +use OC\Files\Storage\Temporary; + +class LocalTest extends ObjectStoreTest { + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + protected function getInstance() { + $storage = new Temporary(); + return new StorageObjectStore($storage); + } + +} diff --git a/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php b/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php new file mode 100644 index 00000000000..c9d6c1bd922 --- /dev/null +++ b/tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php @@ -0,0 +1,169 @@ + + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE + * License as published by the Free Software Foundation; either + * version 3 of the License, or any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU AFFERO GENERAL PUBLIC LICENSE for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with this library. If not, see . + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\ObjectStore\StorageObjectStore; +use OC\Files\Storage\Temporary; +use OCP\Files\ObjectStore\IObjectStore; +use Test\Files\Storage\Storage; + +/** + * @group DB + */ +class ObjectStoreStorageTest extends Storage { + + /** + * @var IObjectStore + */ + private $objectStorage; + + protected function setUp() { + parent::setUp(); + + $baseStorage = new Temporary(); + $this->objectStorage = new StorageObjectStore($baseStorage); + $config['objectstore'] = $this->objectStorage; + $this->instance = new ObjectStoreStorage($config); + } + + protected function tearDown() { + if (is_null($this->instance)) { + return; + } + $this->instance->getCache()->clear(); + + parent::tearDown(); + } + + public function testStat() { + + $textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt'; + $ctimeStart = time(); + $this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile)); + $this->assertTrue($this->instance->isReadable('/lorem.txt')); + $ctimeEnd = time(); + $mTime = $this->instance->filemtime('/lorem.txt'); + + // check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1) + $this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime); + $this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime); + $this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt')); + + $stat = $this->instance->stat('/lorem.txt'); + //only size and mtime are required in the result + $this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt')); + $this->assertEquals($stat['mtime'], $mTime); + + if ($this->instance->touch('/lorem.txt', 100) !== false) { + $mTime = $this->instance->filemtime('/lorem.txt'); + $this->assertEquals($mTime, 100); + } + } + + public function testCheckUpdate() { + $this->markTestSkipped('Detecting external changes is not supported on object storages'); + } + + /** + * @dataProvider copyAndMoveProvider + */ + public function testMove($source, $target) { + $this->initSourceAndTarget($source); + $sourceId = $this->instance->getCache()->getId(ltrim('/',$source)); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->rename($source, $target); + + $this->assertTrue($this->instance->file_exists($target), $target.' was not created'); + $this->assertFalse($this->instance->file_exists($source), $source.' still exists'); + $this->assertSameAsLorem($target); + + $targetId = $this->instance->getCache()->getId(ltrim('/',$target)); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameDirectory() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $this->instance->file_put_contents('source/test2.txt', 'qwerty'); + $this->instance->mkdir('source/subfolder'); + $this->instance->file_put_contents('source/subfolder/test.txt', 'bar'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertFalse($this->instance->file_exists('source/test2.txt')); + $this->assertFalse($this->instance->file_exists('source/subfolder')); + $this->assertFalse($this->instance->file_exists('source/subfolder/test.txt')); + + $this->assertTrue($this->instance->file_exists('target')); + $this->assertTrue($this->instance->file_exists('target/test1.txt')); + $this->assertTrue($this->instance->file_exists('target/test2.txt')); + $this->assertTrue($this->instance->file_exists('target/subfolder')); + $this->assertTrue($this->instance->file_exists('target/subfolder/test.txt')); + + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt')); + $this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameOverWriteDirectory() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->mkdir('target'); + $this->instance->file_put_contents('target/test1.txt', 'bar'); + $this->instance->file_put_contents('target/test2.txt', 'bar'); + + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertFalse($this->instance->file_exists('target/test2.txt')); + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } + + public function testRenameOverWriteDirectoryOverFile() { + $this->instance->mkdir('source'); + $this->instance->file_put_contents('source/test1.txt', 'foo'); + $sourceId = $this->instance->getCache()->getId('source'); + $this->assertNotEquals(-1, $sourceId); + + $this->instance->file_put_contents('target', 'bar'); + + $this->instance->rename('source', 'target'); + + $this->assertFalse($this->instance->file_exists('source')); + $this->assertFalse($this->instance->file_exists('source/test1.txt')); + $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); + $targetId = $this->instance->getCache()->getId('target'); + $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + } +} diff --git a/tests/lib/Files/ObjectStore/ObjectStoreTest.php b/tests/lib/Files/ObjectStore/ObjectStoreTest.php new file mode 100644 index 00000000000..2116306053e --- /dev/null +++ b/tests/lib/Files/ObjectStore/ObjectStoreTest.php @@ -0,0 +1,97 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Files\ObjectStore; + +use Test\TestCase; + +abstract class ObjectStoreTest extends TestCase { + + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + abstract protected function getInstance(); + + private function stringToStream($data) { + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $data); + rewind($stream); + return $stream; + } + + public function testWriteRead() { + $stream = $this->stringToStream('foobar'); + + $instance = $this->getInstance(); + + $instance->writeObject('1', $stream); + + $result = $instance->readObject('1'); + $instance->deleteObject('1'); + + $this->assertEquals('foobar', stream_get_contents($result)); + + } + + public function testDelete() { + $stream = $this->stringToStream('foobar'); + + $instance = $this->getInstance(); + + $instance->writeObject('2', $stream); + + $instance->deleteObject('2'); + + try { + // to to read to verify that the object no longer exists + $instance->readObject('2'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } + + public function testReadNonExisting() { + $instance = $this->getInstance(); + + try { + $instance->readObject('non-existing'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } + + public function testDeleteNonExisting() { + $instance = $this->getInstance(); + + try { + $instance->deleteObject('non-existing'); + $this->fail(); + } catch (\Exception $e) { + // dummy assert to keep phpunit happy + $this->assertEquals(1, 1); + } + } +} diff --git a/tests/lib/Files/ObjectStore/SwiftTest.php b/tests/lib/Files/ObjectStore/SwiftTest.php index 9f69e040f87..f2aeace769c 100644 --- a/tests/lib/Files/ObjectStore/SwiftTest.php +++ b/tests/lib/Files/ObjectStore/SwiftTest.php @@ -1,198 +1,38 @@ + * @copyright Copyright (c) 2016 Robin Appelman * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE - * License as published by the Free Software Foundation; either - * version 3 of the License, or any later version. + * @license GNU AGPL version 3 or any later version * - * This library is distributed in the hope that it will be useful, + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU AFFERO GENERAL PUBLIC LICENSE for more details. + * GNU Affero General Public License for more details. * - * You should have received a copy of the GNU Affero General Public - * License along with this library. If not, see . + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . * */ namespace Test\Files\ObjectStore; -use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\ObjectStore\Swift; -/** - * Class SwiftTest - * - * @group DB - * - * @package Test\Files\Cache\ObjectStore - */ -class SwiftTest extends \Test\Files\Storage\Storage { - +class SwiftTest extends ObjectStoreTest { /** - * @var Swift + * @return \OCP\Files\ObjectStore\IObjectStore */ - private $objectStorage; - - protected function setUp() { - parent::setUp(); - - if (!getenv('RUN_OBJECTSTORE_TESTS')) { - $this->markTestSkipped('objectstore tests are unreliable in some environments'); - } - - // reset backend - \OC_User::clearBackends(); - \OC_User::useBackend('database'); - - // create users - $users = array('test'); - foreach($users as $userName) { - $user = \OC::$server->getUserManager()->get($userName); - if ($user !== null) { $user->delete(); } - \OC::$server->getUserManager()->createUser($userName, $userName); - } - - // main test user - \OC_Util::tearDownFS(); - \OC_User::setUserId(''); - \OC\Files\Filesystem::tearDown(); - \OC_User::setUserId('test'); - + protected function getInstance() { $config = \OC::$server->getConfig()->getSystemValue('objectstore'); - $this->objectStorage = new Swift($config['arguments']); - $config['objectstore'] = $this->objectStorage; - $this->instance = new ObjectStoreStorage($config); - } - - protected function tearDown() { - if (is_null($this->instance)) { - return; - } - $this->objectStorage->deleteContainer(true); - $this->instance->getCache()->clear(); - - $users = array('test'); - foreach($users as $userName) { - $user = \OC::$server->getUserManager()->get($userName); - if ($user !== null) { $user->delete(); } - } - parent::tearDown(); - } - - public function testStat() { - - $textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt'; - $ctimeStart = time(); - $this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile)); - $this->assertTrue($this->instance->isReadable('/lorem.txt')); - $ctimeEnd = time(); - $mTime = $this->instance->filemtime('/lorem.txt'); - - // check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1) - $this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime); - $this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime); - $this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt')); - - $stat = $this->instance->stat('/lorem.txt'); - //only size and mtime are required in the result - $this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt')); - $this->assertEquals($stat['mtime'], $mTime); - - if ($this->instance->touch('/lorem.txt', 100) !== false) { - $mTime = $this->instance->filemtime('/lorem.txt'); - $this->assertEquals($mTime, 100); + if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\Swift') { + $this->markTestSkipped('objectstore not configured for swift'); } - } - - public function testCheckUpdate() { - $this->markTestSkipped('Detecting external changes is not supported on object storages'); - } - - /** - * @dataProvider copyAndMoveProvider - */ - public function testMove($source, $target) { - $this->initSourceAndTarget($source); - $sourceId = $this->instance->getCache()->getId(ltrim('/',$source)); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->rename($source, $target); - - $this->assertTrue($this->instance->file_exists($target), $target.' was not created'); - $this->assertFalse($this->instance->file_exists($source), $source.' still exists'); - $this->assertSameAsLorem($target); - - $targetId = $this->instance->getCache()->getId(ltrim('/',$target)); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameDirectory() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $this->instance->file_put_contents('source/test2.txt', 'qwerty'); - $this->instance->mkdir('source/subfolder'); - $this->instance->file_put_contents('source/subfolder/test.txt', 'bar'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - $this->instance->rename('source', 'target'); - - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertFalse($this->instance->file_exists('source/test2.txt')); - $this->assertFalse($this->instance->file_exists('source/subfolder')); - $this->assertFalse($this->instance->file_exists('source/subfolder/test.txt')); - - $this->assertTrue($this->instance->file_exists('target')); - $this->assertTrue($this->instance->file_exists('target/test1.txt')); - $this->assertTrue($this->instance->file_exists('target/test2.txt')); - $this->assertTrue($this->instance->file_exists('target/subfolder')); - $this->assertTrue($this->instance->file_exists('target/subfolder/test.txt')); - - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt')); - $this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameOverWriteDirectory() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->mkdir('target'); - $this->instance->file_put_contents('target/test1.txt', 'bar'); - $this->instance->file_put_contents('target/test2.txt', 'bar'); - - $this->instance->rename('source', 'target'); - - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertFalse($this->instance->file_exists('target/test2.txt')); - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); - } - - public function testRenameOverWriteDirectoryOverFile() { - $this->instance->mkdir('source'); - $this->instance->file_put_contents('source/test1.txt', 'foo'); - $sourceId = $this->instance->getCache()->getId('source'); - $this->assertNotEquals(-1, $sourceId); - - $this->instance->file_put_contents('target', 'bar'); - - $this->instance->rename('source', 'target'); - $this->assertFalse($this->instance->file_exists('source')); - $this->assertFalse($this->instance->file_exists('source/test1.txt')); - $this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt')); - $targetId = $this->instance->getCache()->getId('target'); - $this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break'); + return new Swift($config['arguments']); } } -- cgit v1.2.3 From eefd0597166b26b85690fba784d843486badefef Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 8 Aug 2016 15:50:15 +0200 Subject: add amazon s3 objectstore backend Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3.php | 116 +++++++++++++++++++++++++++++++++ tests/lib/Files/ObjectStore/S3Test.php | 38 +++++++++++ 2 files changed, 154 insertions(+) create mode 100644 lib/private/Files/ObjectStore/S3.php create mode 100644 tests/lib/Files/ObjectStore/S3Test.php (limited to 'lib/private/Files') diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php new file mode 100644 index 00000000000..641c1722b58 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3.php @@ -0,0 +1,116 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +use Guzzle\Http\EntityBody; +use Guzzle\Http\Message\RequestInterface; +use Guzzle\Service\Command\CommandInterface; +use Guzzle\Stream\PhpStreamRequestFactory; +use Icewind\Streams\CallbackWrapper; +use OCP\Files\ObjectStore\IObjectStore; + +// TODO: proper composer +set_include_path(get_include_path() . PATH_SEPARATOR . + \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php'); +require_once 'aws-autoloader.php'; + +class S3 implements IObjectStore { + use S3ConnectionTrait; + + public function __construct($parameters) { + $this->parseParams($parameters); + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + function getStorageId() { + return $this->id; + } + + /** + * Serialize and sign a command, returning a request object + * + * @param CommandInterface $command Command to sign + * + * @return RequestInterface + */ + protected function getSignedRequest($command) { + $request = $command->prepare(); + $request->dispatch('request.before_send', array('request' => $request)); + + return $request; + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + // Create the command and serialize the request + $request = $this->getSignedRequest($this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn + ])); + // Create a stream that uses the EntityBody object + $factory = new PhpStreamRequestFactory(); + /** @var EntityBody $body */ + $body = $factory->fromRequest($request, array(), array('stream_class' => 'Guzzle\Http\EntityBody')); + $stream = $body->getStream(); + + // we need to keep the guzzle request in scope untill the stream is closed + return CallbackWrapper::wrap($stream, null, null, function () use ($body) { + $body->close(); + }); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $this->getConnection()->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream + ]); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->getConnection()->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ]); + } + +} diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php new file mode 100644 index 00000000000..10afb9a7aa8 --- /dev/null +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -0,0 +1,38 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Files\ObjectStore; + +use OC\Files\ObjectStore\S3; + +class S3Test extends ObjectStoreTest { + /** + * @return \OCP\Files\ObjectStore\IObjectStore + */ + protected function getInstance() { + $config = \OC::$server->getConfig()->getSystemValue('objectstore'); + if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') { + $this->markTestSkipped('objectstore not configured for s3'); + } + + return new S3($config['arguments']); + } +} -- cgit v1.2.3 From 0ee958595e42bc29543e294958b2090b80f85d2d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 27 Oct 2016 14:26:13 +0200 Subject: Improve isMounted to handle primary storage object store Signed-off-by: Robin Appelman --- lib/private/Files/FileInfo.php | 7 ++++++- lib/private/Files/Mount/ObjectHomeMountProvider.php | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib/private/Files') diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 14a32ba8f76..3ad2932e597 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -33,6 +33,7 @@ namespace OC\Files; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage\IStorage; +use OCP\Files\IHomeStorage; use OCP\IUser; class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { @@ -305,7 +306,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { } public function isMounted() { - $sid = $this->getStorage()->getId(); + $storage = $this->getStorage(); + if ($storage->instanceOfStorage('\OCP\Files\IHomeStorage')) { + return false; + } + $sid = $storage->getId(); if (!is_null($sid)) { $sid = explode(':', $sid); return ($sid[0] !== 'home' and $sid[0] !== 'shared'); diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 87878562a42..440a8bc4608 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -51,7 +51,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { * * @param IUser $user * @param IStorageFactory $loader - * @return \OCP\Files\Mount\IMountPoint[] + * @return \OCP\Files\Mount\IMountPoint */ public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { -- cgit v1.2.3 From 3ab160dd5a9071bb993fe95361d29f1f8d74ab93 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 28 Oct 2016 16:35:39 +0200 Subject: fopen s3 objects directly to work around unexplainable guzzle bug For some reason when a text file started with a valid hex character ([0-9a-f]) it would eat the text untill the first newline The new code does basically the same thing as guzzle/s3-sdk did only without wrapping everything in a guzzle stream Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3.php | 51 +++++++++++++++--------------------- 1 file changed, 21 insertions(+), 30 deletions(-) (limited to 'lib/private/Files') diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 641c1722b58..5251b473bdf 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -21,11 +21,6 @@ namespace OC\Files\ObjectStore; -use Guzzle\Http\EntityBody; -use Guzzle\Http\Message\RequestInterface; -use Guzzle\Service\Command\CommandInterface; -use Guzzle\Stream\PhpStreamRequestFactory; -use Icewind\Streams\CallbackWrapper; use OCP\Files\ObjectStore\IObjectStore; // TODO: proper composer @@ -48,20 +43,6 @@ class S3 implements IObjectStore { return $this->id; } - /** - * Serialize and sign a command, returning a request object - * - * @param CommandInterface $command Command to sign - * - * @return RequestInterface - */ - protected function getSignedRequest($command) { - $request = $command->prepare(); - $request->dispatch('request.before_send', array('request' => $request)); - - return $request; - } - /** * @param string $urn the unified resource name used to identify the object * @return resource stream with the read data @@ -70,20 +51,30 @@ class S3 implements IObjectStore { */ function readObject($urn) { // Create the command and serialize the request - $request = $this->getSignedRequest($this->getConnection()->getCommand('GetObject', [ + $request = $this->getConnection()->getCommand('GetObject', [ 'Bucket' => $this->bucket, 'Key' => $urn - ])); - // Create a stream that uses the EntityBody object - $factory = new PhpStreamRequestFactory(); - /** @var EntityBody $body */ - $body = $factory->fromRequest($request, array(), array('stream_class' => 'Guzzle\Http\EntityBody')); - $stream = $body->getStream(); + ])->prepare(); + + $request->dispatch('request.before_send', array( + 'request' => $request + )); + + $headers = $request->getHeaderLines(); + $headers[] = 'Connection: close'; + + $opts = [ + 'http' => [ + 'method' => "GET", + 'header' => $headers + ], + 'ssl' => [ + 'verify_peer' => true + ] + ]; - // we need to keep the guzzle request in scope untill the stream is closed - return CallbackWrapper::wrap($stream, null, null, function () use ($body) { - $body->close(); - }); + $context = stream_context_create($opts); + return fopen($request->getUrl(), 'r', false, $context); } /** -- cgit v1.2.3