From c112a1d323e363eaf12574bf094a90ce2f89f028 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 20 Jun 2014 12:27:47 +0200 Subject: [PATCH] move to stream based IObjectStore interface, rearrange & reformat code --- .../files/objectstore/objectstorestorage.php | 225 +++++++++--------- lib/private/files/objectstore/swift.php | 47 ++-- lib/public/files/objectstore/iobjectstore.php | 12 +- 3 files changed, 140 insertions(+), 144 deletions(-) diff --git a/lib/private/files/objectstore/objectstorestorage.php b/lib/private/files/objectstore/objectstorestorage.php index b925f2e67fc..0054ee113f0 100644 --- a/lib/private/files/objectstore/objectstorestorage.php +++ b/lib/private/files/objectstore/objectstorestorage.php @@ -24,21 +24,19 @@ use OCP\Files\ObjectStore\IObjectStore; class ObjectStoreStorage extends \OC\Files\Storage\Common { + /** + * @var array + */ + private static $tmpFiles = array(); /** * @var \OCP\Files\ObjectStore\IObjectStore $objectStore */ protected $objectStore; - /** * @var \OC\User\User $user */ protected $user; - /** - * @var array - */ - private static $tmpFiles = array(); - public function __construct($params) { if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { $this->objectStore = $params['objectstore']; @@ -46,57 +44,20 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { throw new \Exception('missing IObjectStore instance'); } if (isset($params['storageid'])) { - $this->id = 'object::store:'.$params['storageid']; + $this->id = 'object::store:' . $params['storageid']; } else { - $this->id = 'object::store:'.$this->objectStore->getStorageId(); + $this->id = 'object::store:' . $this->objectStore->getStorageId(); } //initialize cache with root directory in cache - if ( ! $this->is_dir('/') ) { + if (!$this->is_dir('/')) { $this->mkdir('/'); } } - /** - * Object Stores use a NoopScanner because metadata is directly stored in - * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. - * @param string $path - * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner - * @return \OC\Files\ObjectStore\NoopScanner - */ - public function getScanner($path = '', $storage = null) { - if (!$storage) { - $storage = $this; - } - if (!isset($this->scanner)) { - $this->scanner = new NoopScanner($storage); - } - return $this->scanner; - } - - /** - * @param string $path - * @return string - */ - private function normalizePath($path) { - $path = trim($path, '/'); - //FIXME why do we sometimes get a path like 'files//username'? - $path = str_replace('//', '/', $path); - - if (!$path) { - $path = '.'; - } - - return $path; - } - - public function getId () { - return $this->id; - } - public function mkdir($path) { $path = $this->normalizePath($path); - if($this->is_dir($path)) { + if ($this->is_dir($path)) { return false; } @@ -113,7 +74,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { 'permissions' => \OCP\PERMISSION_ALL, ); - if ($dirName === '.' && ! $parentExists ) { + if ($dirName === '.' && !$parentExists) { //create root on the fly $data['etag'] = $this->getETag($dirName); $this->getCache()->put('', $data); @@ -128,9 +89,56 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return false; } - public function file_exists($path) { + /** + * @param string $path + * @return string + */ + private function normalizePath($path) { + $path = trim($path, '/'); + //FIXME why do we sometimes get a path like 'files//username'? + $path = str_replace('//', '/', $path); + + if (!$path) { + $path = '.'; + } + + return $path; + } + + /** + * Object Stores use a NoopScanner because metadata is directly stored in + * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. + * + * @param string $path + * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner + * @return \OC\Files\ObjectStore\NoopScanner + */ + public function getScanner($path = '', $storage = null) { + if (!$storage) { + $storage = $this; + } + if (!isset($this->scanner)) { + $this->scanner = new NoopScanner($storage); + } + return $this->scanner; + } + + public function getId() { + return $this->id; + } + + public function rmdir($path) { $path = $this->normalizePath($path); - return (bool)$this->stat($path); + + if (!$this->is_dir($path)) { + return false; + } + + $this->rmObjects($path); + + $this->getCache()->remove($path); + + return true; } private function rmObjects($path) { @@ -144,23 +152,52 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - public function rmdir($path) { + public function unlink($path) { $path = $this->normalizePath($path); + $stat = $this->stat($path); - if (!$this->is_dir($path)) { - return false; + if ($stat && isset($stat['fileid'])) { + if ($stat['mimetype'] === 'httpd/unix-directory') { + return $this->rmdir($path); + } + try { + $this->objectStore->deleteObject($this->getURN($stat['fileid'])); + } catch (\Exception $ex) { + if ($ex->getCode() !== 404) { + \OCP\Util::writeLog('objectstore', 'Could not delete object: ' . $ex->getMessage(), \OCP\Util::ERROR); + return false; + } else { + //removing from cache is ok as it does not exist in the objectstore anyway + } + } + $this->getCache()->remove($path); + return true; } + return false; + } - $this->rmObjects($path); - - $this->getCache()->remove($path); + public function stat($path) { + return $this->getCache()->get($path); + } - return true; + /** + * Override this method if you need a different unique resource identifier for your object storage implementation. + * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. + * You may need a mapping table to store your URN if it cannot be generated from the fileid. + * + * @param int $fileId the fileid + * @return null|string the unified resource name used to identify the object + */ + protected function getURN($fileId) { + if (is_numeric($fileId)) { + return 'urn:oid:' . $fileId; + } + return null; } public function opendir($path) { $path = $this->normalizePath($path); - + if ($path === '.') { $path = ''; } else if ($path) { @@ -173,7 +210,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { foreach ($folderContents as $file) { $files[] = $file['name']; } - + \OC\Files\Stream\Dir::register('objstore' . $path, $files); return opendir('fakedir://objstore' . $path); @@ -183,10 +220,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - public function stat($path) { - return $this->getCache()->get($path); - } - public function filetype($path) { $path = $this->normalizePath($path); $stat = $this->stat($path); @@ -200,31 +233,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - - public function unlink($path) { - $path = $this->normalizePath($path); - $stat = $this->stat($path); - - if ($stat && isset($stat['fileid'])) { - if ($stat['mimetype'] === 'httpd/unix-directory') { - return $this->rmdir($path); - } - try { - $this->objectStore->deleteObject($this->getURN($stat['fileid'])); - } catch (\Exception $ex) { - if ($ex->getCode() !== 404) { - \OCP\Util::writeLog('objectstore', 'Could not delete object: '.$ex->getMessage(), \OCP\Util::ERROR); - return false; - } else { - //removing from cache is ok as it does not exist in the objectstore anyway - } - } - $this->getCache()->remove($path); - return true; - } - return false; - } - public function fopen($path, $mode) { $path = $this->normalizePath($path); @@ -233,15 +241,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { case 'rb': $stat = $this->stat($path); if (is_array($stat)) { - $tmpFile = \OC_Helper::tmpFile(); - self::$tmpFiles[$tmpFile] = $path; try { - $this->objectStore->getObject($this->getURN($stat['fileid']), $tmpFile); + return $this->objectStore->readObject($this->getURN($stat['fileid'])); } catch (\Exception $ex) { - \OCP\Util::writeLog('objectstore', 'Could not get object: '.$ex->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('objectstore', 'Could not get object: ' . $ex->getMessage(), \OCP\Util::ERROR); return false; } - return fopen($tmpFile, 'r'); } else { return false; } @@ -275,6 +280,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return false; } + public function file_exists($path) { + $path = $this->normalizePath($path); + return (bool)$this->stat($path); + } + public function rename($path1, $path2) { $path1 = $this->normalizePath($path1); $path2 = $this->normalizePath($path2); @@ -297,12 +307,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } else { return false; } - + } else { return false; } } - + public function getMimeType($path) { $path = $this->normalizePath($path); $stat = $this->stat($path); @@ -312,7 +322,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return false; } } - + public function touch($path, $mtime = null) { if (is_null($mtime)) { $mtime = time(); @@ -343,30 +353,17 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { ); $fileId = $this->getCache()->put($path, $stat); try { - $this->objectStore->writeObject($this->getURN($fileId)); + //read an empty file from memory + $this->objectStore->writeObject($this->getURN($fileId), fopen('php://memory', 'r')); } catch (\Exception $ex) { $this->getCache()->remove($path); - \OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR); return false; } } return true; } - /** - * Override this method if you need a different unique resource identifier for your object storage implementation. - * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. - * You may need a mapping table to store your URN if it cannot be generated from the fileid. - * @param int $fileId the fileid - * @return null|string the unified resource name used to identify the object - */ - protected function getURN($fileId) { - if (is_numeric($fileId)) { - return 'urn:oid:'.$fileId; - } - return null; - } - public function writeBack($tmpFile) { if (!isset(self::$tmpFiles[$tmpFile])) { return false; @@ -391,10 +388,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $fileId = $this->getCache()->put($path, $stat); try { //upload to object storage - $this->objectStore->writeObject($this->getURN($fileId), $tmpFile); + $this->objectStore->writeObject($this->getURN($fileId), fopen($tmpFile, 'r')); } catch (\Exception $ex) { $this->getCache()->remove($path); - \OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR); return false; } return true; diff --git a/lib/private/files/objectstore/swift.php b/lib/private/files/objectstore/swift.php index 14892d2855e..505b5be35f2 100644 --- a/lib/private/files/objectstore/swift.php +++ b/lib/private/files/objectstore/swift.php @@ -21,23 +21,24 @@ namespace OC\Files\ObjectStore; use Guzzle\Http\Exception\ClientErrorResponseException; +use OCP\Files\ObjectStore\IObjectStore; use OpenCloud\OpenStack; -class Swift implements \OCP\Files\ObjectStore\IObjectStore { +class Swift implements IObjectStore { /** * @var \OpenCloud\ObjectStore\Service */ private $objectStoreService; - + /** * @var \OpenCloud\ObjectStore\Resource\Container */ private $container; public function __construct($params) { - if (!isset($params['username']) || !isset($params['password']) ) { - throw new \Exception("Access Key and Secret have to be configured."); + if (!isset($params['username']) || !isset($params['password'])) { + throw new \Exception('Access Key and Secret have to be configured.'); } if (!isset($params['container'])) { $params['container'] = 'owncloud'; @@ -80,40 +81,42 @@ class Swift implements \OCP\Files\ObjectStore\IObjectStore { } } + /** + * @return string the container name where objects are stored + */ public function getStorageId() { return $this->container->name; } /** - * @param string $urn Unified Resource Name - * @param string $tmpFile - * @return void + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write * @throws Exception from openstack lib when something goes wrong */ - public function writeObject($urn, $tmpFile = null) { - $fileData = ''; - if ($tmpFile) { - $fileData = fopen($tmpFile, 'r'); - } - - $this->container->uploadObject($urn, $fileData); + public function writeObject($urn, $stream) { + $this->container->uploadObject($urn, $stream); } /** - * @param string $urn Unified Resource Name - * @param string $tmpFile - * @return void + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data * @throws Exception from openstack lib when something goes wrong */ - public function getObject($urn, $tmpFile) { + public function readObject($urn) { $object = $this->container->getObject($urn); - /** @var $objectContent \Guzzle\Http\EntityBody **/ + // we need to keep a reference to objectContent or + // the stream will be closed before we can do anything with it + /** @var $objectContent \Guzzle\Http\EntityBody * */ $objectContent = $object->getContent(); - $objectContent->rewind(); - $stream = $objectContent->getStream(); - file_put_contents($tmpFile, $stream); + + // directly returning the object stream does not work because the GC seems to collect it, so we need a copy + $tmpStream = fopen('php://temp', 'r+'); + stream_copy_to_stream($objectContent->getStream(), $tmpStream); + rewind($tmpStream); + + return $tmpStream; } /** diff --git a/lib/public/files/objectstore/iobjectstore.php b/lib/public/files/objectstore/iobjectstore.php index 3b6bd98d338..b2c5a9da134 100644 --- a/lib/public/files/objectstore/iobjectstore.php +++ b/lib/public/files/objectstore/iobjectstore.php @@ -11,21 +11,17 @@ interface IObjectStore { /** * @param string $urn the unified resource name used to identify the object - * @param string $tmpFile path to the local temporary file that should be - * used to store the object - * @return void + * @return resource stream with the read data * @throws Exception when something goes wrong, message will be logged */ - function getObject($urn, $tmpFile); + function readObject($urn); /** * @param string $urn the unified resource name used to identify the object - * @param string $tmpFile path to the local temporary file that the object - * should be loaded from - * @return void + * @param resource $stream stream with the data to write * @throws Exception when something goes wrong, message will be logged */ - function writeObject($urn, $tmpFile = null); + function writeObject($urn, $stream); /** * @param string $urn the unified resource name used to identify the object -- 2.39.5