From 1369f2c6840e6bbac50526236771cb730a74f975 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 22 Jan 2016 13:01:37 +0100 Subject: [PATCH] cleanup shared cache --- apps/files_sharing/lib/cache.php | 490 ++---------------- apps/files_sharing/lib/sharedstorage.php | 25 +- apps/files_sharing/lib/watcher.php | 81 --- apps/files_sharing/tests/testcase.php | 4 +- apps/files_versions/tests/versions.php | 4 +- lib/private/files/cache/wrapper/cachejail.php | 2 +- .../cache/wrapper/cachepermissionsmask.php | 2 +- .../files/cache/wrapper/cachewrapper.php | 9 +- 8 files changed, 70 insertions(+), 547 deletions(-) delete mode 100644 apps/files_sharing/lib/watcher.php diff --git a/apps/files_sharing/lib/cache.php b/apps/files_sharing/lib/cache.php index 10d1e787922..c44bebbceaf 100644 --- a/apps/files_sharing/lib/cache.php +++ b/apps/files_sharing/lib/cache.php @@ -32,228 +32,75 @@ namespace OC\Files\Cache; -use OC\User\NoUserException; +use OC\Files\Cache\Wrapper\CacheJail; use OCP\Files\Cache\ICacheEntry; -use OCP\Share_Backend_Collection; +use OCP\Files\Storage\IStorage; /** * Metadata cache for shared files * * don't use this class directly if you need to get metadata, use \OC\Files\Filesystem::getFileInfo instead */ -class Shared_Cache extends Cache { - - private $storage; - private $files = array(); - +class Shared_Cache extends CacheJail { /** - * @param \OC\Files\Storage\Shared $storage + * @var \OC\Files\Storage\Shared */ - public function __construct($storage) { - parent::__construct($storage); - $this->storage = $storage; - } + private $storage; /** - * Get the source cache of a shared file or folder - * - * @param string $target Shared target file path - * @return \OC\Files\Cache\Cache|false + * @var IStorage */ - private function getSourceCache($target) { - if ($target === false || $target === $this->storage->getMountPoint()) { - $target = ''; - } - $source = \OC_Share_Backend_File::getSource($target, $this->storage->getShare()); - if (isset($source['path']) && isset($source['fileOwner'])) { - try { - \OC\Files\Filesystem::initMountPoints($source['fileOwner']); - } catch(NoUserException $e) { - \OC::$server->getLogger()->logException($e, ['app' => 'files_sharing']); - return false; - } - $mounts = \OC\Files\Filesystem::getMountByNumericId($source['storage']); - if (is_array($mounts) and !empty($mounts)) { - $fullPath = $mounts[0]->getMountPoint() . $source['path']; - list($storage, $internalPath) = \OC\Files\Filesystem::resolvePath($fullPath); - if ($storage) { - $this->files[$target] = $internalPath; - $cache = $storage->getCache(); - $this->storageId = $storage->getId(); - $this->numericId = $cache->getNumericStorageId(); - return $cache; - } - } - } - return false; - } - - public function getNumericStorageId() { - if (isset($this->numericId)) { - return $this->numericId; - } else { - return false; - } - } + private $sourceStorage; /** - * get the stored metadata of a file or folder - * - * @param string|int $file - * @return ICacheEntry|false + * @var ICacheEntry */ - public function get($file) { - $mimetypeLoader = \OC::$server->getMimeTypeLoader(); - if (is_string($file)) { - $cache = $this->getSourceCache($file); - if ($cache) { - $data = $cache->get($this->files[$file]); - if ($data) { - $data['displayname_owner'] = \OC_User::getDisplayName($this->storage->getSharedFrom()); - $data['path'] = $file; - if ($file === '') { - $data['is_share_mount_point'] = true; - } - $data['uid_owner'] = $this->storage->getOwner($file); - if (isset($data['permissions'])) { - $data['permissions'] &= $this->storage->getPermissions($file); - } else { - $data['permissions'] = $this->storage->getPermissions($file); - } - } - return $data; - } - } else { - $sourceId = $file; - // if we are at the root of the mount point we want to return the - // cache information for the source item - if (!is_int($sourceId) || $sourceId === 0) { - $sourceId = $this->storage->getSourceId(); - } - $query = \OCP\DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`,' - . ' `size`, `mtime`, `encrypted`, `storage_mtime`, `etag`, `permissions`' - . ' FROM `*PREFIX*filecache` WHERE `fileid` = ?'); - $result = $query->execute(array($sourceId)); - $data = $result->fetchRow(); - $data['fileid'] = (int)$data['fileid']; - $data['mtime'] = (int)$data['mtime']; - $data['storage_mtime'] = (int)$data['storage_mtime']; - $data['encrypted'] = (bool)$data['encrypted']; - $data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']); - $data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']); - if ($data['storage_mtime'] === 0) { - $data['storage_mtime'] = $data['mtime']; - } - $data['size'] = (int)$data['size']; - $data['permissions'] = (int)$data['permissions']; - if (!is_int($file) || $file === 0) { - $data['path'] = ''; - $data['name'] = basename($this->storage->getMountPoint()); - $data['is_share_mount_point'] = true; - } - $data['permissions'] &= $this->storage->getPermissions(''); - return $data; - } - return false; - } + private $sourceRootInfo; /** - * get the metadata of all files stored in $folder - * - * @param string $folderId - * @return ICacheEntry[]|false + * @var \OCP\Files\Cache\ICache */ - public function getFolderContentsById($folderId) { - $cache = $this->getSourceCache(''); - if ($cache) { - $owner = $this->storage->getSharedFrom(); - $parentPath = $this->getPathById($folderId); - if ($parentPath !== '') { - $parentPath .= '/'; - } - $sourceFolderContent = $cache->getFolderContentsById($folderId); - foreach ($sourceFolderContent as &$c) { - $c['path'] = ltrim($parentPath . $c['name'], '/'); - $c['uid_owner'] = $owner; - $c['displayname_owner'] = \OC_User::getDisplayName($owner); - $c['permissions'] = $c['permissions'] & $this->storage->getPermissions(false); - } - - return $sourceFolderContent; - } - - return false; - } + private $sourceCache; /** - * store meta data for a file or folder - * - * @param string $file - * @param array $data - * - * @return int|false file id + * @param \OC\Files\Storage\Shared $storage + * @param IStorage $sourceStorage + * @param ICacheEntry $sourceRootInfo */ - public function put($file, array $data) { - $file = ($file === false) ? '' : $file; - if ($cache = $this->getSourceCache($file)) { - return $cache->put($this->files[$file], $data); - } - return false; + public function __construct($storage, IStorage $sourceStorage, ICacheEntry $sourceRootInfo) { + $this->storage = $storage; + $this->sourceStorage = $sourceStorage; + $this->sourceRootInfo = $sourceRootInfo; + $this->sourceCache = $sourceStorage->getCache(); + parent::__construct( + $this->sourceCache, + $this->sourceRootInfo->getPath() + ); } - /** - * get the file id for a file - * - * @param string $file - * @return int - */ - public function getId($file) { - if ($file === false) { - return $this->storage->getSourceId(); - } - $cache = $this->getSourceCache($file); - if ($cache) { - return $cache->getId($this->files[$file]); + public function getNumericStorageId() { + if (isset($this->numericId)) { + return $this->numericId; + } else { + return false; } - return -1; } - /** - * check if a file is available in the cache - * - * @param string $file - * @return bool - */ - public function inCache($file) { - if ($file == '') { - return true; + protected function formatCacheEntry($entry) { + $path = $entry['path']; + $entry = parent::formatCacheEntry($entry); + $sharePermissions = $this->storage->getPermissions($path); + if (isset($entry['permissions'])) { + $entry['permissions'] &= $sharePermissions; + } else { + $entry['permissions'] = $sharePermissions; } - return parent::inCache($file); - } - - /** - * remove a file or folder from the cache - * - * @param string $file - */ - public function remove($file) { - $file = ($file === false) ? '' : $file; - if ($cache = $this->getSourceCache($file)) { - $cache->remove($this->files[$file]); + $entry['uid_owner'] = $this->storage->getOwner($path); + $entry['displayname_owner'] = \OC_User::getDisplayName($entry['uid_owner']); + if ($path === '') { + $entry['is_share_mount_point'] = true; } - } - - /** - * Get the storage id and path needed for a move - * - * @param string $path - * @return array [$storageId, $internalPath] - */ - protected function getMoveInfo($path) { - $cache = $this->getSourceCache($path); - $file = \OC_Share_Backend_File::getSource($path, $this->storage->getShare()); - return [$cache->getNumericStorageId(), $file['path']]; + return $entry; } /** @@ -262,259 +109,4 @@ class Shared_Cache extends Cache { public function clear() { // Not a valid action for Shared Cache } - - /** - * @param string $file - * - * @return int, Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE - */ - public function getStatus($file) { - if ($file == '') { - return self::COMPLETE; - } - if ($cache = $this->getSourceCache($file)) { - return $cache->getStatus($this->files[$file]); - } - return self::NOT_FOUND; - } - - /** - * search for files matching $pattern - * - * @param string $pattern - * @return ICacheEntry[] of file data - */ - public function search($pattern) { - - $pattern = trim($pattern, '%'); - - $normalizedPattern = $this->normalize($pattern); - - $result = array(); - $exploreDirs = array(''); - while (count($exploreDirs) > 0) { - $dir = array_pop($exploreDirs); - $files = $this->getFolderContents($dir); - // no results? - if (!$files) { - // maybe it's a single shared file - $file = $this->get(''); - if ($normalizedPattern === '' || stristr($file['name'], $normalizedPattern) !== false) { - $result[] = $file; - } - continue; - } - foreach ($files as $file) { - if ($normalizedPattern === '' || stristr($file['name'], $normalizedPattern) !== false) { - $result[] = $file; - } - if ($file['mimetype'] === 'httpd/unix-directory') { - $exploreDirs[] = ltrim($dir . '/' . $file['name'], '/'); - } - } - } - return $result; - - } - - /** - * search for files by mimetype - * - * @param string $mimetype - * @return ICacheEntry[] - */ - public function searchByMime($mimetype) { - $mimepart = null; - if (strpos($mimetype, '/') === false) { - $mimepart = $mimetype; - $mimetype = null; - } - - $result = array(); - $exploreDirs = array(''); - while (count($exploreDirs) > 0) { - $dir = array_pop($exploreDirs); - $files = $this->getFolderContents($dir); - // no results? - if (!$files) { - // maybe it's a single shared file - $file = $this->get(''); - if (($mimepart && $file['mimepart'] === $mimepart) || ($mimetype && $file['mimetype'] === $mimetype)) { - $result[] = $file; - } - continue; - } - foreach ($files as $file) { - if ($file['mimetype'] === 'httpd/unix-directory') { - $exploreDirs[] = ltrim($dir . '/' . $file['name'], '/'); - } else if (($mimepart && $file['mimepart'] === $mimepart) || ($mimetype && $file['mimetype'] === $mimetype)) { - $result[] = $file; - } - } - } - return $result; - } - - /** - * Checks whether the given file has the given tag. - * - * @param \OCP\ITags $tagger - * @param array $fileData file data - * @param string $tag tag to check for - * @return boolean true if the given file has the expected tag, - * false otherwise - */ - private function hasTag($tagger, $fileData, $tag) { - $tags = $tagger->getTagsForObjects(array((int)$fileData['fileid'])); - return (!empty($tags) && in_array($tag, current($tags))); - } - - /** - * search for files by tag - * - * @param string|int $tag tag to search for - * @param string $userId owner of the tags - * @return ICacheEntry[] file data - */ - public function searchByTag($tag, $userId) { - // TODO: inject this - $tagger = \OC::$server->getTagManager()->load('files', null, null, $userId); - $result = array(); - $exploreDirs = array(''); - // check if root is tagged - $file = $this->get(''); - if ($this->hasTag($tagger, $file, $tag)) { - $result[] = $file; - } - // FIXME: this is so wrong and unefficient, need to replace with actual DB queries - while (count($exploreDirs) > 0) { - $dir = array_pop($exploreDirs); - $files = $this->getFolderContents($dir); - if (!$files) { - continue; - } - foreach ($files as $file) { - if ($this->hasTag($tagger, $file, $tag)) { - $result[] = $file; - } - if ($file['mimetype'] === 'httpd/unix-directory') { - $exploreDirs[] = ltrim($dir . '/' . $file['name'], '/'); - } - } - } - return $result; - } - - /** - * update the folder size and the size of all parent folders - * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder - */ - public function correctFolderSize($path, $data = null) { - $this->calculateFolderSize($path, $data); - if ($path !== '') { - $parent = dirname($path); - if ($parent === '.' or $parent === '/') { - $parent = ''; - } - $this->correctFolderSize($parent); - } else { - // bubble up to source cache - $sourceCache = $this->getSourceCache($path); - if (isset($this->files[$path])) { - $parent = dirname($this->files[$path]); - if ($sourceCache) { - $sourceCache->correctFolderSize($parent); - } - } - } - } - - /** - * get the size of a folder and set it in the cache - * - * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int - */ - public function calculateFolderSize($path, $entry = null) { - $path = ($path === false) ? '' : $path; - if ($cache = $this->getSourceCache($path)) { - return $cache->calculateFolderSize($this->files[$path]); - } - return 0; - } - - /** - * get all file ids on the files on the storage - * - * @return int[] - */ - public function getAll() { - $ids = \OCP\Share::getItemsSharedWith('file', \OC_Share_Backend_File::FORMAT_GET_ALL); - $folderBackend = \OCP\Share::getBackend('folder'); - if ($folderBackend instanceof Share_Backend_Collection) { - foreach ($ids as $file) { - /** @var $folderBackend Share_Backend_Collection */ - $children = $folderBackend->getChildren($file); - foreach ($children as $child) { - $ids[] = (int)$child['source']; - } - - } - } - - return $ids; - } - - /** - * find a folder in the cache which has not been fully scanned - * - * If multiply incomplete folders are in the cache, the one with the highest id will be returned, - * use the one with the highest id gives the best result with the background scanner, since that is most - * likely the folder where we stopped scanning previously - * - * @return boolean the path of the folder or false when no folder matched - */ - public function getIncomplete() { - return false; - } - - /** - * get the path of a file on this storage relative to the mount point by it's id - * - * @param int $id - * @param string $pathEnd (optional) used internally for recursive calls - * @return string|null - */ - public function getPathById($id, $pathEnd = '') { - // direct shares are easy - if ($id === $this->storage->getSourceId()) { - return ltrim($pathEnd, '/'); - } else { - // if the item is a direct share we try and get the path of the parent and append the name of the item to it - list($parent, $name) = $this->getParentInfo($id); - if ($parent > 0) { - return $this->getPathById($parent, '/' . $name . $pathEnd); - } else { - return null; - } - } - } - - /** - * @param integer $id - * @return array - */ - private function getParentInfo($id) { - $sql = 'SELECT `parent`, `name` FROM `*PREFIX*filecache` WHERE `fileid` = ?'; - $query = \OCP\DB::prepare($sql); - $result = $query->execute(array($id)); - if ($row = $result->fetchRow()) { - return array((int)$row['parent'], $row['name']); - } else { - return array(-1, ''); - } - } } diff --git a/apps/files_sharing/lib/sharedstorage.php b/apps/files_sharing/lib/sharedstorage.php index 542d0e9e48c..3ae5749ea87 100644 --- a/apps/files_sharing/lib/sharedstorage.php +++ b/apps/files_sharing/lib/sharedstorage.php @@ -32,6 +32,8 @@ namespace OC\Files\Storage; use OC\Files\Filesystem; use OCA\Files_Sharing\ISharedStorage; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Storage\IStorage; use OCP\Lock\ILockingProvider; /** @@ -41,7 +43,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { private $share; // the shared resource private $files = array(); - private static $isInitialized = array(); /** * @var \OC\Files\View @@ -55,6 +56,16 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { private $initialized = false; + /** + * @var ICacheEntry + */ + private $sourceRootInfo; + + /** + * @var IStorage + */ + private $sourceStorage; + public function __construct($arguments) { $this->share = $arguments['share']; $this->ownerView = $arguments['ownerView']; @@ -67,6 +78,9 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { } $this->initialized = true; Filesystem::initMountPoints($this->share['uid_owner']); + $sourcePath = $this->ownerView->getPath($this->share['file_source']); + list($this->sourceStorage, $sourceInternalPath) = $this->ownerView->resolvePath($sourcePath); + $this->sourceRootInfo = $this->sourceStorage->getCache()->get($sourceInternalPath); } /** @@ -543,7 +557,7 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { if (!$storage) { $storage = $this; } - return new \OC\Files\Cache\Shared_Cache($storage); + return new \OC\Files\Cache\Shared_Cache($storage, $this->sourceStorage, $this->sourceRootInfo); } public function getScanner($path = '', $storage = null) { @@ -553,13 +567,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { return new \OC\Files\Cache\SharedScanner($storage); } - public function getWatcher($path = '', $storage = null) { - if (!$storage) { - $storage = $this; - } - return new \OC\Files\Cache\Shared_Watcher($storage); - } - public function getPropagator($storage = null) { if (!$storage) { $storage = $this; diff --git a/apps/files_sharing/lib/watcher.php b/apps/files_sharing/lib/watcher.php deleted file mode 100644 index 5b4736c21e1..00000000000 --- a/apps/files_sharing/lib/watcher.php +++ /dev/null @@ -1,81 +0,0 @@ - - * @author Michael Gapczynski - * @author Morris Jobke - * @author Robin Appelman - * @author Vincent Petry - * - * @copyright Copyright (c) 2016, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Files\Cache; -use OCP\Files\Cache\ICacheEntry; - -/** - * check the storage backends for updates and change the cache accordingly - */ -class Shared_Watcher extends Watcher { - /** - * @var \OC\Files\Storage\Shared $storage - */ - protected $storage; - - /** - * Update the cache for changes to $path - * - * @param string $path - * @param ICacheEntry $cachedData - */ - public function update($path, $cachedData) { - parent::update($path, $cachedData); - // since parent::update() has already updated the size of the subdirs, - // only apply the update to the owner's parent dirs - - // find last parent before reaching the shared storage root, - // which is the actual shared dir from the owner - $sepPos = strpos($path, '/'); - if ($sepPos > 0) { - $baseDir = substr($path, 0, $sepPos); - } else { - $baseDir = $path; - } - - // find the path relative to the data dir - $file = $this->storage->getFile($baseDir); - $view = new \OC\Files\View('/' . $file['fileOwner']); - - // find the owner's storage and path - /** @var \OC\Files\Storage\Storage $storage */ - list($storage, $internalPath) = $view->resolvePath($file['path']); - - // update the parent dirs' sizes in the owner's cache - $storage->getCache()->correctFolderSize(dirname($internalPath)); - } - - /** - * remove deleted files in $path from the cache - * - * @param string $path - */ - public function cleanFolder($path) { - if ($path != '') { - parent::cleanFolder($path); - } - } - -} diff --git a/apps/files_sharing/tests/testcase.php b/apps/files_sharing/tests/testcase.php index a9da39f446b..ce0a8beeec8 100644 --- a/apps/files_sharing/tests/testcase.php +++ b/apps/files_sharing/tests/testcase.php @@ -174,9 +174,9 @@ abstract class TestCase extends \Test\TestCase { */ protected static function resetStorage() { $storage = new \ReflectionClass('\OC\Files\Storage\Shared'); - $isInitialized = $storage->getProperty('isInitialized'); + $isInitialized = $storage->getProperty('initialized'); $isInitialized->setAccessible(true); - $isInitialized->setValue(array()); + $isInitialized->setValue($storage, false); $isInitialized->setAccessible(false); } diff --git a/apps/files_versions/tests/versions.php b/apps/files_versions/tests/versions.php index ac922b74b9f..b85877cebd3 100644 --- a/apps/files_versions/tests/versions.php +++ b/apps/files_versions/tests/versions.php @@ -802,9 +802,9 @@ class Test_Files_Versioning extends \Test\TestCase { } $storage = new \ReflectionClass('\OC\Files\Storage\Shared'); - $isInitialized = $storage->getProperty('isInitialized'); + $isInitialized = $storage->getProperty('initialized'); $isInitialized->setAccessible(true); - $isInitialized->setValue(array()); + $isInitialized->setValue($storage, false); $isInitialized->setAccessible(false); \OC_Util::tearDownFS(); diff --git a/lib/private/files/cache/wrapper/cachejail.php b/lib/private/files/cache/wrapper/cachejail.php index 4d708c2d191..4188fb72622 100644 --- a/lib/private/files/cache/wrapper/cachejail.php +++ b/lib/private/files/cache/wrapper/cachejail.php @@ -34,7 +34,7 @@ class CacheJail extends CacheWrapper { protected $root; /** - * @param \OC\Files\Cache\Cache $cache + * @param \OCP\Files\Cache\ICache $cache * @param string $root */ public function __construct($cache, $root) { diff --git a/lib/private/files/cache/wrapper/cachepermissionsmask.php b/lib/private/files/cache/wrapper/cachepermissionsmask.php index 1eeb3c4783d..b3a7bcb3a73 100644 --- a/lib/private/files/cache/wrapper/cachepermissionsmask.php +++ b/lib/private/files/cache/wrapper/cachepermissionsmask.php @@ -29,7 +29,7 @@ class CachePermissionsMask extends CacheWrapper { protected $mask; /** - * @param \OC\Files\Cache\Cache $cache + * @param \OCP\Files\Cache\ICache $cache * @param int $mask */ public function __construct($cache, $mask) { diff --git a/lib/private/files/cache/wrapper/cachewrapper.php b/lib/private/files/cache/wrapper/cachewrapper.php index 39a6b63205b..1ce4f028c75 100644 --- a/lib/private/files/cache/wrapper/cachewrapper.php +++ b/lib/private/files/cache/wrapper/cachewrapper.php @@ -26,15 +26,16 @@ namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\ICache; class CacheWrapper extends Cache { /** - * @var \OC\Files\Cache\Cache + * @var \OCP\Files\Cache\ICache */ protected $cache; /** - * @param \OC\Files\Cache\Cache $cache + * @param \OCP\Files\Cache\ICache $cache */ public function __construct($cache) { $this->cache = $cache; @@ -159,6 +160,10 @@ class CacheWrapper extends Cache { $this->cache->move($source, $target); } + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $this->cache->moveFromCache($sourceCache, $sourcePath, $targetPath); + } + /** * remove all entries for files that are stored on the storage from the cache */ -- 2.39.5