diff options
Diffstat (limited to 'lib/private/Files/Cache')
| -rw-r--r-- | lib/private/Files/Cache/Cache.php | 837 | ||||
| -rw-r--r-- | lib/private/Files/Cache/CacheEntry.php | 114 | ||||
| -rw-r--r-- | lib/private/Files/Cache/FailedCache.php | 142 | ||||
| -rw-r--r-- | lib/private/Files/Cache/HomeCache.php | 86 | ||||
| -rw-r--r-- | lib/private/Files/Cache/HomePropagator.php | 50 | ||||
| -rw-r--r-- | lib/private/Files/Cache/MoveFromCacheTrait.php | 87 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Propagator.php | 74 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Scanner.php | 503 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Storage.php | 189 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Updater.php | 228 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Watcher.php | 140 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/CacheJail.php | 300 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/CachePermissionsMask.php | 46 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/CacheWrapper.php | 309 |
14 files changed, 3105 insertions, 0 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php new file mode 100644 index 00000000000..53467c278d2 --- /dev/null +++ b/lib/private/Files/Cache/Cache.php @@ -0,0 +1,837 @@ +<?php +/** + * @author Andreas Fischer <bantu@owncloud.com> + * @author Björn Schießle <schiessle@owncloud.com> + * @author Florin Peter <github@florin-peter.de> + * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch> + * @author Joas Schilling <nickvergessen@owncloud.com> + * @author Jörn Friedrich Dreyer <jfd@butonic.de> + * @author Lukas Reschke <lukas@owncloud.com> + * @author Michael Gapczynski <GapczynskiM@gmail.com> + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Robin McCorkell <robin@mccorkell.me.uk> + * @author Roeland Jago Douma <rullzer@owncloud.com> + * @author TheSFReader <TheSFReader@gmail.com> + * @author Thomas Müller <thomas.mueller@tmit.eu> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use \OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; + +/** + * Metadata cache for a storage + * + * The cache stores the metadata for all files and folders in a storage and is kept up to date trough the following mechanisms: + * + * - Scanner: scans the storage and updates the cache where needed + * - Watcher: checks for changes made to the filesystem outside of the ownCloud instance and rescans files and folder when a change is detected + * - Updater: listens to changes made to the filesystem inside of the ownCloud instance and updates the cache where needed + * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater + */ +class Cache implements ICache { + use MoveFromCacheTrait { + MoveFromCacheTrait::moveFromCache as moveFromCacheFallback; + } + + /** + * @var array partial data for the cache + */ + protected $partial = array(); + + /** + * @var string + */ + protected $storageId; + + /** + * @var Storage $storageCache + */ + protected $storageCache; + + /** @var IMimeTypeLoader */ + protected $mimetypeLoader; + + /** + * @var IDBConnection + */ + protected $connection; + + /** + * @param \OC\Files\Storage\Storage|string $storage + */ + public function __construct($storage) { + if ($storage instanceof \OC\Files\Storage\Storage) { + $this->storageId = $storage->getId(); + } else { + $this->storageId = $storage; + } + if (strlen($this->storageId) > 64) { + $this->storageId = md5($this->storageId); + } + + $this->storageCache = new Storage($storage); + $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); + $this->connection = \OC::$server->getDatabaseConnection(); + } + + /** + * Get the numeric storage id for this cache's storage + * + * @return int + */ + public function getNumericStorageId() { + return $this->storageCache->getNumericId(); + } + + /** + * get the stored metadata of a file or folder + * + * @param string | int $file either the path of a file or folder or the file id for a file or folder + * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache + */ + public function get($file) { + if (is_string($file) or $file == '') { + // normalize file + $file = $this->normalize($file); + + $where = 'WHERE `storage` = ? AND `path_hash` = ?'; + $params = array($this->getNumericStorageId(), md5($file)); + } else { //file id + $where = 'WHERE `fileid` = ?'; + $params = array($file); + } + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, + `storage_mtime`, `encrypted`, `etag`, `permissions`, `checksum` + FROM `*PREFIX*filecache` ' . $where; + $result = $this->connection->executeQuery($sql, $params); + $data = $result->fetch(); + + //FIXME hide this HACK in the next database layer, or just use doctrine and get rid of MDB2 and PDO + //PDO returns false, MDB2 returns null, oracle always uses MDB2, so convert null to false + if ($data === null) { + $data = false; + } + + //merge partial data + if (!$data and is_string($file)) { + if (isset($this->partial[$file])) { + $data = $this->partial[$file]; + } + return $data; + } else { + //fix types + $data['fileid'] = (int)$data['fileid']; + $data['parent'] = (int)$data['parent']; + $data['size'] = 0 + $data['size']; + $data['mtime'] = (int)$data['mtime']; + $data['storage_mtime'] = (int)$data['storage_mtime']; + $data['encryptedVersion'] = (int)$data['encrypted']; + $data['encrypted'] = (bool)$data['encrypted']; + $data['storage'] = $this->storageId; + $data['mimetype'] = $this->mimetypeLoader->getMimetypeById($data['mimetype']); + $data['mimepart'] = $this->mimetypeLoader->getMimetypeById($data['mimepart']); + if ($data['storage_mtime'] == 0) { + $data['storage_mtime'] = $data['mtime']; + } + $data['permissions'] = (int)$data['permissions']; + return new CacheEntry($data); + } + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return ICacheEntry[] + */ + public function getFolderContentsById($fileId) { + if ($fileId > -1) { + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, + `storage_mtime`, `encrypted`, `etag`, `permissions`, `checksum` + FROM `*PREFIX*filecache` WHERE `parent` = ? ORDER BY `name` ASC'; + $result = $this->connection->executeQuery($sql, [$fileId]); + $files = $result->fetchAll(); + foreach ($files as &$file) { + $file['mimetype'] = $this->mimetypeLoader->getMimetypeById($file['mimetype']); + $file['mimepart'] = $this->mimetypeLoader->getMimetypeById($file['mimepart']); + if ($file['storage_mtime'] == 0) { + $file['storage_mtime'] = $file['mtime']; + } + $file['permissions'] = (int)$file['permissions']; + $file['mtime'] = (int)$file['mtime']; + $file['storage_mtime'] = (int)$file['storage_mtime']; + $file['size'] = 0 + $file['size']; + } + return array_map(function (array $data) { + return new CacheEntry($data); + }, $files); + } else { + return array(); + } + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + // normalize file + $file = $this->normalize($file); + + if (isset($this->partial[$file])) { //add any saved partial data + $data = array_merge($this->partial[$file], $data); + unset($this->partial[$file]); + } + + $requiredFields = array('size', 'mtime', 'mimetype'); + foreach ($requiredFields as $field) { + if (!isset($data[$field])) { //data not complete save as partial and return + $this->partial[$file] = $data; + return -1; + } + } + + $data['path'] = $file; + $data['parent'] = $this->getParentId($file); + $data['name'] = \OC_Util::basename($file); + + list($queryParts, $params) = $this->buildParts($data); + $queryParts[] = '`storage`'; + $params[] = $this->getNumericStorageId(); + + $queryParts = array_map(function ($item) { + return trim($item, "`"); + }, $queryParts); + $values = array_combine($queryParts, $params); + if (\OC::$server->getDatabaseConnection()->insertIfNotExist('*PREFIX*filecache', $values, [ + 'storage', + 'path_hash', + ]) + ) { + return (int)$this->connection->lastInsertId('*PREFIX*filecache'); + } + + // The file was created in the mean time + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + throw new \RuntimeException('File entry could not be inserted with insertIfNotExist() but could also not be selected with getId() in order to perform an update. Please try again.'); + } + } + + /** + * update the metadata of an existing file or folder in the cache + * + * @param int $id the fileid of the existing file or folder + * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged + */ + public function update($id, array $data) { + + if (isset($data['path'])) { + // normalize path + $data['path'] = $this->normalize($data['path']); + } + + if (isset($data['name'])) { + // normalize path + $data['name'] = $this->normalize($data['name']); + } + + list($queryParts, $params) = $this->buildParts($data); + // duplicate $params because we need the parts twice in the SQL statement + // once for the SET part, once in the WHERE clause + $params = array_merge($params, $params); + $params[] = $id; + + // don't update if the data we try to set is the same as the one in the record + // some databases (Postgres) don't like superfluous updates + $sql = 'UPDATE `*PREFIX*filecache` SET ' . implode(' = ?, ', $queryParts) . '=? ' . + 'WHERE (' . + implode(' <> ? OR ', $queryParts) . ' <> ? OR ' . + implode(' IS NULL OR ', $queryParts) . ' IS NULL' . + ') AND `fileid` = ? '; + $this->connection->executeQuery($sql, $params); + + } + + /** + * extract query parts and params array from data array + * + * @param array $data + * @return array [$queryParts, $params] + * $queryParts: string[], the (escaped) column names to be set in the query + * $params: mixed[], the new values for the columns, to be passed as params to the query + */ + protected function buildParts(array $data) { + $fields = array( + 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', + 'etag', 'permissions', 'checksum'); + + $doNotCopyStorageMTime = false; + if (array_key_exists('mtime', $data) && $data['mtime'] === null) { + // this horrific magic tells it to not copy storage_mtime to mtime + unset($data['mtime']); + $doNotCopyStorageMTime = true; + } + + $params = array(); + $queryParts = array(); + foreach ($data as $name => $value) { + if (array_search($name, $fields) !== false) { + if ($name === 'path') { + $params[] = md5($value); + $queryParts[] = '`path_hash`'; + } elseif ($name === 'mimetype') { + $params[] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/'))); + $queryParts[] = '`mimepart`'; + $value = $this->mimetypeLoader->getId($value); + } elseif ($name === 'storage_mtime') { + if (!$doNotCopyStorageMTime && !isset($data['mtime'])) { + $params[] = $value; + $queryParts[] = '`mtime`'; + } + } elseif ($name === 'encrypted') { + if(isset($data['encryptedVersion'])) { + $value = $data['encryptedVersion']; + } else { + // Boolean to integer conversion + $value = $value ? 1 : 0; + } + } + $params[] = $value; + $queryParts[] = '`' . $name . '`'; + } + } + return array($queryParts, $params); + } + + /** + * get the file id for a file + * + * A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file + * + * File ids are easiest way for apps to store references to a file since unlike paths they are not affected by renames or sharing + * + * @param string $file + * @return int + */ + public function getId($file) { + // normalize file + $file = $this->normalize($file); + + $pathHash = md5($file); + + $sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'; + $result = $this->connection->executeQuery($sql, array($this->getNumericStorageId(), $pathHash)); + if ($row = $result->fetch()) { + return $row['fileid']; + } else { + return -1; + } + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + if ($file === '') { + return -1; + } else { + $parent = $this->getParentPath($file); + return (int)$this->getId($parent); + } + } + + private function getParentPath($path) { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + return $parent; + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getId($file) != -1; + } + + /** + * remove a file or folder from the cache + * + * when removing a folder from the cache all files and folders inside the folder will be removed as well + * + * @param string $file + */ + public function remove($file) { + $entry = $this->get($file); + $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `fileid` = ?'; + $this->connection->executeQuery($sql, array($entry['fileid'])); + if ($entry['mimetype'] === 'httpd/unix-directory') { + $this->removeChildren($entry); + } + } + + /** + * Get all sub folders of a folder + * + * @param array $entry the cache entry of the folder to get the subfolders for + * @return array[] the cache entries for the subfolders + */ + private function getSubFolders($entry) { + $children = $this->getFolderContentsById($entry['fileid']); + return array_filter($children, function ($child) { + return $child['mimetype'] === 'httpd/unix-directory'; + }); + } + + /** + * Recursively remove all children of a folder + * + * @param array $entry the cache entry of the folder to remove the children of + * @throws \OC\DatabaseException + */ + private function removeChildren($entry) { + $subFolders = $this->getSubFolders($entry); + foreach ($subFolders as $folder) { + $this->removeChildren($folder); + } + $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `parent` = ?'; + $this->connection->executeQuery($sql, array($entry['fileid'])); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->moveFromCache($this, $source, $target); + } + + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $path]; + } + + /** + * Move a file or folder in the cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + * @throws \OC\DatabaseException + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache instanceof Cache) { + // normalize source and target + $sourcePath = $this->normalize($sourcePath); + $targetPath = $this->normalize($targetPath); + + $sourceData = $sourceCache->get($sourcePath); + $sourceId = $sourceData['fileid']; + $newParentId = $this->getParentId($targetPath); + + list($sourceStorageId, $sourcePath) = $sourceCache->getMoveInfo($sourcePath); + list($targetStorageId, $targetPath) = $this->getMoveInfo($targetPath); + + // sql for final update + $moveSql = 'UPDATE `*PREFIX*filecache` SET `storage` = ?, `path` = ?, `path_hash` = ?, `name` = ?, `parent` =? WHERE `fileid` = ?'; + + if ($sourceData['mimetype'] === 'httpd/unix-directory') { + //find all child entries + $sql = 'SELECT `path`, `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path` LIKE ?'; + $result = $this->connection->executeQuery($sql, [$sourceStorageId, $this->connection->escapeLikeParameter($sourcePath) . '/%']); + $childEntries = $result->fetchAll(); + $sourceLength = strlen($sourcePath); + $this->connection->beginTransaction(); + $query = $this->connection->prepare('UPDATE `*PREFIX*filecache` SET `storage` = ?, `path` = ?, `path_hash` = ? WHERE `fileid` = ?'); + + foreach ($childEntries as $child) { + $newTargetPath = $targetPath . substr($child['path'], $sourceLength); + $query->execute([$targetStorageId, $newTargetPath, md5($newTargetPath), $child['fileid']]); + } + $this->connection->executeQuery($moveSql, [$targetStorageId, $targetPath, md5($targetPath), basename($targetPath), $newParentId, $sourceId]); + $this->connection->commit(); + } else { + $this->connection->executeQuery($moveSql, [$targetStorageId, $targetPath, md5($targetPath), basename($targetPath), $newParentId, $sourceId]); + } + } else { + $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath); + } + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `storage` = ?'; + $this->connection->executeQuery($sql, array($this->getNumericStorageId())); + + $sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?'; + $this->connection->executeQuery($sql, array($this->storageId)); + } + + /** + * Get the scan status of a file + * + * - Cache::NOT_FOUND: File is not in the cache + * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known + * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned + * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned + * + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + // normalize file + $file = $this->normalize($file); + + $pathHash = md5($file); + $sql = 'SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'; + $result = $this->connection->executeQuery($sql, array($this->getNumericStorageId(), $pathHash)); + if ($row = $result->fetch()) { + if ((int)$row['size'] === -1) { + return self::SHALLOW; + } else { + return self::COMPLETE; + } + } else { + if (isset($this->partial[$file])) { + return self::PARTIAL; + } else { + return self::NOT_FOUND; + } + } + } + + /** + * search for files matching $pattern + * + * @param string $pattern the search pattern using SQL search syntax (e.g. '%searchstring%') + * @return ICacheEntry[] an array of cache entries where the name matches the search pattern + */ + public function search($pattern) { + // normalize pattern + $pattern = $this->normalize($pattern); + + + $sql = ' + SELECT `fileid`, `storage`, `path`, `parent`, `name`, + `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, + `etag`, `permissions`, `checksum` + FROM `*PREFIX*filecache` + WHERE `storage` = ? AND `name` ILIKE ?'; + $result = $this->connection->executeQuery($sql, + [$this->getNumericStorageId(), $pattern] + ); + + $files = []; + while ($row = $result->fetch()) { + $row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']); + $row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']); + $files[] = $row; + } + return array_map(function(array $data) { + return new CacheEntry($data); + }, $files); + } + + /** + * search for files by mimetype + * + * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') + * where it will search for all mimetypes in the group ('image/*') + * @return ICacheEntry[] an array of cache entries where the mimetype matches the search + */ + public function searchByMime($mimetype) { + if (strpos($mimetype, '/')) { + $where = '`mimetype` = ?'; + } else { + $where = '`mimepart` = ?'; + } + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`, `permissions`, `checksum` + FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?'; + $mimetype = $this->mimetypeLoader->getId($mimetype); + $result = $this->connection->executeQuery($sql, array($mimetype, $this->getNumericStorageId())); + $files = array(); + while ($row = $result->fetch()) { + $row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']); + $row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']); + $files[] = $row; + } + return array_map(function (array $data) { + return new CacheEntry($data); + }, $files); + } + + /** + * Search for files by tag of a given users. + * + * Note that every user can tag files differently. + * + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return ICacheEntry[] file data + */ + public function searchByTag($tag, $userId) { + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, ' . + '`mimetype`, `mimepart`, `size`, `mtime`, ' . + '`encrypted`, `etag`, `permissions`, `checksum` ' . + 'FROM `*PREFIX*filecache` `file`, ' . + '`*PREFIX*vcategory_to_object` `tagmap`, ' . + '`*PREFIX*vcategory` `tag` ' . + // JOIN filecache to vcategory_to_object + 'WHERE `file`.`fileid` = `tagmap`.`objid` ' . + // JOIN vcategory_to_object to vcategory + 'AND `tagmap`.`type` = `tag`.`type` ' . + 'AND `tagmap`.`categoryid` = `tag`.`id` ' . + // conditions + 'AND `file`.`storage` = ? ' . + 'AND `tag`.`type` = \'files\' ' . + 'AND `tag`.`uid` = ? '; + if (is_int($tag)) { + $sql .= 'AND `tag`.`id` = ? '; + } else { + $sql .= 'AND `tag`.`category` = ? '; + } + $result = $this->connection->executeQuery( + $sql, + [ + $this->getNumericStorageId(), + $userId, + $tag + ] + ); + $files = array(); + while ($row = $result->fetch()) { + $files[] = $row; + } + return array_map(function (array $data) { + return new CacheEntry($data); + }, $files); + } + + /** + * Re-calculate the folder size and the size of all parent folders + * + * @param string|boolean $path + * @param array $data (optional) meta data of the folder + */ + public function correctFolderSize($path, $data = null) { + $this->calculateFolderSize($path, $data); + if ($path !== '') { + $parent = dirname($path); + if ($parent === '.' or $parent === '/') { + $parent = ''; + } + $this->correctFolderSize($parent); + } + } + + /** + * calculate the size of a folder and set it in the cache + * + * @param string $path + * @param array $entry (optional) meta data of the folder + * @return int + */ + public function calculateFolderSize($path, $entry = null) { + $totalSize = 0; + if (is_null($entry) or !isset($entry['fileid'])) { + $entry = $this->get($path); + } + if (isset($entry['mimetype']) && $entry['mimetype'] === 'httpd/unix-directory') { + $id = $entry['fileid']; + $sql = 'SELECT SUM(`size`) AS f1, MIN(`size`) AS f2 ' . + 'FROM `*PREFIX*filecache` ' . + 'WHERE `parent` = ? AND `storage` = ?'; + $result = $this->connection->executeQuery($sql, array($id, $this->getNumericStorageId())); + if ($row = $result->fetch()) { + $result->closeCursor(); + list($sum, $min) = array_values($row); + $sum = 0 + $sum; + $min = 0 + $min; + if ($min === -1) { + $totalSize = $min; + } else { + $totalSize = $sum; + } + $update = array(); + if ($entry['size'] !== $totalSize) { + $update['size'] = $totalSize; + } + if (count($update) > 0) { + $this->update($id, $update); + } + } else { + $result->closeCursor(); + } + } + return $totalSize; + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + $sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ?'; + $result = $this->connection->executeQuery($sql, array($this->getNumericStorageId())); + $ids = array(); + while ($row = $result->fetch()) { + $ids[] = $row['fileid']; + } + return $ids; + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|bool the path of the folder or false when no folder matched + */ + public function getIncomplete() { + $query = $this->connection->prepare('SELECT `path` FROM `*PREFIX*filecache`' + . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC', 1); + $query->execute([$this->getNumericStorageId()]); + if ($row = $query->fetch()) { + return $row['path']; + } else { + return false; + } + } + + /** + * get the path of a file on this storage by it's file id + * + * @param int $id the file id of the file or folder to search + * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache + */ + public function getPathById($id) { + $sql = 'SELECT `path` FROM `*PREFIX*filecache` WHERE `fileid` = ? AND `storage` = ?'; + $result = $this->connection->executeQuery($sql, array($id, $this->getNumericStorageId())); + if ($row = $result->fetch()) { + // Oracle stores empty strings as null... + if ($row['path'] === null) { + return ''; + } + return $row['path']; + } else { + return null; + } + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @deprecated use getPathById() instead + * @return array first element holding the storage id, second the path + */ + static public function getById($id) { + $connection = \OC::$server->getDatabaseConnection(); + $sql = 'SELECT `storage`, `path` FROM `*PREFIX*filecache` WHERE `fileid` = ?'; + $result = $connection->executeQuery($sql, array($id)); + if ($row = $result->fetch()) { + $numericId = $row['storage']; + $path = $row['path']; + } else { + return null; + } + + if ($id = Storage::getStorageId($numericId)) { + return array($id, $path); + } else { + return null; + } + } + + /** + * normalize the given path + * + * @param string $path + * @return string + */ + public function normalize($path) { + + return trim(\OC_Util::normalizeUnicode($path), '/'); + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php new file mode 100644 index 00000000000..6d3c5d5b089 --- /dev/null +++ b/lib/private/Files/Cache/CacheEntry.php @@ -0,0 +1,114 @@ +<?php +/** + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +/** + * meta data for a file or folder + */ +class CacheEntry implements ICacheEntry, \ArrayAccess { + /** + * @var array + */ + private $data; + + public function __construct(array $data) { + $this->data = $data; + } + + public function offsetSet($offset, $value) { + $this->data[$offset] = $value; + } + + public function offsetExists($offset) { + return isset($this->data[$offset]); + } + + public function offsetUnset($offset) { + unset($this->data[$offset]); + } + + public function offsetGet($offset) { + if (isset($this->data[$offset])) { + return $this->data[$offset]; + } else { + return null; + } + } + + public function getId() { + return (int)$this->data['fileid']; + } + + public function getStorageId() { + return $this->data['storage']; + } + + + public function getPath() { + return $this->data['path']; + } + + + public function getName() { + return $this->data['name']; + } + + + public function getMimeType() { + return $this->data['mimetype']; + } + + + public function getMimePart() { + return $this->data['mimepart']; + } + + public function getSize() { + return $this->data['size']; + } + + public function getMTime() { + return $this->data['mtime']; + } + + public function getStorageMTime() { + return $this->data['storage_mtime']; + } + + public function getEtag() { + return $this->data['etag']; + } + + public function getPermissions() { + return $this->data['permissions']; + } + + public function isEncrypted() { + return isset($this->data['encrypted']) && $this->data['encrypted']; + } + + public function getData() { + return $this->data; + } +} diff --git a/lib/private/Files/Cache/FailedCache.php b/lib/private/Files/Cache/FailedCache.php new file mode 100644 index 00000000000..0386ba3ca32 --- /dev/null +++ b/lib/private/Files/Cache/FailedCache.php @@ -0,0 +1,142 @@ +<?php +/** + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Constants; +use OCP\Files\Cache\ICache; + +/** + * Storage placeholder to represent a missing precondition, storage unavailable + */ +class FailedCache implements ICache { + /** @var bool whether to show the failed storage in the ui */ + private $visible; + + /** + * FailedCache constructor. + * + * @param bool $visible + */ + public function __construct($visible = true) { + $this->visible = $visible; + } + + + public function getNumericStorageId() { + return -1; + } + + public function get($file) { + if ($file === '') { + return new CacheEntry([ + 'fileid' => -1, + 'size' => 0, + 'mimetype' => 'httpd/unix-directory', + 'mimepart' => 'httpd', + 'permissions' => $this->visible ? Constants::PERMISSION_READ : 0, + 'mtime' => time() + ]); + } else { + return false; + } + } + + public function getFolderContents($folder) { + return []; + } + + public function getFolderContentsById($fileId) { + return []; + } + + public function put($file, array $data) { + return; + } + + public function insert($file, array $data) { + return; + } + + public function update($id, array $data) { + return; + } + + public function getId($file) { + return -1; + } + + public function getParentId($file) { + return -1; + } + + public function inCache($file) { + return false; + } + + public function remove($file) { + return; + } + + public function move($source, $target) { + return; + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + return; + } + + public function clear() { + return; + } + + public function getStatus($file) { + return ICache::NOT_FOUND; + } + + public function search($pattern) { + return []; + } + + public function searchByMime($mimetype) { + return []; + } + + public function searchByTag($tag, $userId) { + return []; + } + + public function getAll() { + return []; + } + + public function getIncomplete() { + return []; + } + + public function getPathById($id) { + return null; + } + + public function normalize($path) { + return $path; + } +} diff --git a/lib/private/Files/Cache/HomeCache.php b/lib/private/Files/Cache/HomeCache.php new file mode 100644 index 00000000000..ae92504ddd6 --- /dev/null +++ b/lib/private/Files/Cache/HomeCache.php @@ -0,0 +1,86 @@ +<?php +/** + * @author Andreas Fischer <bantu@owncloud.com> + * @author Björn Schießle <schiessle@owncloud.com> + * @author Jörn Friedrich Dreyer <jfd@butonic.de> + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +class HomeCache extends Cache { + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array $entry (optional) meta data of the folder + * @return int + */ + public function calculateFolderSize($path, $entry = null) { + if ($path !== '/' and $path !== '' and $path !== 'files' and $path !== 'files_trashbin' and $path !== 'files_versions') { + return parent::calculateFolderSize($path, $entry); + } elseif ($path === '' or $path === '/') { + // since the size of / isn't used (the size of /files is used instead) there is no use in calculating it + return 0; + } + + $totalSize = 0; + if (is_null($entry)) { + $entry = $this->get($path); + } + if ($entry && $entry['mimetype'] === 'httpd/unix-directory') { + $id = $entry['fileid']; + $sql = 'SELECT SUM(`size`) AS f1 ' . + 'FROM `*PREFIX*filecache` ' . + 'WHERE `parent` = ? AND `storage` = ? AND `size` >= 0'; + $result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId())); + if ($row = $result->fetchRow()) { + $result->closeCursor(); + list($sum) = array_values($row); + $totalSize = 0 + $sum; + $entry['size'] += 0; + if ($entry['size'] !== $totalSize) { + $this->update($id, array('size' => $totalSize)); + } + } + } + return $totalSize; + } + + /** + * @param string $path + * @return ICacheEntry + */ + public function get($path) { + $data = parent::get($path); + if ($path === '' or $path === '/') { + // only the size of the "files" dir counts + $filesData = parent::get('files'); + + if (isset($filesData['size'])) { + $data['size'] = $filesData['size']; + } + } + return $data; + } +} diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php new file mode 100644 index 00000000000..8edca9c0c87 --- /dev/null +++ b/lib/private/Files/Cache/HomePropagator.php @@ -0,0 +1,50 @@ +<?php +/** + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +class HomePropagator extends Propagator { + private $ignoredBaseFolders; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + parent::__construct($storage); + $this->ignoredBaseFolders = ['files_encryption']; + } + + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + * @return array[] all propagated entries + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + list($baseFolder) = explode('/', $internalPath, 2); + if (in_array($baseFolder, $this->ignoredBaseFolders)) { + return []; + } else { + return parent::propagateChange($internalPath, $time, $sizeDifference); + } + } +} diff --git a/lib/private/Files/Cache/MoveFromCacheTrait.php b/lib/private/Files/Cache/MoveFromCacheTrait.php new file mode 100644 index 00000000000..7d8ed7b5d21 --- /dev/null +++ b/lib/private/Files/Cache/MoveFromCacheTrait.php @@ -0,0 +1,87 @@ +<?php +/** + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; + +/** + * Fallback implementation for moveFromCache + */ +trait MoveFromCacheTrait { + /** + * store meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + abstract public function put($file, array $data); + + /** + * Move a file or folder in the cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $sourceEntry = $sourceCache->get($sourcePath); + + $this->copyFromCache($sourceCache, $sourceEntry, $targetPath); + + $sourceCache->remove($sourcePath); + } + + /** + * Copy a file or folder in the cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param ICacheEntry $sourceEntry + * @param string $targetPath + */ + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath) { + $this->put($targetPath, $this->cacheEntryToArray($sourceEntry)); + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); + foreach ($folderContent as $subEntry) { + $subTargetPath = $targetPath . '/' . $subEntry->getName(); + $this->copyFromCache($sourceCache, $subEntry, $subTargetPath); + } + } + } + + private function cacheEntryToArray(ICacheEntry $entry) { + return [ + 'size' => $entry->getSize(), + 'mtime' => $entry->getMTime(), + 'storage_mtime' => $entry->getStorageMTime(), + 'mimetype' => $entry->getMimeType(), + 'mimepart' => $entry->getMimePart(), + 'etag' => $entry->getEtag(), + 'permissions' => $entry->getPermissions(), + 'encrypted' => $entry->isEncrypted() + ]; + } +} diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php new file mode 100644 index 00000000000..50264e54d44 --- /dev/null +++ b/lib/private/Files/Cache/Propagator.php @@ -0,0 +1,74 @@ +<?php +/** + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OCP\Files\Cache\IPropagator; + +/** + * Propagate etags and mtimes within the storage + */ +class Propagator implements IPropagator { + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + } + + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + * @return array[] all propagated entries + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + $cache = $this->storage->getCache($internalPath); + + $parentId = $cache->getParentId($internalPath); + $propagatedEntries = []; + while ($parentId !== -1) { + $entry = $cache->get($parentId); + $propagatedEntries[] = $entry; + if (!$entry) { + return $propagatedEntries; + } + $mtime = max($time, $entry['mtime']); + + if ($entry['size'] === -1) { + $newSize = -1; + } else { + $newSize = $entry['size'] + $sizeDifference; + } + $cache->update($parentId, ['mtime' => $mtime, 'etag' => $this->storage->getETag($entry['path']), 'size' => $newSize]); + + $parentId = $entry['parent']; + } + + return $propagatedEntries; + } +} diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php new file mode 100644 index 00000000000..8730707f1c2 --- /dev/null +++ b/lib/private/Files/Cache/Scanner.php @@ -0,0 +1,503 @@ +<?php +/** + * @author Arthur Schiwon <blizzz@owncloud.com> + * @author Björn Schießle <schiessle@owncloud.com> + * @author Joas Schilling <nickvergessen@owncloud.com> + * @author Jörn Friedrich Dreyer <jfd@butonic.de> + * @author Lukas Reschke <lukas@owncloud.com> + * @author Martin Mattel <martin.mattel@diemattels.at> + * @author Michael Gapczynski <GapczynskiM@gmail.com> + * @author Morris Jobke <hey@morrisjobke.de> + * @author Owen Winkler <a_github@midnightcircus.com> + * @author Robin Appelman <icewind@owncloud.com> + * @author Robin McCorkell <robin@mccorkell.me.uk> + * @author Thomas Müller <thomas.mueller@tmit.eu> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +use OC\Files\Filesystem; +use OC\Hooks\BasicEmitter; +use OCP\Config; +use OCP\Files\Cache\IScanner; +use OCP\Files\Storage\ILockingStorage; +use OCP\Lock\ILockingProvider; + +/** + * Class Scanner + * + * Hooks available in scope \OC\Files\Cache\Scanner: + * - scanFile(string $path, string $storageId) + * - scanFolder(string $path, string $storageId) + * - postScanFile(string $path, string $storageId) + * - postScanFolder(string $path, string $storageId) + * + * @package OC\Files\Cache + */ +class Scanner extends BasicEmitter implements IScanner { + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var string $storageId + */ + protected $storageId; + + /** + * @var \OC\Files\Cache\Cache $cache + */ + protected $cache; + + /** + * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache + */ + protected $cacheActive; + + /** + * @var bool $useTransactions whether to use transactions + */ + protected $useTransactions = true; + + /** + * @var \OCP\Lock\ILockingProvider + */ + protected $lockingProvider; + + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->storageId = $this->storage->getId(); + $this->cache = $storage->getCache(); + $this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false); + $this->lockingProvider = \OC::$server->getLockingProvider(); + } + + /** + * Whether to wrap the scanning of a folder in a database transaction + * On default transactions are used + * + * @param bool $useTransactions + */ + public function setUseTransactions($useTransactions) { + $this->useTransactions = $useTransactions; + } + + /** + * get all the metadata of a file or folder + * * + * + * @param string $path + * @return array an array of metadata of the file + */ + protected function getData($path) { + $data = $this->storage->getMetaData($path); + if (is_null($data)) { + \OCP\Util::writeLog('OC\Files\Cache\Scanner', "!!! Path '$path' is not accessible or present !!!", \OCP\Util::DEBUG); + } + return $data; + } + + /** + * scan a single file and store it in the cache + * + * @param string $file + * @param int $reuseExisting + * @param int $parentId + * @param array | null $cacheData existing data in the cache for the file to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @return array an array of metadata of the scanned file + * @throws \OC\ServerNotAvailableException + * @throws \OCP\Lock\LockedException + */ + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { + + // only proceed if $file is not a partial file nor a blacklisted file + if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) { + + //acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + $data = $this->getData($file); + + if ($data) { + + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', array($file, $this->storageId)); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); + } + + $parent = dirname($file); + if ($parent === '.' or $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getId($parent); + } + + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file and $parentId === -1) { + $parentData = $this->scanFile($parent); + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } + if (is_null($cacheData)) { + /** @var CacheEntry $cacheData */ + $cacheData = $this->cache->get($file); + } + if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { + // prevent empty etag + if (empty($cacheData['etag'])) { + $etag = $data['etag']; + } else { + $etag = $cacheData['etag']; + } + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; + } + if ($reuseExisting & self::REUSE_ETAG) { + $data['etag'] = $etag; + } + } + // Only update metadata that has changed + $newData = array_diff_assoc($data, $cacheData->getData()); + } else { + $newData = $data; + $fileId = -1; + } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); + } + if (isset($cacheData['size'])) { + $data['oldSize'] = $cacheData['size']; + } else { + $data['oldSize'] = 0; + } + + if (isset($cacheData['encrypted'])) { + $data['encrypted'] = $cacheData['encrypted']; + } + + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); + } + + } else { + $this->removeFromCache($file); + } + + //release the acquired lock + if ($lock) { + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + if ($data && !isset($data['encrypted'])) { + $data['encrypted'] = false; + } + return $data; + } + + return null; + } + + protected function removeFromCache($path) { + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path)); + $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path)); + if ($this->cacheActive) { + $this->cache->remove($path); + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + * @return int the id of the added file + */ + protected function addToCache($path, $data, $fileId = -1) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data)); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + return $fileId; + } else { + return $this->cache->put($path, $data); + } + } else { + return -1; + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + */ + protected function updateCache($path, $data, $fileId = -1) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); + $this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data)); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + } else { + $this->cache->put($path, $data); + } + } + } + + /** + * scan a folder and all it's children + * + * @param string $path + * @param bool $recursive + * @param int $reuse + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @return array an array of the meta data of the scanned file or folder + */ + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + if ($lock) { + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + $data = $this->scanFile($path, $reuse, -1, null, $lock); + if ($data and $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data, $lock); + $data['size'] = $size; + } + if ($lock) { + if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + return $data; + } + + /** + * Get the children currently in the cache + * + * @param int $folderId + * @return array[] + */ + protected function getExistingChildren($folderId) { + $existingChildren = array(); + $children = $this->cache->getFolderContentsById($folderId); + foreach ($children as $child) { + $existingChildren[$child['name']] = $child; + } + return $existingChildren; + } + + /** + * Get the children from the storage + * + * @param string $folder + * @return string[] + */ + protected function getNewChildren($folder) { + $children = array(); + if ($dh = $this->storage->opendir($folder)) { + if (is_resource($dh)) { + while (($file = readdir($dh)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + $children[] = $file; + } + } + } + } + return $children; + } + + /** + * scan all the files and folders in a folder + * + * @param string $path + * @param bool $recursive + * @param int $reuse + * @param array $folderData existing cache data for the folder to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @return int the size of the scanned folder or -1 if the size is unknown at this stage + */ + protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderData = null, $lock = true) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', array($path, $this->storageId)); + $size = 0; + $childQueue = array(); + if (is_array($folderData) and isset($folderData['fileid'])) { + $folderId = $folderData['fileid']; + } else { + $folderId = $this->cache->getId($path); + } + $existingChildren = $this->getExistingChildren($folderId); + $newChildren = $this->getNewChildren($path); + + if ($this->useTransactions) { + \OC::$server->getDatabaseConnection()->beginTransaction(); + } + $exceptionOccurred = false; + foreach ($newChildren as $file) { + $child = ($path) ? $path . '/' . $file : $file; + try { + $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null; + $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); + if ($data) { + if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { + $childQueue[$child] = $data; + } else if ($data['size'] === -1) { + $size = -1; + } else if ($size !== -1) { + $size += $data['size']; + } + } + } catch (\Doctrine\DBAL\DBALException $ex) { + // might happen if inserting duplicate while a scanning + // process is running in parallel + // log and ignore + \OCP\Util::writeLog('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OCP\Util::DEBUG); + $exceptionOccurred = true; + } catch (\OCP\Lock\LockedException $e) { + if ($this->useTransactions) { + \OC::$server->getDatabaseConnection()->rollback(); + } + throw $e; + } + } + $removedChildren = \array_diff(array_keys($existingChildren), $newChildren); + foreach ($removedChildren as $childName) { + $child = ($path) ? $path . '/' . $childName : $childName; + $this->removeFromCache($child); + } + if ($this->useTransactions) { + \OC::$server->getDatabaseConnection()->commit(); + } + if ($exceptionOccurred) { + // It might happen that the parallel scan process has already + // inserted mimetypes but those weren't available yet inside the transaction + // To make sure to have the updated mime types in such cases, + // we reload them here + \OC::$server->getMimeTypeLoader()->reset(); + } + + foreach ($childQueue as $child => $childData) { + $childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse, $childData, $lock); + if ($childSize === -1) { + $size = -1; + } else if ($size !== -1) { + $size += $childSize; + } + } + if (!is_array($folderData) or !isset($folderData['size']) or $folderData['size'] !== $size) { + $this->updateCache($path, array('size' => $size), $folderId); + } + $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); + return $size; + } + + /** + * check if the file should be ignored when scanning + * NOTE: files with a '.part' extension are ignored as well! + * prevents unfinished put requests to be scanned + * + * @param string $file + * @return boolean + */ + public static function isPartialFile($file) { + if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { + return true; + } + if (strpos($file, '.part/') !== false) { + return true; + } + + return false; + } + + /** + * walk over any folders that are not fully scanned yet and scan them + */ + public function backgroundScan() { + if (!$this->cache->inCache('')) { + $this->runBackgroundScanJob(function () { + $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); + }, ''); + } else { + $lastPath = null; + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function() use ($path) { + $this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG); + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } + } + } + + private function runBackgroundScanJob(callable $callback, $path) { + try { + $callback(); + \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); + if ($this->cacheActive) { + $this->cache->correctFolderSize($path); + } + } catch (\OCP\Files\StorageInvalidException $e) { + // skip unavailable storages + } catch (\OCP\Files\StorageNotAvailableException $e) { + // skip unavailable storages + } catch (\OCP\Files\ForbiddenException $e) { + // skip forbidden storages + } catch (\OCP\Lock\LockedException $e) { + // skip unavailable storages + } + } + + /** + * Set whether the cache is affected by scan operations + * + * @param boolean $active The active state of the cache + */ + public function setCacheActive($active) { + $this->cacheActive = $active; + } +} diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php new file mode 100644 index 00000000000..90c451ecc21 --- /dev/null +++ b/lib/private/Files/Cache/Storage.php @@ -0,0 +1,189 @@ +<?php +/** + * @author Joas Schilling <nickvergessen@owncloud.com> + * @author Jörn Friedrich Dreyer <jfd@butonic.de> + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Robin McCorkell <robin@mccorkell.me.uk> + * @author Thomas Müller <thomas.mueller@tmit.eu> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; + +/** + * Handle the mapping between the string and numeric storage ids + * + * Each storage has 2 different ids + * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') + * and a numeric storage id which is referenced in the file cache + * + * A mapping between the two storage ids is stored in the database and accessible trough this class + * + * @package OC\Files\Cache + */ +class Storage { + private $storageId; + private $numericId; + + /** + * @param \OC\Files\Storage\Storage|string $storage + * @param bool $isAvailable + * @throws \RuntimeException + */ + public function __construct($storage, $isAvailable = true) { + if ($storage instanceof \OC\Files\Storage\Storage) { + $this->storageId = $storage->getId(); + } else { + $this->storageId = $storage; + } + $this->storageId = self::adjustStorageId($this->storageId); + + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = $row['numeric_id']; + } else { + $connection = \OC::$server->getDatabaseConnection(); + $available = $isAvailable ? 1 : 0; + if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { + $this->numericId = $connection->lastInsertId('*PREFIX*storages'); + } else { + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = $row['numeric_id']; + } else { + throw new \RuntimeException('Storage could neither be inserted nor be selected from the database'); + } + } + } + } + + /** + * @param string $storageId + * @return array|null + */ + public static function getStorageById($storageId) { + $sql = 'SELECT * FROM `*PREFIX*storages` WHERE `id` = ?'; + $result = \OC_DB::executeAudited($sql, array($storageId)); + return $result->fetchRow(); + } + + /** + * Adjusts the storage id to use md5 if too long + * @param string $storageId storage id + * @return string unchanged $storageId if its length is less than 64 characters, + * else returns the md5 of $storageId + */ + public static function adjustStorageId($storageId) { + if (strlen($storageId) > 64) { + return md5($storageId); + } + return $storageId; + } + + /** + * Get the numeric id for the storage + * + * @return int + */ + public function getNumericId() { + return $this->numericId; + } + + /** + * Get the string id for the storage + * + * @param int $numericId + * @return string|null either the storage id string or null if the numeric id is not known + */ + public static function getStorageId($numericId) { + + $sql = 'SELECT `id` FROM `*PREFIX*storages` WHERE `numeric_id` = ?'; + $result = \OC_DB::executeAudited($sql, array($numericId)); + if ($row = $result->fetchRow()) { + return $row['id']; + } else { + return null; + } + } + + /** + * Get the numeric of the storage with the provided string id + * + * @param $storageId + * @return int|null either the numeric storage id or null if the storage id is not knwon + */ + public static function getNumericStorageId($storageId) { + $storageId = self::adjustStorageId($storageId); + + if ($row = self::getStorageById($storageId)) { + return $row['numeric_id']; + } else { + return null; + } + } + + /** + * @return array|null [ available, last_checked ] + */ + public function getAvailability() { + if ($row = self::getStorageById($this->storageId)) { + return [ + 'available' => ((int)$row['available'] === 1), + 'last_checked' => $row['last_checked'] + ]; + } else { + return null; + } + } + + /** + * @param bool $isAvailable + */ + public function setAvailability($isAvailable) { + $sql = 'UPDATE `*PREFIX*storages` SET `available` = ?, `last_checked` = ? WHERE `id` = ?'; + $available = $isAvailable ? 1 : 0; + \OC_DB::executeAudited($sql, array($available, time(), $this->storageId)); + } + + /** + * Check if a string storage id is known + * + * @param string $storageId + * @return bool + */ + public static function exists($storageId) { + return !is_null(self::getNumericStorageId($storageId)); + } + + /** + * remove the entry for the storage + * + * @param string $storageId + */ + public static function remove($storageId) { + $storageId = self::adjustStorageId($storageId); + $numericId = self::getNumericStorageId($storageId); + $sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?'; + \OC_DB::executeAudited($sql, array($storageId)); + + if (!is_null($numericId)) { + $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `storage` = ?'; + \OC_DB::executeAudited($sql, array($numericId)); + } + } +} diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php new file mode 100644 index 00000000000..3f80f2b6167 --- /dev/null +++ b/lib/private/Files/Cache/Updater.php @@ -0,0 +1,228 @@ +<?php +/** + * @author Björn Schießle <schiessle@owncloud.com> + * @author Michael Gapczynski <GapczynskiM@gmail.com> + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Storage\IStorage; + +/** + * Update the cache and propagate changes + * + */ +class Updater implements IUpdater { + /** + * @var bool + */ + protected $enabled = true; + + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @var \OC\Files\Cache\Propagator + */ + protected $propagator; + + /** + * @var Scanner + */ + protected $scanner; + + /** + * @var Cache + */ + protected $cache; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->propagator = $storage->getPropagator(); + $this->scanner = $storage->getScanner(); + $this->cache = $storage->getCache(); + } + + /** + * Disable updating the cache trough this updater + */ + public function disable() { + $this->enabled = false; + } + + /** + * Re-enable the updating of the cache trough this updater + */ + public function enable() { + $this->enabled = true; + } + + /** + * Get the propagator for etags and mtime for the view the updater works on + * + * @return Propagator + */ + public function getPropagator() { + return $this->propagator; + } + + /** + * Propagate etag and mtime changes for the parent folders of $path up to the root of the filesystem + * + * @param string $path the path of the file to propagate the changes for + * @param int|null $time the timestamp to set as mtime for the parent folders, if left out the current time is used + */ + public function propagate($path, $time = null) { + if (Scanner::isPartialFile($path)) { + return; + } + $this->propagator->propagateChange($path, $time); + } + + /** + * Update the cache for $path and update the size, etag and mtime of the parent folders + * + * @param string $path + * @param int $time + */ + public function update($path, $time = null) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + if (is_null($time)) { + $time = time(); + } + + $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); + if ( + isset($data['oldSize']) && isset($data['size']) && + !$data['encrypted'] // encryption is a pita and touches the cache itself + ) { + $sizeDifference = $data['size'] - $data['oldSize']; + } else { + // scanner didn't provide size info, fallback to full size calculation + $sizeDifference = 0; + $this->cache->correctFolderSize($path, $data); + } + $this->correctParentStorageMtime($path); + $this->propagator->propagateChange($path, $time, $sizeDifference); + } + + /** + * Remove $path from the cache and update the size, etag and mtime of the parent folders + * + * @param string $path + */ + public function remove($path) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + + $this->cache->remove($path); + $this->cache->correctFolderSize($parent); + $this->correctParentStorageMtime($path); + $this->propagator->propagateChange($path, time()); + } + + /** + * Rename a file or folder in the cache and update the size, etag and mtime of the parent folders + * + * @param IStorage $sourceStorage + * @param string $source + * @param string $target + */ + public function renameFromStorage(IStorage $sourceStorage, $source, $target) { + if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) { + return; + } + + $time = time(); + + $sourceCache = $sourceStorage->getCache(); + $sourceUpdater = $sourceStorage->getUpdater(); + $sourcePropagator = $sourceStorage->getPropagator(); + + if ($sourceCache->inCache($source)) { + if ($this->cache->inCache($target)) { + $this->cache->remove($target); + } + + if ($sourceStorage === $this->storage) { + $this->cache->move($source, $target); + } else { + $this->cache->moveFromCache($sourceCache, $source, $target); + } + } + + if (pathinfo($source, PATHINFO_EXTENSION) !== pathinfo($target, PATHINFO_EXTENSION)) { + // handle mime type change + $mimeType = $this->storage->getMimeType($target); + $fileId = $this->cache->getId($target); + $this->cache->update($fileId, ['mimetype' => $mimeType]); + } + + $sourceCache->correctFolderSize($source); + $this->cache->correctFolderSize($target); + if ($sourceUpdater instanceof Updater) { + $sourceUpdater->correctParentStorageMtime($source); + } + $this->correctParentStorageMtime($target); + $this->updateStorageMTimeOnly($target); + $sourcePropagator->propagateChange($source, $time); + $this->propagator->propagateChange($target, $time); + } + + private function updateStorageMTimeOnly($internalPath) { + $fileId = $this->cache->getId($internalPath); + if ($fileId !== -1) { + $this->cache->update( + $fileId, [ + 'mtime' => null, // this magic tells it to not overwrite mtime + 'storage_mtime' => $this->storage->filemtime($internalPath) + ] + ); + } + } + + /** + * update the storage_mtime of the direct parent in the cache to the mtime from the storage + * + * @param string $internalPath + */ + private function correctParentStorageMtime($internalPath) { + $parentId = $this->cache->getParentId($internalPath); + $parent = dirname($internalPath); + if ($parentId != -1) { + $this->cache->update($parentId, array('storage_mtime' => $this->storage->filemtime($parent))); + } + } +} diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php new file mode 100644 index 00000000000..a00e875a2d4 --- /dev/null +++ b/lib/private/Files/Cache/Watcher.php @@ -0,0 +1,140 @@ +<?php +/** + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IWatcher; + +/** + * check the storage backends for updates and change the cache accordingly + */ +class Watcher implements IWatcher { + + protected $watchPolicy = self::CHECK_ONCE; + + protected $checkedPaths = array(); + + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var Cache $cache + */ + protected $cache; + + /** + * @var Scanner $scanner ; + */ + protected $scanner; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->cache = $storage->getCache(); + $this->scanner = $storage->getScanner(); + } + + /** + * @param int $policy either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function setPolicy($policy) { + $this->watchPolicy = $policy; + } + + /** + * @return int either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function getPolicy() { + return $this->watchPolicy; + } + + /** + * check $path for updates and update if needed + * + * @param string $path + * @param ICacheEntry|null $cachedEntry + * @return boolean true if path was updated + */ + public function checkUpdate($path, $cachedEntry = null) { + if (is_null($cachedEntry)) { + $cachedEntry = $this->cache->get($path); + } + if ($this->needsUpdate($path, $cachedEntry)) { + $this->update($path, $cachedEntry); + return true; + } else { + return false; + } + } + + /** + * Update the cache for changes to $path + * + * @param string $path + * @param ICacheEntry $cachedData + */ + public function update($path, $cachedData) { + if ($this->storage->is_dir($path)) { + $this->scanner->scan($path, Scanner::SCAN_SHALLOW); + } else { + $this->scanner->scanFile($path); + } + if ($cachedData['mimetype'] === 'httpd/unix-directory') { + $this->cleanFolder($path); + } + $this->cache->correctFolderSize($path); + } + + /** + * Check if the cache for $path needs to be updated + * + * @param string $path + * @param ICacheEntry $cachedData + * @return bool + */ + public function needsUpdate($path, $cachedData) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and array_search($path, $this->checkedPaths) === false)) { + $this->checkedPaths[] = $path; + return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + } + return false; + } + + /** + * remove deleted files in $path from the cache + * + * @param string $path + */ + public function cleanFolder($path) { + $cachedContent = $this->cache->getFolderContents($path); + foreach ($cachedContent as $entry) { + if (!$this->storage->file_exists($entry['path'])) { + $this->cache->remove($entry['path']); + } + } + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php new file mode 100644 index 00000000000..88b0f23a1fc --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -0,0 +1,300 @@ +<?php +/** + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Robin McCorkell <robin@mccorkell.me.uk> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache\Wrapper; + +/** + * Jail to a subdirectory of the wrapped cache + */ +class CacheJail extends CacheWrapper { + /** + * @var string + */ + protected $root; + + /** + * @param \OCP\Files\Cache\ICache $cache + * @param string $root + */ + public function __construct($cache, $root) { + parent::__construct($cache); + $this->root = $root; + } + + protected function getSourcePath($path) { + if ($path === '') { + return $this->root; + } else { + return $this->root . '/' . ltrim($path, '/'); + } + } + + /** + * @param string $path + * @return null|string the jailed path or null if the path is outside the jail + */ + protected function getJailedPath($path) { + $rootLength = strlen($this->root) + 1; + if ($path === $this->root) { + return ''; + } else if (substr($path, 0, $rootLength) === $this->root . '/') { + return substr($path, $rootLength); + } else { + return null; + } + } + + /** + * @param array $entry + * @return array + */ + protected function formatCacheEntry($entry) { + if (isset($entry['path'])) { + $entry['path'] = $this->getJailedPath($entry['path']); + } + return $entry; + } + + protected function filterCacheEntry($entry) { + $rootLength = strlen($this->root) + 1; + return ($entry['path'] === $this->root) or (substr($entry['path'], 0, $rootLength) === $this->root . '/'); + } + + /** + * get the stored metadata of a file or folder + * + * @param string /int $file + * @return array|false + */ + public function get($file) { + if (is_string($file) or $file == '') { + $file = $this->getSourcePath($file); + } + return parent::get($file); + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->cache->insert($this->getSourcePath($file), $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->cache->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->cache->getId($this->getSourcePath($file)); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + if ($file === '') { + return -1; + } else { + return $this->cache->getParentId($this->getSourcePath($file)); + } + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->cache->inCache($this->getSourcePath($file)); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->cache->remove($this->getSourcePath($file)); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->cache->move($this->getSourcePath($source), $this->getSourcePath($target)); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->cache->remove($this->root); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->cache->getStatus($this->getSourcePath($file)); + } + + private function formatSearchResults($results) { + $results = array_filter($results, array($this, 'filterCacheEntry')); + $results = array_values($results); + return array_map(array($this, 'formatCacheEntry'), $results); + } + + /** + * search for files matching $pattern + * + * @param string $pattern + * @return array an array of file data + */ + public function search($pattern) { + $results = $this->cache->search($pattern); + return $this->formatSearchResults($results); + } + + /** + * search for files by mimetype + * + * @param string $mimetype + * @return array + */ + public function searchByMime($mimetype) { + $results = $this->cache->searchByMime($mimetype); + return $this->formatSearchResults($results); + } + + /** + * search for files by mimetype + * + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return array + */ + public function searchByTag($tag, $userId) { + $results = $this->cache->searchByTag($tag, $userId); + return $this->formatSearchResults($results); + } + + /** + * update the folder size and the size of all parent folders + * + * @param string|boolean $path + * @param array $data (optional) meta data of the folder + */ + public function correctFolderSize($path, $data = null) { + $this->cache->correctFolderSize($this->getSourcePath($path), $data); + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array $entry (optional) meta data of the folder + * @return int + */ + public function calculateFolderSize($path, $entry = null) { + return $this->cache->calculateFolderSize($this->getSourcePath($path), $entry); + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + // not supported + return array(); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiply incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|bool the path of the folder or false when no folder matched + */ + public function getIncomplete() { + // not supported + return false; + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + $path = $this->cache->getPathById($id); + return $this->getJailedPath($path); + } + + /** + * Move a file or folder in the cache + * + * Note that this should make sure the entries are removed from the source cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(\OCP\Files\Cache\ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache === $this) { + return $this->move($sourcePath, $targetPath); + } + return $this->cache->moveFromCache($sourceCache, $sourcePath, $targetPath); + } +} diff --git a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php new file mode 100644 index 00000000000..b3a7bcb3a73 --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php @@ -0,0 +1,46 @@ +<?php +/** + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache\Wrapper; + +class CachePermissionsMask extends CacheWrapper { + /** + * @var int + */ + protected $mask; + + /** + * @param \OCP\Files\Cache\ICache $cache + * @param int $mask + */ + public function __construct($cache, $mask) { + parent::__construct($cache); + $this->mask = $mask; + } + + protected function formatCacheEntry($entry) { + if (isset($entry['permissions'])) { + $entry['permissions'] &= $this->mask; + } + return $entry; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php new file mode 100644 index 00000000000..8c77e3c340e --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -0,0 +1,309 @@ +<?php +/** + * @author Morris Jobke <hey@morrisjobke.de> + * @author Robin Appelman <icewind@owncloud.com> + * @author Robin McCorkell <robin@mccorkell.me.uk> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Cache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\ICache; + +class CacheWrapper extends Cache { + /** + * @var \OCP\Files\Cache\ICache + */ + protected $cache; + + /** + * @param \OCP\Files\Cache\ICache $cache + */ + public function __construct($cache) { + $this->cache = $cache; + } + + /** + * Make it easy for wrappers to modify every returned cache entry + * + * @param ICacheEntry $entry + * @return ICacheEntry + */ + protected function formatCacheEntry($entry) { + return $entry; + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file + * @return ICacheEntry|false + */ + public function get($file) { + $result = $this->cache->get($file); + if ($result) { + $result = $this->formatCacheEntry($result); + } + return $result; + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + // can't do a simple $this->cache->.... call here since getFolderContentsById needs to be called on this + // and not the wrapped cache + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return array + */ + public function getFolderContentsById($fileId) { + $results = $this->cache->getFolderContentsById($fileId); + return array_map(array($this, 'formatCacheEntry'), $results); + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->cache->insert($file, $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->cache->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->cache->getId($file); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + return $this->cache->getParentId($file); + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->cache->inCache($file); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->cache->remove($file); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->cache->move($source, $target); + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $this->cache->moveFromCache($sourceCache, $sourcePath, $targetPath); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->cache->clear(); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->cache->getStatus($file); + } + + /** + * search for files matching $pattern + * + * @param string $pattern + * @return ICacheEntry[] an array of file data + */ + public function search($pattern) { + $results = $this->cache->search($pattern); + return array_map(array($this, 'formatCacheEntry'), $results); + } + + /** + * search for files by mimetype + * + * @param string $mimetype + * @return ICacheEntry[] + */ + public function searchByMime($mimetype) { + $results = $this->cache->searchByMime($mimetype); + return array_map(array($this, 'formatCacheEntry'), $results); + } + + /** + * search for files by tag + * + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return ICacheEntry[] file data + */ + public function searchByTag($tag, $userId) { + $results = $this->cache->searchByTag($tag, $userId); + return array_map(array($this, 'formatCacheEntry'), $results); + } + + /** + * update the folder size and the size of all parent folders + * + * @param string|boolean $path + * @param array $data (optional) meta data of the folder + */ + public function correctFolderSize($path, $data = null) { + $this->cache->correctFolderSize($path, $data); + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array $entry (optional) meta data of the folder + * @return int + */ + public function calculateFolderSize($path, $entry = null) { + return $this->cache->calculateFolderSize($path, $entry); + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + return $this->cache->getAll(); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|bool the path of the folder or false when no folder matched + */ + public function getIncomplete() { + return $this->cache->getIncomplete(); + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + return $this->cache->getPathById($id); + } + + /** + * Returns the numeric storage id + * + * @return int + */ + public function getNumericStorageId() { + return $this->cache->getNumericStorageId(); + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @return array first element holding the storage id, second the path + */ + static public function getById($id) { + return parent::getById($id); + } +} |