diff options
Diffstat (limited to 'lib/private/Files/Cache')
24 files changed, 2249 insertions, 1352 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 2513abd525f..329466e682d 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -1,69 +1,47 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Ari Selseng <ari@selseng.net> - * @author Artem Kochnev <MrJeos@gmail.com> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Florin Peter <github@florin-peter.de> - * @author Frédéric Fortier <frederic.fortier@oronospolytechnique.com> - * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Michael Gapczynski <GapczynskiM@gmail.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; -use OCP\DB\IResult; +use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery; +use OC\Files\Storage\Wrapper\Encryption; +use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Cache\CacheEntryInsertedEvent; +use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheEntryUpdatedEvent; use OCP\Files\Cache\CacheInsertEvent; -use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheUpdateEvent; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\FileInfo; use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; +use OCP\Util; +use Psr\Log\LoggerInterface; /** * Metadata cache for a storage * - * The cache stores the metadata for all files and folders in a storage and is kept up to date trough the following mechanisms: + * The cache stores the metadata for all files and folders in a storage and is kept up to date through the following mechanisms: * * - Scanner: scans the storage and updates the cache where needed - * - Watcher: checks for changes made to the filesystem outside of the ownCloud instance and rescans files and folder when a change is detected - * - Updater: listens to changes made to the filesystem inside of the ownCloud instance and updates the cache where needed + * - Watcher: checks for changes made to the filesystem outside of the Nextcloud instance and rescans files and folder when a change is detected + * - Updater: listens to changes made to the filesystem inside of the Nextcloud instance and updates the cache where needed * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater */ class Cache implements ICache { @@ -74,62 +52,51 @@ class Cache implements ICache { /** * @var array partial data for the cache */ - protected $partial = []; - - /** - * @var string - */ - protected $storageId; - - private $storage; - - /** - * @var Storage $storageCache - */ - protected $storageCache; - - /** @var IMimeTypeLoader */ - protected $mimetypeLoader; - - /** - * @var IDBConnection - */ - protected $connection; - - /** - * @var IEventDispatcher - */ - protected $eventDispatcher; - - /** @var QuerySearchHelper */ - protected $querySearchHelper; - - /** - * @param IStorage $storage - */ - public function __construct(IStorage $storage) { + protected array $partial = []; + protected string $storageId; + protected Storage $storageCache; + protected IMimeTypeLoader $mimetypeLoader; + protected IDBConnection $connection; + protected SystemConfig $systemConfig; + protected LoggerInterface $logger; + protected QuerySearchHelper $querySearchHelper; + protected IEventDispatcher $eventDispatcher; + protected IFilesMetadataManager $metadataManager; + + public function __construct( + private IStorage $storage, + // this constructor is used in to many pleases to easily do proper di + // so instead we group it all together + ?CacheDependencies $dependencies = null, + ) { $this->storageId = $storage->getId(); - $this->storage = $storage; if (strlen($this->storageId) > 64) { $this->storageId = md5($this->storageId); } - - $this->storageCache = new Storage($storage); - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); - $this->connection = \OC::$server->getDatabaseConnection(); - $this->eventDispatcher = \OC::$server->get(IEventDispatcher::class); - $this->querySearchHelper = new QuerySearchHelper($this->mimetypeLoader); + if (!$dependencies) { + $dependencies = \OCP\Server::get(CacheDependencies::class); + } + $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->systemConfig = $dependencies->getSystemConfig(); + $this->logger = $dependencies->getLogger(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + $this->eventDispatcher = $dependencies->getEventDispatcher(); + $this->metadataManager = $dependencies->getMetadataManager(); } - private function getQueryBuilder() { + protected function getQueryBuilder() { return new CacheQueryBuilder( - $this->connection, - \OC::$server->getSystemConfig(), - \OC::$server->getLogger(), - $this + $this->connection->getQueryBuilder(), + $this->metadataManager, ); } + public function getStorageCache(): Storage { + return $this->storageCache; + } + /** * Get the numeric storage id for this cache's storage * @@ -142,35 +109,39 @@ class Cache implements ICache { /** * get the stored metadata of a file or folder * - * @param string | int $file either the path of a file or folder or the file id for a file or folder - * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache + * @param string|int $file either the path of a file or folder or the file id for a file or folder + * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache */ public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); - if (is_string($file) or $file == '') { + if (is_string($file) || $file == '') { // normalize file $file = $this->normalize($file); - $query->whereStorageId() - ->wherePath($file); + $query->wherePath($file); } else { //file id $query->whereFileId($file); } + $query->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $data = $result->fetch(); $result->closeCursor(); - //merge partial data - if (!$data and is_string($file) and isset($this->partial[$file])) { - return $this->partial[$file]; - } elseif (!$data) { - return $data; - } else { + if ($data !== false) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } } + + return false; } /** @@ -182,9 +153,12 @@ class Cache implements ICache { */ public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) { //fix types + $data['name'] = (string)$data['name']; + $data['path'] = (string)$data['path']; $data['fileid'] = (int)$data['fileid']; $data['parent'] = (int)$data['parent']; - $data['size'] = 0 + $data['size']; + $data['size'] = Util::numericToNumber($data['size']); + $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0); $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; $data['encryptedVersion'] = (int)$data['encrypted']; @@ -196,12 +170,15 @@ class Cache implements ICache { if ($data['storage_mtime'] == 0) { $data['storage_mtime'] = $data['mtime']; } + if (isset($data['f_permissions'])) { + $data['scan_permissions'] = $data['f_permissions']; + } $data['permissions'] = (int)$data['permissions']; if (isset($data['creation_time'])) { - $data['creation_time'] = (int) $data['creation_time']; + $data['creation_time'] = (int)$data['creation_time']; } if (isset($data['upload_time'])) { - $data['upload_time'] = (int) $data['upload_time']; + $data['upload_time'] = (int)$data['upload_time']; } return new CacheEntry($data); } @@ -228,13 +205,17 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->selectFileCache() ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) ->orderBy('name', 'ASC'); - $result = $query->execute(); + $metadataQuery = $query->selectMetadata(); + + $result = $query->executeQuery(); $files = $result->fetchAll(); $result->closeCursor(); - return array_map(function (array $data) { + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); } @@ -273,14 +254,14 @@ class Cache implements ICache { $file = $this->normalize($file); if (isset($this->partial[$file])) { //add any saved partial data - $data = array_merge($this->partial[$file], $data); + $data = array_merge($this->partial[$file]->getData(), $data); unset($this->partial[$file]); } $requiredFields = ['size', 'mtime', 'mimetype']; foreach ($requiredFields as $field) { if (!isset($data[$field])) { //data not complete save as partial and return - $this->partial[$file] = $data; + $this->partial[$file] = new CacheEntry($data); return -1; } } @@ -289,6 +270,9 @@ class Cache implements ICache { if (!isset($data['parent'])) { $data['parent'] = $this->getParentId($file); } + if ($data['parent'] === -1 && $file !== '') { + throw new \Exception('Parent folder not in filecache for ' . $file); + } $data['name'] = basename($file); [$values, $extensionValues] = $this->normalizeData($data); @@ -309,12 +293,13 @@ class Cache implements ICache { if (count($extensionValues)) { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $storageId); $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { $query->setValue($column, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId); @@ -363,6 +348,7 @@ class Cache implements ICache { $query->update('filecache') ->whereFileId($id) + ->whereStorageId($this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -374,13 +360,14 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } if (count($extensionValues)) { try { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $this->getNumericStorageId()); $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { @@ -392,6 +379,7 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->update('filecache_extended') ->whereFileId($id) + ->hintShardKey('storage', $this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -403,7 +391,7 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } } @@ -425,7 +413,7 @@ class Cache implements ICache { protected function normalizeData(array $data): array { $fields = [ 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', - 'etag', 'permissions', 'checksum', 'storage']; + 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; $extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; $doNotCopyStorageMTime = false; @@ -438,7 +426,7 @@ class Cache implements ICache { $params = []; $extensionParams = []; foreach ($data as $name => $value) { - if (array_search($name, $fields) !== false) { + if (in_array($name, $fields)) { if ($name === 'path') { $params['path_hash'] = md5($value); } elseif ($name === 'mimetype') { @@ -458,7 +446,7 @@ class Cache implements ICache { } $params[$name] = $value; } - if (array_search($name, $extensionFields) !== false) { + if (in_array($name, $extensionFields)) { $extensionParams[$name] = $value; } } @@ -482,10 +470,10 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('fileid') ->from('filecache') - ->whereStorageId() + ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $id = $result->fetchOne(); $result->closeCursor(); @@ -535,16 +523,18 @@ class Cache implements ICache { public function remove($file) { $entry = $this->get($file); - if ($entry) { + if ($entry instanceof ICacheEntry) { $query = $this->getQueryBuilder(); $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereFileId($entry->getId()); - $query->execute(); + $query->executeStatement(); $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->whereFileId($entry->getId()); - $query->execute(); + ->whereFileId($entry->getId()) + ->hintShardKey('storage', $this->getNumericStorageId()); + $query->executeStatement(); if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) { $this->removeChildren($entry); @@ -555,20 +545,7 @@ class Cache implements ICache { } /** - * Get all sub folders of a folder - * - * @param ICacheEntry $entry the cache entry of the folder to get the subfolders for - * @return ICacheEntry[] the cache entries for the subfolders - */ - private function getSubFolders(ICacheEntry $entry) { - $children = $this->getFolderContentsById($entry->getId()); - return array_filter($children, function ($child) { - return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER; - }); - } - - /** - * Recursively remove all children of a folder + * Remove all children of a folder * * @param ICacheEntry $entry the cache entry of the folder to remove the children of * @throws \OC\DatabaseException @@ -576,24 +553,45 @@ class Cache implements ICache { private function removeChildren(ICacheEntry $entry) { $parentIds = [$entry->getId()]; $queue = [$entry->getId()]; + $deletedIds = []; + $deletedPaths = []; - // we walk depth first trough the file tree, removing all filecache_extended attributes while we walk + // we walk depth first through the file tree, removing all filecache_extended attributes while we walk // and collecting all folder ids to later use to delete the filecache entries while ($entryId = array_pop($queue)) { $children = $this->getFolderContentsById($entryId); $childIds = array_map(function (ICacheEntry $cacheEntry) { return $cacheEntry->getId(); }, $children); + $childPaths = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getPath(); + }, $children); + + foreach ($childIds as $childId) { + $deletedIds[] = $childId; + } + + foreach ($childPaths as $childPath) { + $deletedPaths[] = $childPath; + } $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->where($query->expr()->in('fileid', $query->createNamedParameter($childIds, IQueryBuilder::PARAM_INT_ARRAY))); - $query->execute(); + ->where($query->expr()->in('fileid', $query->createParameter('childIds'))) + ->hintShardKey('storage', $this->getNumericStorageId()); + + foreach (array_chunk($childIds, 1000) as $childIdChunk) { + $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } /** @var ICacheEntry[] $childFolders */ - $childFolders = array_filter($children, function ($child) { - return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER; - }); + $childFolders = []; + foreach ($children as $child) { + if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $childFolders[] = $child; + } + } foreach ($childFolders as $folder) { $parentIds[] = $folder->getId(); $queue[] = $folder->getId(); @@ -602,8 +600,26 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache') - ->whereParentIn($parentIds); - $query->execute(); + ->whereStorageId($this->getNumericStorageId()) + ->whereParentInParameter('parentIds'); + + // Sorting before chunking allows the db to find the entries close to each + // other in the index + sort($parentIds, SORT_NUMERIC); + foreach (array_chunk($parentIds, 1000) as $parentIdChunk) { + $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + + foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { + $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( + $this->storage, + $filePath, + $fileId, + $this->getNumericStorageId() + ); + $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); + } } /** @@ -626,10 +642,14 @@ class Cache implements ICache { return [$this->getNumericStorageId(), $path]; } + protected function hasEncryptionWrapper(): bool { + return $this->storage->instanceOfStorage(Encryption::class); + } + /** * Move a file or folder in the cache * - * @param \OCP\Files\Cache\ICache $sourceCache + * @param ICache $sourceCache * @param string $sourcePath * @param string $targetPath * @throws \OC\DatabaseException @@ -642,6 +662,19 @@ class Cache implements ICache { $targetPath = $this->normalize($targetPath); $sourceData = $sourceCache->get($sourcePath); + if (!$sourceData) { + throw new \Exception('Source path not found in cache: ' . $sourcePath); + } + + $shardDefinition = $this->connection->getShardDefinition('filecache'); + if ( + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + ) { + $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); + return; + } + $sourceId = $sourceData['fileid']; $newParentId = $this->getParentId($targetPath); @@ -655,11 +688,15 @@ class Cache implements ICache { throw new \Exception('Invalid target storage id: ' . $targetStorageId); } - $this->connection->beginTransaction(); if ($sourceData['mimetype'] === 'httpd/unix-directory') { //update all child entries $sourceLength = mb_strlen($sourcePath); - $query = $this->connection->getQueryBuilder(); + + $childIds = $this->getChildIds($sourceStorageId, $sourcePath); + + $childChunks = array_chunk($childIds, 1000); + + $query = $this->getQueryBuilder(); $fun = $query->func(); $newPathFunction = $fun->concat( @@ -667,29 +704,73 @@ class Cache implements ICache { $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash ); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT)) ->set('path_hash', $fun->md5($newPathFunction)) ->set('path', $newPathFunction) - ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%'))); - - try { - $query->execute(); - } catch (\OC\DatabaseException $e) { - $this->connection->rollBack(); - throw $e; + ->whereStorageId($sourceStorageId) + ->andWhere($query->expr()->in('fileid', $query->createParameter('files'))); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + // Retry transaction in case of RetryableException like deadlocks. + // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend + $retryLimit = 4; + for ($i = 1; $i <= $retryLimit; $i++) { + try { + $this->connection->beginTransaction(); + foreach ($childChunks as $chunk) { + $query->setParameter('files', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + break; + } catch (\OC\DatabaseException $e) { + $this->connection->rollBack(); + throw $e; + } catch (DbalException $e) { + $this->connection->rollBack(); + + if (!$e->isRetryable()) { + throw $e; + } + + // Simply throw if we already retried 4 times. + if ($i === $retryLimit) { + throw $e; + } + + // Sleep a bit to give some time to the other transaction to finish. + usleep(100 * 1000 * $i); + } } + } else { + $this->connection->beginTransaction(); } $query = $this->getQueryBuilder(); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId)) ->set('path', $query->createNamedParameter($targetPath)) ->set('path_hash', $query->createNamedParameter(md5($targetPath))) ->set('name', $query->createNamedParameter(basename($targetPath))) ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT)) + ->whereStorageId($sourceStorageId) ->whereFileId($sourceId); - $query->execute(); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + $query->executeStatement(); $this->connection->commit(); @@ -708,19 +789,28 @@ class Cache implements ICache { } } + private function getChildIds(int $storageId, string $path): array { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($path) . '/%'))); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } + /** * remove all entries for files that are stored on the storage from the cache */ public function clear() { $query = $this->getQueryBuilder(); $query->delete('filecache') - ->whereStorageId(); - $query->execute(); + ->whereStorageId($this->getNumericStorageId()); + $query->executeStatement(); $query = $this->connection->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); - $query->execute(); + $query->executeStatement(); } /** @@ -742,10 +832,10 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('size') ->from('filecache') - ->whereStorageId() + ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $size = $result->fetchOne(); $result->closeCursor(); @@ -771,126 +861,50 @@ class Cache implements ICache { * @return ICacheEntry[] an array of cache entries where the name matches the search pattern */ public function search($pattern) { - // normalize pattern - $pattern = $this->normalize($pattern); - - if ($pattern === '%%') { - return []; - } - - $query = $this->getQueryBuilder(); - $query->selectFileCache() - ->whereStorageId() - ->andWhere($query->expr()->iLike('name', $query->createNamedParameter($pattern))); - - $result = $query->execute(); - $files = $result->fetchAll(); - $result->closeCursor(); - - return array_map(function (array $data) { - return self::cacheEntryFromData($data, $this->mimetypeLoader); - }, $files); - } - - /** - * @param IResult $result - * @return CacheEntry[] - */ - private function searchResultToCacheEntries(IResult $result): array { - $files = $result->fetchAll(); - - return array_map(function (array $data) { - return self::cacheEntryFromData($data, $this->mimetypeLoader); - }, $files); + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', $pattern); + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } /** * search for files by mimetype * * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') - * where it will search for all mimetypes in the group ('image/*') + * where it will search for all mimetypes in the group ('image/*') * @return ICacheEntry[] an array of cache entries where the mimetype matches the search */ public function searchByMime($mimetype) { - $mimeId = $this->mimetypeLoader->getId($mimetype); - - $query = $this->getQueryBuilder(); - $query->selectFileCache() - ->whereStorageId(); - - if (strpos($mimetype, '/')) { - $query->andWhere($query->expr()->eq('mimetype', $query->createNamedParameter($mimeId, IQueryBuilder::PARAM_INT))); + if (!str_contains($mimetype, '/')) { + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'); } else { - $query->andWhere($query->expr()->eq('mimepart', $query->createNamedParameter($mimeId, IQueryBuilder::PARAM_INT))); + $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype); } - - $result = $query->execute(); - $files = $result->fetchAll(); - $result->closeCursor(); - - return array_map(function (array $data) { - return self::cacheEntryFromData($data, $this->mimetypeLoader); - }, $files); + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } - public function searchQuery(ISearchQuery $searchQuery) { - $builder = $this->getQueryBuilder(); - - $query = $builder->selectFileCache('file'); - - $query->whereStorageId(); - - if ($this->querySearchHelper->shouldJoinTags($searchQuery->getSearchOperation())) { - $query - ->innerJoin('file', 'vcategory_to_object', 'tagmap', $builder->expr()->eq('file.fileid', 'tagmap.objid')) - ->innerJoin('tagmap', 'vcategory', 'tag', $builder->expr()->andX( - $builder->expr()->eq('tagmap.type', 'tag.type'), - $builder->expr()->eq('tagmap.categoryid', 'tag.id') - )) - ->andWhere($builder->expr()->eq('tag.type', $builder->createNamedParameter('files'))) - ->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($searchQuery->getUser()->getUID()))); - } - - $searchExpr = $this->querySearchHelper->searchOperatorToDBExpr($builder, $searchQuery->getSearchOperation()); - if ($searchExpr) { - $query->andWhere($searchExpr); - } - - if ($searchQuery->limitToHome() && ($this instanceof HomeCache)) { - $query->andWhere($builder->expr()->like('path', $query->expr()->literal('files/%'))); - } - - $this->querySearchHelper->addSearchOrdersToQuery($query, $searchQuery->getOrder()); - - if ($searchQuery->getLimit()) { - $query->setMaxResults($searchQuery->getLimit()); - } - if ($searchQuery->getOffset()) { - $query->setFirstResult($searchQuery->getOffset()); - } - - $result = $query->execute(); - $cacheEntries = $this->searchResultToCacheEntries($result); - $result->closeCursor(); - return $cacheEntries; + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** * Re-calculate the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { $this->calculateFolderSize($path, $data); + if ($path !== '') { $parent = dirname($path); - if ($parent === '.' or $parent === '/') { + if ($parent === '.' || $parent === '/') { $parent = ''; } + if ($isBackgroundScan) { $parentData = $this->get($parent); - if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) { + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { $this->correctFolderSize($parent, $parentData, $isBackgroundScan); } } else { @@ -911,9 +925,10 @@ class Cache implements ICache { $query->select($query->func()->count()) ->from('filecache') ->whereParent($fileId) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $size = (int)$result->fetchOne(); $result->closeCursor(); @@ -926,30 +941,61 @@ class Cache implements ICache { * calculate the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { + return $this->calculateFolderSizeInner($path, $entry); + } + + + /** + * inner function because we can't add new params to the public function without breaking any child classes + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown + * @return int|float + */ + protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) { $totalSize = 0; - if (is_null($entry) or !isset($entry['fileid'])) { + if (is_null($entry) || !isset($entry['fileid'])) { $entry = $this->get($path); } if (isset($entry['mimetype']) && $entry['mimetype'] === FileInfo::MIMETYPE_FOLDER) { $id = $entry['fileid']; $query = $this->getQueryBuilder(); - $query->selectAlias($query->func()->sum('size'), 'f1') - ->selectAlias($query->func()->min('size'), 'f2') + $query->select('size', 'unencrypted_size') ->from('filecache') - ->whereStorageId() + ->whereStorageId($this->getNumericStorageId()) ->whereParent($id); + if ($ignoreUnknown) { + $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); + } - $result = $query->execute(); - $row = $result->fetch(); + $result = $query->executeQuery(); + $rows = $result->fetchAll(); $result->closeCursor(); - if ($row) { - [$sum, $min] = array_values($row); + if ($rows) { + $sizes = array_map(function (array $row) { + return Util::numericToNumber($row['size']); + }, $rows); + $unencryptedOnlySizes = array_map(function (array $row) { + return Util::numericToNumber($row['unencrypted_size']); + }, $rows); + $unencryptedSizes = array_map(function (array $row) { + return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']); + }, $rows); + + $sum = array_sum($sizes); + $min = min($sizes); + + $unencryptedSum = array_sum($unencryptedSizes); + $unencryptedMin = min($unencryptedSizes); + $unencryptedMax = max($unencryptedOnlySizes); + $sum = 0 + $sum; $min = 0 + $min; if ($min === -1) { @@ -957,8 +1003,34 @@ class Cache implements ICache { } else { $totalSize = $sum; } - if ($entry['size'] !== $totalSize) { - $this->update($id, ['size' => $totalSize]); + if ($unencryptedMin === -1 || $min === -1) { + $unencryptedTotal = $unencryptedMin; + } else { + $unencryptedTotal = $unencryptedSum; + } + } else { + $totalSize = 0; + $unencryptedTotal = 0; + $unencryptedMax = 0; + } + + // only set unencrypted size for a folder if any child entries have it set, or the folder is empty + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + if ($shouldWriteUnEncryptedSize) { + // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes + if ($unencryptedMax === 0) { + $unencryptedTotal = 0; + } + + $this->update($id, [ + 'size' => $totalSize, + 'unencrypted_size' => $unencryptedTotal, + ]); + } else { + $this->update($id, [ + 'size' => $totalSize, + ]); } } } @@ -974,9 +1046,9 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('fileid') ->from('filecache') - ->whereStorageId(); + ->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $files = $result->fetchAll(\PDO::FETCH_COLUMN); $result->closeCursor(); @@ -992,22 +1064,22 @@ class Cache implements ICache { * use the one with the highest id gives the best result with the background scanner, since that is most * likely the folder where we stopped scanning previously * - * @return string|bool the path of the folder or false when no folder matched + * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { $query = $this->getQueryBuilder(); $query->select('path') ->from('filecache') - ->whereStorageId() - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('fileid', 'DESC') ->setMaxResults(1); - $result = $query->execute(); + $result = $query->executeQuery(); $path = $result->fetchOne(); $result->closeCursor(); - return $path; + return $path === false ? false : (string)$path; } /** @@ -1020,10 +1092,10 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('path') ->from('filecache') - ->whereStorageId() + ->whereStorageId($this->getNumericStorageId()) ->whereFileId($id); - $result = $query->execute(); + $result = $query->executeQuery(); $path = $result->fetchOne(); $result->closeCursor(); @@ -1031,7 +1103,7 @@ class Cache implements ICache { return null; } - return (string) $path; + return (string)$path; } /** @@ -1041,7 +1113,7 @@ class Cache implements ICache { * * @param int $id * @return array first element holding the storage id, second the path - * @deprecated use getPathById() instead + * @deprecated 17.0.0 use getPathById() instead */ public static function getById($id) { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); @@ -1049,7 +1121,7 @@ class Cache implements ICache { ->from('filecache') ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); @@ -1076,4 +1148,137 @@ class Cache implements ICache { public function normalize($path) { return trim(\OC_Util::normalizeUnicode($path), '/'); } + + /** + * Copy a file or folder in the cache + * + * @param ICache $sourceCache + * @param ICacheEntry $sourceEntry + * @param string $targetPath + * @return int fileId of copied entry + */ + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + if ($sourceEntry->getId() < 0) { + throw new \RuntimeException('Invalid source cache entry on copyFromCache'); + } + $data = $this->cacheEntryToArray($sourceEntry); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $data['encrypted'] = 0; + } + + $fileId = $this->put($targetPath, $data); + if ($fileId <= 0) { + throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' '); + } + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); + foreach ($folderContent as $subEntry) { + $subTargetPath = $targetPath . '/' . $subEntry->getName(); + $this->copyFromCache($sourceCache, $subEntry, $subTargetPath); + } + } + return $fileId; + } + + private function cacheEntryToArray(ICacheEntry $entry): array { + $data = [ + 'size' => $entry->getSize(), + 'mtime' => $entry->getMTime(), + 'storage_mtime' => $entry->getStorageMTime(), + 'mimetype' => $entry->getMimeType(), + 'mimepart' => $entry->getMimePart(), + 'etag' => $entry->getEtag(), + 'permissions' => $entry->getPermissions(), + 'encrypted' => $entry->isEncrypted(), + 'creation_time' => $entry->getCreationTime(), + 'upload_time' => $entry->getUploadTime(), + 'metadata_etag' => $entry->getMetadataEtag(), + ]; + if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) { + $data['permissions'] = $entry['scan_permissions']; + } + return $data; + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', $this->getNumericStorageId()); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($rawEntry->getStorageId() === $this->getNumericStorageId()) { + return $rawEntry; + } else { + return null; + } + } + + private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void { + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath()); + } else { + $fileIds = []; + } + $fileIds[] = $sourceEntry->getId(); + + $helper = $this->connection->getCrossShardMoveHelper(); + + $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId()); + $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId()); + + $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper(); + + $sourcePathLength = strlen($sourceEntry->getPath()); + foreach ($cacheItems as &$cacheItem) { + if ($cacheItem['path'] === $sourceEntry->getPath()) { + $cacheItem['path'] = $targetPath; + $cacheItem['parent'] = $this->getParentId($targetPath); + $cacheItem['name'] = basename($cacheItem['path']); + } else { + $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash + } + $cacheItem['path_hash'] = md5($cacheItem['path']); + $cacheItem['storage'] = $this->getNumericStorageId(); + if ($removeEncryptedFlag) { + $cacheItem['encrypted'] = 0; + } + } + + $targetConnection->beginTransaction(); + + try { + $helper->saveItems($targetConnection, 'filecache', $cacheItems); + $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems); + $helper->saveItems($targetConnection, 'files_metadata', $metadataItems); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + + $sourceConnection->beginTransaction(); + + try { + $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + } catch (\Exception $e) { + $targetConnection->rollback(); + $sourceConnection->rollBack(); + throw $e; + } + + try { + $sourceConnection->commit(); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + $targetConnection->commit(); + } } diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php new file mode 100644 index 00000000000..61d2a2f9646 --- /dev/null +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -0,0 +1,61 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDependencies { + public function __construct( + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $metadataManager, + private DisplayNameCache $displayNameCache, + ) { + } + + public function getMimeTypeLoader(): IMimeTypeLoader { + return $this->mimeTypeLoader; + } + + public function getConnection(): IDBConnection { + return $this->connection; + } + + public function getEventDispatcher(): IEventDispatcher { + return $this->eventDispatcher; + } + + public function getQuerySearchHelper(): QuerySearchHelper { + return $this->querySearchHelper; + } + + public function getSystemConfig(): SystemConfig { + return $this->systemConfig; + } + + public function getLogger(): LoggerInterface { + return $this->logger; + } + + public function getDisplayNameCache(): DisplayNameCache { + return $this->displayNameCache; + } + + public function getMetadataManager(): IFilesMetadataManager { + return $this->metadataManager; + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index a9d2a1acbf0..ab5bae316f4 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use OCP\Files\Cache\ICacheEntry; @@ -38,18 +22,22 @@ class CacheEntry implements ICacheEntry { $this->data = $data; } - public function offsetSet($offset, $value) { + public function offsetSet($offset, $value): void { $this->data[$offset] = $value; } - public function offsetExists($offset) { + public function offsetExists($offset): bool { return isset($this->data[$offset]); } - public function offsetUnset($offset) { + public function offsetUnset($offset): void { unset($this->data[$offset]); } + /** + * @return mixed + */ + #[\ReturnTypeWillChange] public function offsetGet($offset) { if (isset($this->data[$offset])) { return $this->data[$offset]; @@ -111,18 +99,34 @@ class CacheEntry implements ICacheEntry { } public function getMetadataEtag(): ?string { - return $this->data['metadata_etag']; + return $this->data['metadata_etag'] ?? null; } public function getCreationTime(): ?int { - return $this->data['creation_time']; + return $this->data['creation_time'] ?? null; } public function getUploadTime(): ?int { - return $this->data['upload_time']; + return $this->data['upload_time'] ?? null; + } + + public function getParentId(): int { + return $this->data['parent']; } public function getData() { return $this->data; } + + public function __clone() { + $this->data = array_merge([], $this->data); + } + + public function getUnencryptedSize(): int { + if ($this->data['encrypted'] && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return $this->data['size'] ?? 0; + } + } } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index ac17cfaffb2..5492452273b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -3,62 +3,66 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2019 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache; -use OC\DB\QueryBuilder\QueryBuilder; -use OC\SystemConfig; +use OC\DB\QueryBuilder\ExtendedQueryBuilder; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\IDBConnection; -use OCP\ILogger; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; /** * Query builder with commonly used helpers for filecache queries */ -class CacheQueryBuilder extends QueryBuilder { - private $cache; - private $alias = null; +class CacheQueryBuilder extends ExtendedQueryBuilder { + private ?string $alias = null; + + public function __construct( + IQueryBuilder $queryBuilder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($queryBuilder); + } - public function __construct(IDBConnection $connection, SystemConfig $systemConfig, ILogger $logger, Cache $cache) { - parent::__construct($connection, $systemConfig, $logger); + public function selectTagUsage(): self { + $this + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') + ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') + ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') + ->from('filecache', 'filecache') + ->leftJoin('filecache', 'systemtag_object_mapping', 'systemtagmap', $this->expr()->andX( + $this->expr()->eq('filecache.fileid', $this->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $this->expr()->eq('systemtagmap.objecttype', $this->createNamedParameter('files')) + )) + ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $this->expr()->andX( + $this->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'), + $this->expr()->eq('systemtag.visibility', $this->createNamedParameter(true)) + )) + ->groupBy('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable'); - $this->cache = $cache; + return $this; } - public function selectFileCache(string $alias = null) { - $name = $alias ? $alias : 'filecache'; - $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime', - 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time') - ->from('filecache', $name) - ->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); + public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true) { + $name = $alias ?: 'filecache'; + $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', + 'storage_mtime', 'encrypted', "$name.etag", "$name.permissions", 'checksum', 'unencrypted_size') + ->from('filecache', $name); + + if ($joinExtendedCache) { + $this->addSelect('metadata_etag', 'creation_time', 'upload_time'); + $this->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); + } $this->alias = $name; return $this; } - public function whereStorageId() { - $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT))); + public function whereStorageId(int $storageId) { + $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); return $this; } @@ -95,7 +99,7 @@ class CacheQueryBuilder extends QueryBuilder { return $this; } - public function whereParentIn(array $parents) { + public function whereParentInParameter(string $parameter) { $alias = $this->alias; if ($alias) { $alias .= '.'; @@ -103,8 +107,19 @@ class CacheQueryBuilder extends QueryBuilder { $alias = ''; } - $this->andWhere($this->expr()->in("{$alias}parent", $this->createNamedParameter($parents, IQueryBuilder::PARAM_INT_ARRAY))); + $this->andWhere($this->expr()->in("{$alias}parent", $this->createParameter($parameter))); return $this; } + + /** + * join metadata to current query builder and returns an helper + * + * @return IMetadataQuery + */ + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } } diff --git a/lib/private/Files/Cache/FailedCache.php b/lib/private/Files/Cache/FailedCache.php index b57ca2b2572..44c1016ca8e 100644 --- a/lib/private/Files/Cache/FailedCache.php +++ b/lib/private/Files/Cache/FailedCache.php @@ -1,29 +1,18 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; +use OC\Files\Search\SearchComparison; use OCP\Constants; use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; /** @@ -134,4 +123,16 @@ class FailedCache implements ICache { public function normalize($path) { return $path; } + + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + throw new \Exception('Invalid cache'); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', -1); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + return null; + } } diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php new file mode 100644 index 00000000000..c3f3614f3ca --- /dev/null +++ b/lib/private/Files/Cache/FileAccess.php @@ -0,0 +1,222 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\SystemConfig; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IFileAccess; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Low level access to the file cache + */ +class FileAccess implements IFileAccess { + public function __construct( + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private FilesMetadataManager $metadataManager, + private IMimeTypeLoader $mimeTypeLoader, + ) { + } + + private function getQuery(): CacheQueryBuilder { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getByFileIdInStorage(int $fileId, int $storageId): ?CacheEntry { + $items = array_values($this->getByFileIdsInStorage([$fileId], $storageId)); + return $items[0] ?? null; + } + + public function getByPathInStorage(string $path, int $storageId): ?CacheEntry { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->eq('filecache.path_hash', $query->createNamedParameter(md5($path)))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $row = $query->executeQuery()->fetch(); + return $row ? Cache::cacheEntryFromData($row, $this->mimeTypeLoader) : null; + } + + public function getByFileId(int $fileId): ?CacheEntry { + $items = array_values($this->getByFileIds([$fileId])); + return $items[0] ?? null; + } + + /** + * @param array[] $rows + * @return array<int, CacheEntry> + */ + private function rowsToEntries(array $rows): array { + $result = []; + foreach ($rows as $row) { + $entry = Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + $result[$entry->getId()] = $entry; + } + return $result; + } + + /** + * @param int[] $fileIds + * @return array<int, CacheEntry> + */ + public function getByFileIds(array $fileIds): array { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + /** + * @param int[] $fileIds + * @param int $storageId + * @return array<int, CacheEntry> + */ + public function getByFileIdsInStorage(array $fileIds, int $storageId): array { + $fileIds = array_values($fileIds); + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } +} diff --git a/lib/private/Files/Cache/HomeCache.php b/lib/private/Files/Cache/HomeCache.php index 93906bfc93f..248cdc818f0 100644 --- a/lib/private/Files/Cache/HomeCache.php +++ b/lib/private/Files/Cache/HomeCache.php @@ -1,32 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use OCP\Files\Cache\ICacheEntry; @@ -36,8 +14,8 @@ class HomeCache extends Cache { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { if ($path !== '/' and $path !== '' and $path !== 'files' and $path !== 'files_trashbin' and $path !== 'files_versions') { @@ -45,46 +23,18 @@ class HomeCache extends Cache { } elseif ($path === '' or $path === '/') { // since the size of / isn't used (the size of /files is used instead) there is no use in calculating it return 0; + } else { + return $this->calculateFolderSizeInner($path, $entry, true); } - - $totalSize = 0; - if (is_null($entry)) { - $entry = $this->get($path); - } - if ($entry && $entry['mimetype'] === 'httpd/unix-directory') { - $id = $entry['fileid']; - - $query = $this->connection->getQueryBuilder(); - $query->selectAlias($query->func()->sum('size'), 'f1') - ->from('filecache') - ->where($query->expr()->eq('parent', $query->createNamedParameter($id))) - ->andWhere($query->expr()->eq('storage', $query->createNamedParameter($this->getNumericStorageId()))) - ->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); - - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - - if ($row) { - list($sum) = array_values($row); - $totalSize = 0 + $sum; - $entry['size'] += 0; - if ($entry['size'] !== $totalSize) { - $this->update($id, ['size' => $totalSize]); - } - } - $result->closeCursor(); - } - return $totalSize; } /** - * @param string $path + * @param string $file * @return ICacheEntry */ - public function get($path) { - $data = parent::get($path); - if ($path === '' or $path === '/') { + public function get($file) { + $data = parent::get($file); + if ($file === '' or $file === '/') { // only the size of the "files" dir counts $filesData = parent::get('files'); diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php index ab78b1d3a5a..d4ac8a7c8e3 100644 --- a/lib/private/Files/Cache/HomePropagator.php +++ b/lib/private/Files/Cache/HomePropagator.php @@ -1,25 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use OCP\IDBConnection; @@ -42,7 +27,7 @@ class HomePropagator extends Propagator { * @param int $sizeDifference number of bytes the file has grown */ public function propagateChange($internalPath, $time, $sizeDifference = 0) { - list($baseFolder) = explode('/', $internalPath, 2); + [$baseFolder] = explode('/', $internalPath, 2); if (in_array($baseFolder, $this->ignoredBaseFolders)) { return []; } else { diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index a10f51150e1..3f4f70b865b 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -3,27 +3,9 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache; class LocalRootScanner extends Scanner { @@ -45,6 +27,6 @@ class LocalRootScanner extends Scanner { private function shouldScanPath(string $path): bool { $path = trim($path, '/'); - return $path === '' || strpos($path, 'appdata_') === 0 || strpos($path, '__groupfolders') === 0; + return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Files/Cache/MoveFromCacheTrait.php b/lib/private/Files/Cache/MoveFromCacheTrait.php index 0f42b00df1c..db35c6bb7f8 100644 --- a/lib/private/Files/Cache/MoveFromCacheTrait.php +++ b/lib/private/Files/Cache/MoveFromCacheTrait.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use OCP\Files\Cache\ICache; @@ -41,6 +25,8 @@ trait MoveFromCacheTrait { */ abstract public function put($file, array $data); + abstract public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int; + /** * Move a file or folder in the cache * @@ -55,38 +41,4 @@ trait MoveFromCacheTrait { $sourceCache->remove($sourcePath); } - - /** - * Copy a file or folder in the cache - * - * @param \OCP\Files\Cache\ICache $sourceCache - * @param ICacheEntry $sourceEntry - * @param string $targetPath - */ - public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath) { - $this->put($targetPath, $this->cacheEntryToArray($sourceEntry)); - if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { - $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); - foreach ($folderContent as $subEntry) { - $subTargetPath = $targetPath . '/' . $subEntry->getName(); - $this->copyFromCache($sourceCache, $subEntry, $subTargetPath); - } - } - } - - private function cacheEntryToArray(ICacheEntry $entry) { - return [ - 'size' => $entry->getSize(), - 'mtime' => $entry->getMTime(), - 'storage_mtime' => $entry->getStorageMTime(), - 'mimetype' => $entry->getMimeType(), - 'mimepart' => $entry->getMimePart(), - 'etag' => $entry->getEtag(), - 'permissions' => $entry->getPermissions(), - 'encrypted' => $entry->isEncrypted(), - 'creation_time' => $entry->getCreationTime(), - 'upload_time' => $entry->getUploadTime(), - 'metadata_etag' => $entry->getMetadataEtag(), - ]; - } } diff --git a/lib/private/Files/Cache/NullWatcher.php b/lib/private/Files/Cache/NullWatcher.php index 146961d1c51..e3659214849 100644 --- a/lib/private/Files/Cache/NullWatcher.php +++ b/lib/private/Files/Cache/NullWatcher.php @@ -3,27 +3,9 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache; class NullWatcher extends Watcher { diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index cdbe8b60175..a6ba87896f4 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -1,37 +1,27 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; +use OC\DB\Exceptions\DbalException; +use OC\Files\Storage\Wrapper\Encryption; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IPropagator; +use OCP\Files\Storage\IReliableEtagStorage; use OCP\IDBConnection; +use OCP\Server; +use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; /** * Propagate etags and mtimes within the storage */ class Propagator implements IPropagator { + public const MAX_RETRIES = 3; private $inBatch = false; private $batch = []; @@ -51,27 +41,31 @@ class Propagator implements IPropagator { */ private $ignore = []; + private ClockInterface $clock; + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { $this->storage = $storage; $this->connection = $connection; $this->ignore = $ignore; + $this->clock = Server::get(ClockInterface::class); } - /** * @param string $internalPath * @param int $time * @param int $sizeDifference number of bytes the file has grown */ public function propagateChange($internalPath, $time, $sizeDifference = 0) { - // Do not propogate changes in ignored paths + // Do not propagate changes in ignored paths foreach ($this->ignore as $ignore) { - if (strpos($internalPath, $ignore) === 0) { + if (str_starts_with($internalPath, $ignore)) { return; } } - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); + $time = min((int)$time, $this->clock->now()->getTimestamp()); + + $storageId = $this->storage->getStorageCache()->getNumericId(); $parents = $this->getParents($internalPath); @@ -91,26 +85,55 @@ class Propagator implements IPropagator { }, $parentHashes); $builder->update('filecache') - ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT))) - ->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)) + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT))) ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($builder->expr()->in('path_hash', $hashParams)); - - $builder->execute(); + if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); + } if ($sizeDifference !== 0) { - // we need to do size separably so we can ignore entries with uncalculated size - $builder = $this->connection->getQueryBuilder(); - $builder->update('filecache') - ->set('size', $builder->func()->greatest( - $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), + $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $newSize = $builder->func()->greatest( + $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + ); + + // Only update if row had a previously calculated size + $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); + + if ($this->storage->instanceOfStorage(Encryption::class)) { + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); + $newUnencryptedSize = $builder->func()->greatest( + $builder->func()->add( + $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), + $builder->createNamedParameter($sizeDifference) + ), $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) - )) - ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($builder->expr()->in('path_hash', $hashParams)) - ->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + ); + + // Only update if row had a previously calculated size + $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); + } + } - $builder->execute(); + for ($i = 0; $i < self::MAX_RETRIES; $i++) { + try { + $builder->executeStatement(); + break; + } catch (DbalException $e) { + if (!$e->isRetryable()) { + throw $e; + } + + /** @var LoggerInterface $loggerInterface */ + $loggerInterface = \OCP\Server::get(LoggerInterface::class); + $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); + } } } @@ -142,7 +165,7 @@ class Propagator implements IPropagator { $this->batch[$internalPath] = [ 'hash' => md5($internalPath), 'time' => $time, - 'size' => $sizeDifference + 'size' => $sizeDifference, ]; } else { $this->batch[$internalPath]['size'] += $sizeDifference; @@ -169,27 +192,27 @@ class Propagator implements IPropagator { $query->update('filecache') ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) ->set('etag', $query->expr()->literal(uniqid())) - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); $sizeQuery = $this->connection->getQueryBuilder(); $sizeQuery->update('filecache') ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) - ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + ->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash'))) + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); foreach ($this->batch as $item) { $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); $query->setParameter('hash', $item['hash']); - $query->execute(); + $query->executeStatement(); if ($item['size']) { $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); $sizeQuery->setParameter('hash', $item['hash']); - $sizeQuery->execute(); + $sizeQuery->executeStatement(); } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 574b4c18d5d..3ddcf1ca4e6 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -1,226 +1,240 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tobias Kaminsky <tobias@kaminsky.me> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache; +use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\QueryOptimizer\QueryOptimizer; +use OC\Files\Search\SearchBinaryOperator; +use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; use OCP\Files\IMimeTypeLoader; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; -use OCP\Files\Search\ISearchComparison; -use OCP\Files\Search\ISearchOperator; -use OCP\Files\Search\ISearchOrder; +use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; +use Psr\Log\LoggerInterface; -/** - * Tools for transforming search queries into database queries - */ class QuerySearchHelper { - protected static $searchOperatorMap = [ - ISearchComparison::COMPARE_LIKE => 'iLike', - ISearchComparison::COMPARE_EQUAL => 'eq', - ISearchComparison::COMPARE_GREATER_THAN => 'gt', - ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', - ISearchComparison::COMPARE_LESS_THAN => 'lt', - ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte' - ]; - - protected static $searchOperatorNegativeMap = [ - ISearchComparison::COMPARE_LIKE => 'notLike', - ISearchComparison::COMPARE_EQUAL => 'neq', - ISearchComparison::COMPARE_GREATER_THAN => 'lte', - ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', - ISearchComparison::COMPARE_LESS_THAN => 'gte', - ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lt' - ]; - - public const TAG_FAVORITE = '_$!<Favorite>!$_'; - - /** @var IMimeTypeLoader */ - private $mimetypeLoader; + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, + ) { + } - /** - * QuerySearchUtil constructor. - * - * @param IMimeTypeLoader $mimetypeLoader - */ - public function __construct(IMimeTypeLoader $mimetypeLoader) { - $this->mimetypeLoader = $mimetypeLoader; + protected function getQueryBuilder() { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->filesMetadataManager, + ); } /** - * Whether or not the tag tables should be joined to complete the search - * - * @param ISearchOperator $operator - * @return boolean + * @param CacheQueryBuilder $query + * @param ISearchQuery $searchQuery + * @param array $caches + * @param IMetadataQuery|null $metadataQuery */ - public function shouldJoinTags(ISearchOperator $operator) { - if ($operator instanceof ISearchBinaryOperator) { - return array_reduce($operator->getArguments(), function ($shouldJoin, ISearchOperator $operator) { - return $shouldJoin || $this->shouldJoinTags($operator); - }, false); - } elseif ($operator instanceof ISearchComparison) { - return $operator->getField() === 'tagname' || $operator->getField() === 'favorite'; + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null, + ): void { + $storageFilters = array_values(array_map(function (ICache $cache) { + return $cache->getQueryFilterForStorage(); + }, $caches)); + $storageFilter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters); + $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); + $this->queryOptimizer->processOperator($filter); + + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); + if ($searchExpr) { + $query->andWhere($searchExpr); + } + + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); + + if ($searchQuery->getLimit()) { + $query->setMaxResults($searchQuery->getLimit()); + } + if ($searchQuery->getOffset()) { + $query->setFirstResult($searchQuery->getOffset()); } - return false; } + /** - * @param IQueryBuilder $builder - * @param ISearchOperator $operator + * @return array<array-key, array{id: int, name: string, visibility: int, editable: int, ref_file_id: int, number_files: int}> */ - public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) { - return array_filter(array_map(function ($operator) use ($builder) { - return $this->searchOperatorToDBExpr($builder, $operator); - }, $operators)); - } + public function findUsedTagsInCaches(ISearchQuery $searchQuery, array $caches): array { + $query = $this->getQueryBuilder(); + $query->selectTagUsage(); - public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) { - $expr = $builder->expr(); - if ($operator instanceof ISearchBinaryOperator) { - if (count($operator->getArguments()) === 0) { - return null; - } + $this->applySearchConstraints($query, $searchQuery, $caches); - switch ($operator->getType()) { - case ISearchBinaryOperator::OPERATOR_NOT: - $negativeOperator = $operator->getArguments()[0]; - if ($negativeOperator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap); - } else { - throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); - } - // no break - case ISearchBinaryOperator::OPERATOR_AND: - return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); - case ISearchBinaryOperator::OPERATOR_OR: - return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); - default: - throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); - } - } elseif ($operator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap); - } else { - throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); - } + $result = $query->executeQuery(); + $tags = $result->fetchAll(); + $result->closeCursor(); + return $tags; } - private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) { - $this->validateComparison($comparison); - - list($field, $value, $type) = $this->getOperatorFieldAndValue($comparison); - if (isset($operatorMap[$type])) { - $queryOperator = $operatorMap[$type]; - return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value)); - } else { - throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); + protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user): void { + $query->leftJoin('file', 'systemtag_object_mapping', 'systemtagmap', $query->expr()->andX( + $query->expr()->eq('file.fileid', $query->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')) + )); + $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid')); + if (!$this->groupManager->isAdmin($user->getUID())) { + $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true))); } + $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on); } - private function getOperatorFieldAndValue(ISearchComparison $operator) { - $field = $operator->getField(); - $value = $operator->getValue(); - $type = $operator->getType(); - if ($field === 'mimetype') { - if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) { - $value = (int)$this->mimetypeLoader->getId($value); - } elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) { - // transform "mimetype='foo/%'" to "mimepart='foo'" - if (preg_match('|(.+)/%|', $value, $matches)) { - $field = 'mimepart'; - $value = (int)$this->mimetypeLoader->getId($matches[1]); - $type = ISearchComparison::COMPARE_EQUAL; - } elseif (strpos($value, '%') !== false) { - throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); - } else { - $field = 'mimetype'; - $value = (int)$this->mimetypeLoader->getId($value); - $type = ISearchComparison::COMPARE_EQUAL; - } - } - } elseif ($field === 'favorite') { - $field = 'tag.category'; - $value = self::TAG_FAVORITE; - } elseif ($field === 'tagname') { - $field = 'tag.category'; - } elseif ($field === 'fileid') { - $field = 'file.fileid'; - } - return [$field, $value, $type]; + protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void { + $query + ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid')) + ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( + $query->expr()->eq('tagmap.categoryid', 'tag.id'), + $query->expr()->eq('tag.type', $query->createNamedParameter('files')), + $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID())) + )); } - private function validateComparison(ISearchComparison $operator) { - $types = [ - 'mimetype' => 'string', - 'mtime' => 'integer', - 'name' => 'string', - 'size' => 'integer', - 'tagname' => 'string', - 'favorite' => 'boolean', - 'fileid' => 'integer' - ]; - $comparisons = [ - 'mimetype' => ['eq', 'like'], - 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], - 'name' => ['eq', 'like'], - 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], - 'tagname' => ['eq', 'like'], - 'favorite' => ['eq'], - 'fileid' => ['eq'] - ]; - - if (!isset($types[$operator->getField()])) { - throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); + + protected function equipQueryForShares(CacheQueryBuilder $query): void { + $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); + } + + /** + * Perform a file system search in multiple caches + * + * the results will be grouped by the same array keys as the $caches argument to allow + * post-processing based on which cache the result came from + * + * @template T of array-key + * @param ISearchQuery $searchQuery + * @param array<T, ICache> $caches + * @return array<T, ICacheEntry[]> + */ + public function searchInCaches(ISearchQuery $searchQuery, array $caches): array { + // search in multiple caches at once by creating one query in the following format + // SELECT ... FROM oc_filecache WHERE + // <filter expressions from the search query> + // AND ( + // <filter expression for storage1> OR + // <filter expression for storage2> OR + // ... + // ); + // + // This gives us all the files matching the search query from all caches + // + // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id) + // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result. + + $builder = $this->getQueryBuilder(); + + $query = $builder->selectFileCache('file', false); + + $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + + if (in_array('systemtag', $requestedFields)) { + $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } - $type = $types[$operator->getField()]; - if (gettype($operator->getValue()) !== $type) { - throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { + $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } - if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { - throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); } - } - private function getParameterForValue(IQueryBuilder $builder, $value) { - if ($value instanceof \DateTime) { - $value = $value->getTimestamp(); + $metadataQuery = $query->selectMetadata(); + + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); + + $result = $query->executeQuery(); + $files = $result->fetchAll(); + + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return Cache::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); + + $result->closeCursor(); + + // loop through all caches for each result to see if the result matches that storage + // results are grouped by the same array keys as the caches argument to allow the caller to distinguish the source of the results + $results = array_fill_keys(array_keys($caches), []); + foreach ($rawEntries as $rawEntry) { + foreach ($caches as $cacheKey => $cache) { + $entry = $cache->getCacheEntryFromSearchResult($rawEntry); + if ($entry) { + $results[$cacheKey][] = $entry; + } + } } - if (is_numeric($value)) { - $type = IQueryBuilder::PARAM_INT; - } else { - $type = IQueryBuilder::PARAM_STR; + return $results; + } + + protected function requireUser(ISearchQuery $searchQuery): IUser { + $user = $searchQuery->getUser(); + if ($user === null) { + throw new \InvalidArgumentException('This search operation requires the user to be set in the query'); } - return $builder->createNamedParameter($value, $type); + return $user; } /** - * @param IQueryBuilder $query - * @param ISearchOrder[] $orders + * @return list{0?: array<array-key, ICache>, 1?: array<array-key, IMountPoint>} */ - public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) { - foreach ($orders as $order) { - $query->addOrderBy($order->getField(), $order->getDirection()); + public function getCachesAndMountPointsForSearch(IRootFolder $root, string $path, bool $limitToHome = false): array { + $rootLength = strlen($path); + $mount = $root->getMount($path); + $storage = $mount->getStorage(); + if ($storage === null) { + return []; + } + $internalPath = $mount->getInternalPath($path); + + if ($internalPath !== '') { + // a temporary CacheJail is used to handle filtering down the results to within this folder + /** @var ICache[] $caches */ + $caches = ['' => new CacheJail($storage->getCache(''), $internalPath)]; + } else { + /** @var ICache[] $caches */ + $caches = ['' => $storage->getCache('')]; + } + /** @var IMountPoint[] $mountByMountPoint */ + $mountByMountPoint = ['' => $mount]; + + if (!$limitToHome) { + $mounts = $root->getMountsIn($path); + foreach ($mounts as $mount) { + $storage = $mount->getStorage(); + if ($storage) { + $relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/'); + $caches[$relativeMountPoint] = $storage->getCache(''); + $mountByMountPoint[$relativeMountPoint] = $mount; + } + } } + + return [$caches, $mountByMountPoint]; } } diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 0dbc34fae2f..b067f70b8cb 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -1,48 +1,25 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Ari Selseng <ari@selseng.net> - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Martin Mattel <martin.mattel@diemattels.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Owen Winkler <a_github@midnightcircus.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use Doctrine\DBAL\Exception; -use OC\Files\Filesystem; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; use OC\Hooks\BasicEmitter; +use OC\SystemConfig; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; -use OCP\ILogger; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use Psr\Log\LoggerInterface; /** * Class Scanner @@ -86,12 +63,18 @@ class Scanner extends BasicEmitter implements IScanner { */ protected $lockingProvider; + protected IDBConnection $connection; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); - $this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false); - $this->lockingProvider = \OC::$server->getLockingProvider(); + /** @var SystemConfig $config */ + $config = \OC::$server->get(SystemConfig::class); + $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); + $this->lockingProvider = \OC::$server->get(ILockingProvider::class); + $this->connection = \OC::$server->get(IDBConnection::class); } /** @@ -100,7 +83,7 @@ class Scanner extends BasicEmitter implements IScanner { * * @param bool $useTransactions */ - public function setUseTransactions($useTransactions) { + public function setUseTransactions($useTransactions): void { $this->useTransactions = $useTransactions; } @@ -109,12 +92,12 @@ class Scanner extends BasicEmitter implements IScanner { * * * * @param string $path - * @return array an array of metadata of the file + * @return array|null an array of metadata of the file */ protected function getData($path) { $data = $this->storage->getMetaData($path); if (is_null($data)) { - \OCP\Util::writeLog(Scanner::class, "!!! Path '$path' is not accessible or present !!!", ILogger::DEBUG); + \OC::$server->get(LoggerInterface::class)->debug("!!! Path '$path' is not accessible or present !!!", ['app' => 'core']); } return $data; } @@ -125,10 +108,10 @@ class Scanner extends BasicEmitter implements IScanner { * @param string $file * @param int $reuseExisting * @param int $parentId - * @param array|null|false $cacheData existing data in the cache for the file to be scanned + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param null $data the metadata for the file, as returned by the storage - * @return array an array of metadata of the scanned file + * @param array|null $data the metadata for the file, as returned by the storage + * @return array|null an array of metadata of the scanned file * @throws \OCP\Lock\LockedException */ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { @@ -139,132 +122,130 @@ class Scanner extends BasicEmitter implements IScanner { return null; } } - // only proceed if $file is not a partial file nor a blacklisted file - if (!self::isPartialFile($file) and !Filesystem::isFileBlacklisted($file)) { - //acquire a lock + // only proceed if $file is not a partial file, blacklist is handled by the storage + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } } - try { - $data = $data ?? $this->getData($file); - } catch (ForbiddenException $e) { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } - } + return null; + } - return null; - } + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + } - try { - if ($data) { + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } - // pre-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; } - $parent = dirname($file); - if ($parent === '.' or $parent === '/') { - $parent = ''; - } - if ($parentId === -1) { - $parentId = $this->cache->getParentId($file); - } + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } - // scan the parent if it's not in the cache (id -1) and the current file is not the root folder - if ($file and $parentId === -1) { - $parentData = $this->scanFile($parent); - if (!$parentData) { - return null; - } - $parentId = $parentData['fileid']; - } - if ($parent) { - $data['parent'] = $parentId; - } - if (is_null($cacheData)) { - /** @var CacheEntry $cacheData */ - $cacheData = $this->cache->get($file); - } - if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { - // prevent empty etag - if (empty($cacheData['etag'])) { - $etag = $data['etag']; - } else { - $etag = $cacheData['etag']; + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; } - $fileId = $cacheData['fileid']; - $data['fileid'] = $fileId; - // only reuse data if the file hasn't explicitly changed - if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { - $data['mtime'] = $cacheData['mtime']; - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { - $data['size'] = $cacheData['size']; - } - if ($reuseExisting & self::REUSE_ETAG) { - $data['etag'] = $etag; - } + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; } - // Only update metadata that has changed - $newData = array_diff_assoc($data, $cacheData->getData()); - } else { - $newData = $data; - $fileId = -1; - } - if (!empty($newData)) { - // Reset the checksum if the data has changed - $newData['checksum'] = ''; - $newData['parent'] = $parentId; - $data['fileid'] = $this->addToCache($file, $newData, $fileId); - } - if ($cacheData && isset($cacheData['size'])) { - $data['oldSize'] = $cacheData['size']; - } else { - $data['oldSize'] = 0; } - if ($cacheData && isset($cacheData['encrypted'])) { - $data['encrypted'] = $cacheData['encrypted']; + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); } - // post-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; } } else { - $this->removeFromCache($file); + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; } - } catch (\Exception $e) { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); } - throw $e; - } - //release the acquired lock - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; } - } - if ($data && !isset($data['encrypted'])) { - $data['encrypted'] = false; + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } - return $data; } - return null; + return $data; } protected function removeFromCache($path) { @@ -286,7 +267,7 @@ class Scanner extends BasicEmitter implements IScanner { $data['permissions'] = $data['scan_permissions']; } \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); - $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data, $fileId]); if ($this->cacheActive) { if ($fileId !== -1) { $this->cache->update($fileId, $data); @@ -323,42 +304,88 @@ class Scanner extends BasicEmitter implements IScanner { * @param bool $recursive * @param int $reuse * @param bool $lock set to false to disable getting an additional read lock during scanning - * @return array an array of the meta data of the scanned file or folder + * @return array|null an array of the meta data of the scanned file or folder */ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } + try { - $data = $this->scanFile($path, $reuse, -1, null, $lock); - if ($data and $data['mimetype'] === 'httpd/unix-directory') { - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock); + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; } finally { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - } + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); } } return $data; } /** + * Compares $array1 against $array2 and returns all the values in $array1 that are not in $array2 + * Note this is a one-way check - i.e. we don't care about things that are in $array2 that aren't in $array1 + * + * Supports multi-dimensional arrays + * Also checks keys/indexes + * Comparisons are strict just like array_diff_assoc + * Order of keys/values does not matter + * + * @param array $array1 + * @param array $array2 + * @return array with the differences between $array1 and $array1 + * @throws \InvalidArgumentException if $array1 isn't an actual array + * + */ + protected function array_diff_assoc_multi(array $array1, array $array2) { + + $result = []; + + foreach ($array1 as $key => $value) { + + // if $array2 doesn't have the same key, that's a result + if (!array_key_exists($key, $array2)) { + $result[$key] = $value; + continue; + } + + // if $array2's value for the same key is different, that's a result + if ($array2[$key] !== $value && !is_array($value)) { + $result[$key] = $value; + continue; + } + + if (is_array($value)) { + $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); + if (!empty($nestedDiff)) { + $result[$key] = $nestedDiff; + continue; + } + } + } + return $result; + } + + /** * Get the children currently in the cache * * @param int $folderId - * @return array[] + * @return array<string, \OCP\Files\Cache\ICacheEntry> */ - protected function getExistingChildren($folderId) { + protected function getExistingChildren($folderId): array { $existingChildren = []; $children = $this->cache->getFolderContentsById($folderId); foreach ($children as $child) { @@ -371,90 +398,128 @@ class Scanner extends BasicEmitter implements IScanner { * scan all the files and folders in a folder * * @param string $path - * @param bool $recursive - * @param int $reuse + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + * @param int $reuse a combination of self::REUSE_* * @param int $folderId id for the folder to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @return int the size of the scanned folder or -1 if the size is unknown at this stage + * @param int|float $oldSize the size of the folder before (re)scanning the children + * @return int|float the size of the scanned folder or -1 if the size is unknown at this stage */ - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]); $size = 0; - if (!is_null($folderId)) { - $folderId = $this->cache->getId($path); - } - $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + + foreach ($childQueue as $child => [$childId, $childSize]) { + // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed + $childEtagChanged = false; + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize, $childEtagChanged); + $etagChanged |= $childEtagChanged; - foreach ($childQueue as $child => $childId) { - $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); if ($childSize === -1) { $size = -1; } elseif ($size !== -1) { $size += $childSize; } } - if ($this->cacheActive) { - $this->cache->update($folderId, ['size' => $size]); + + // for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size + // to make sure we also updated the unencrypted-size where applicable + if ($this->storage->instanceOfStorage(Encryption::class)) { + $this->cache->calculateFolderSize($path); + } else { + if ($this->cacheActive) { + $updatedData = []; + if ($oldSize !== $size) { + $updatedData['size'] = $size; + } + if ($etagChanged) { + $updatedData['etag'] = uniqid(); + } + if ($updatedData) { + $this->cache->update($folderId, $updatedData); + } + } } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); return $size; } - private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { + /** + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + */ + private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array { // we put this in it's own function so it cleans up the memory before we start recursing $existingChildren = $this->getExistingChildren($folderId); $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); + if (count($existingChildren) === 0 && count($newChildren) === 0) { + // no need to do a transaction + return []; + } + if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->beginTransaction(); + $this->connection->beginTransaction(); } $exceptionOccurred = false; $childQueue = []; $newChildNames = []; foreach ($newChildren as $fileMeta) { - $permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions']; + $permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions']; if ($permissions === 0) { continue; } - $file = $fileMeta['name']; + $originalFile = $fileMeta['name']; + $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/'); + if (trim($originalFile, '/') !== $file) { + // encoding mismatch, might require compatibility wrapper + \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".', ['app' => 'core']); + $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]); + // skip this entry + continue; + } + $newChildNames[] = $file; $child = $path ? $path . '/' . $file : $file; try { - $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false; + $existingData = $existingChildren[$file] ?? false; $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta); if ($data) { - if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { - $childQueue[$child] = $data['fileid']; - } elseif ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE_INCOMPLETE and $data['size'] === -1) { + if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) { + $childQueue[$child] = [$data['fileid'], $data['size']]; + } elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) { // only recurse into folders which aren't fully scanned - $childQueue[$child] = $data['fileid']; + $childQueue[$child] = [$data['fileid'], $data['size']]; } elseif ($data['size'] === -1) { $size = -1; } elseif ($size !== -1) { $size += $data['size']; } + + if (isset($data['etag_changed']) && $data['etag_changed']) { + $etagChanged = true; + } } } catch (Exception $ex) { // might happen if inserting duplicate while a scanning // process is running in parallel // log and ignore if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->rollback(); - \OC::$server->getDatabaseConnection()->beginTransaction(); + $this->connection->rollback(); + $this->connection->beginTransaction(); } - \OC::$server->getLogger()->logException($ex, [ - 'message' => 'Exception while scanning file "' . $child . '"', - 'level' => ILogger::DEBUG, + \OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [ 'app' => 'core', + 'exception' => $ex, ]); $exceptionOccurred = true; } catch (\OCP\Lock\LockedException $e) { if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->rollback(); + $this->connection->rollback(); } throw $e; } @@ -465,7 +530,7 @@ class Scanner extends BasicEmitter implements IScanner { $this->removeFromCache($child); } if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->commit(); + $this->connection->commit(); } if ($exceptionOccurred) { // It might happen that the parallel scan process has already @@ -489,7 +554,7 @@ class Scanner extends BasicEmitter implements IScanner { if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { return true; } - if (strpos($file, '.part/') !== false) { + if (str_contains($file, '.part/')) { return true; } @@ -500,24 +565,36 @@ class Scanner extends BasicEmitter implements IScanner { * walk over any folders that are not fully scanned yet and scan them */ public function backgroundScan() { - if (!$this->cache->inCache('')) { - $this->runBackgroundScanJob(function () { - $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); - }, ''); + if ($this->storage->instanceOfStorage(Jail::class)) { + // for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage + // this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient). + // + // Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will + // have to be scanned at some point anyway. + $unJailedScanner = $this->storage->getUnjailedStorage()->getScanner(); + $unJailedScanner->backgroundScan(); } else { - $lastPath = null; - while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { - $this->runBackgroundScanJob(function () use ($path) { - $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); - }, $path); - // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() - // to make this possible - $lastPath = $path; + if (!$this->cache->inCache('')) { + // if the storage isn't in the cache yet, just scan the root completely + $this->runBackgroundScanJob(function () { + $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); + }, ''); + } else { + $lastPath = null; + // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck) + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function () use ($path) { + $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } } } } - private function runBackgroundScanJob(callable $callback, $path) { + protected function runBackgroundScanJob(callable $callback, $path) { try { $callback(); \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]); diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php new file mode 100644 index 00000000000..e1d3c42a8a2 --- /dev/null +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -0,0 +1,355 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; + +/** + * Tools for transforming search queries into database queries + * + * @psalm-import-type ParamSingleValue from ISearchComparison + * @psalm-import-type ParamValue from ISearchComparison + */ +class SearchBuilder { + /** @var array<string, string> */ + protected static $searchOperatorMap = [ + ISearchComparison::COMPARE_LIKE => 'iLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like', + ISearchComparison::COMPARE_EQUAL => 'eq', + ISearchComparison::COMPARE_GREATER_THAN => 'gt', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', + ISearchComparison::COMPARE_LESS_THAN => 'lt', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte', + ISearchComparison::COMPARE_DEFINED => 'isNotNull', + ISearchComparison::COMPARE_IN => 'in', + ]; + + /** @var array<string, string> */ + protected static $searchOperatorNegativeMap = [ + ISearchComparison::COMPARE_LIKE => 'notLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike', + ISearchComparison::COMPARE_EQUAL => 'neq', + ISearchComparison::COMPARE_GREATER_THAN => 'lte', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', + ISearchComparison::COMPARE_LESS_THAN => 'gte', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt', + ISearchComparison::COMPARE_DEFINED => 'isNull', + ISearchComparison::COMPARE_IN => 'notIn', + ]; + + /** @var array<string, string> */ + protected static $fieldTypes = [ + 'mimetype' => 'string', + 'mtime' => 'integer', + 'name' => 'string', + 'path' => 'string', + 'size' => 'integer', + 'tagname' => 'string', + 'systemtag' => 'string', + 'favorite' => 'boolean', + 'fileid' => 'integer', + 'storage' => 'integer', + 'share_with' => 'string', + 'share_type' => 'integer', + 'owner' => 'string', + ]; + + /** @var array<string, int|string> */ + protected static $paramTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR, + 'integer' => IQueryBuilder::PARAM_INT, + 'boolean' => IQueryBuilder::PARAM_BOOL, + ]; + + /** @var array<string, int> */ + protected static $paramArrayTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR_ARRAY, + 'integer' => IQueryBuilder::PARAM_INT_ARRAY, + 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, + ]; + + public const TAG_FAVORITE = '_$!<Favorite>!$_'; + + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @return string[] + */ + public function extractRequestedFields(ISearchOperator $operator): array { + if ($operator instanceof ISearchBinaryOperator) { + return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { + return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); + }, []); + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { + return [$operator->getField()]; + } + return []; + } + + /** + * @param IQueryBuilder $builder + * @param ISearchOperator[] $operators + */ + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null, + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); + }, $operators)); + } + + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null, + ) { + $expr = $builder->expr(); + + if ($operator instanceof ISearchBinaryOperator) { + if (count($operator->getArguments()) === 0) { + return null; + } + + switch ($operator->getType()) { + case ISearchBinaryOperator::OPERATOR_NOT: + $negativeOperator = $operator->getArguments()[0]; + if ($negativeOperator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); + } + // no break + case ISearchBinaryOperator::OPERATOR_AND: + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + case ISearchBinaryOperator::OPERATOR_OR: + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + default: + throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); + } + } elseif ($operator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); + } + } + + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null, + ) { + if ($comparison->getExtra()) { + [$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison); + } + + if (isset($operatorMap[$type])) { + $queryOperator = $operatorMap[$type]; + return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType)); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); + } + } + + /** + * @param ISearchComparison $operator + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValue(ISearchComparison $operator): array { + $this->validateComparison($operator); + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + $pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true); + return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash); + } + + /** + * @param string $field + * @param ParamValue $value + * @param string $type + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array { + $paramType = self::$fieldTypes[$field]; + if ($type === ISearchComparison::COMPARE_IN) { + $resultField = $field; + $values = []; + foreach ($value as $arrayValue) { + /** @var ParamSingleValue $arrayValue */ + [$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash); + $resultField = $arrayField; + $values[] = $arrayValue; + } + return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType]; + } + if ($field === 'mimetype') { + $value = (string)$value; + if ($type === ISearchComparison::COMPARE_EQUAL) { + $value = $this->mimetypeLoader->getId($value); + } elseif ($type === ISearchComparison::COMPARE_LIKE) { + // transform "mimetype='foo/%'" to "mimepart='foo'" + if (preg_match('|(.+)/%|', $value, $matches)) { + $field = 'mimepart'; + $value = $this->mimetypeLoader->getId($matches[1]); + $type = ISearchComparison::COMPARE_EQUAL; + } elseif (str_contains($value, '%')) { + throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); + } else { + $field = 'mimetype'; + $value = $this->mimetypeLoader->getId($value); + $type = ISearchComparison::COMPARE_EQUAL; + } + } + } elseif ($field === 'favorite') { + $field = 'tag.category'; + $value = self::TAG_FAVORITE; + $paramType = 'string'; + } elseif ($field === 'name') { + $field = 'file.name'; + } elseif ($field === 'tagname') { + $field = 'tag.category'; + } elseif ($field === 'systemtag') { + $field = 'systemtag.name'; + } elseif ($field === 'fileid') { + $field = 'file.fileid'; + } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { + $field = 'path_hash'; + $value = md5((string)$value); + } elseif ($field === 'owner') { + $field = 'uid_owner'; + } + return [$field, $value, $type, $paramType]; + } + + private function validateComparison(ISearchComparison $operator) { + $comparisons = [ + 'mimetype' => ['eq', 'like', 'in'], + 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'name' => ['eq', 'like', 'clike', 'in'], + 'path' => ['eq', 'like', 'clike', 'in'], + 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'tagname' => ['eq', 'like'], + 'systemtag' => ['eq', 'like'], + 'favorite' => ['eq'], + 'fileid' => ['eq', 'in'], + 'storage' => ['eq', 'in'], + 'share_with' => ['eq'], + 'share_type' => ['eq'], + 'owner' => ['eq'], + ]; + + if (!isset(self::$fieldTypes[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); + } + $type = self::$fieldTypes[$operator->getField()]; + if ($operator->getType() === ISearchComparison::COMPARE_IN) { + if (!is_array($operator->getValue())) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + foreach ($operator->getValue() as $arrayValue) { + if (gettype($arrayValue) !== $type) { + throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField()); + } + } + } else { + if (gettype($operator->getValue()) !== $type) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + } + if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); + } + } + + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + + switch ($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type, $paramType]; + } + + private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) { + if ($value instanceof \DateTime) { + $value = $value->getTimestamp(); + } + if (is_array($value)) { + $type = self::$paramArrayTypeMap[$paramType]; + } else { + $type = self::$paramTypeMap[$paramType]; + } + return $builder->createNamedParameter($value, $type); + } + + /** + * @param IQueryBuilder $query + * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery + */ + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { + foreach ($orders as $order) { + $field = $order->getField(); + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; + + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } + $query->addOrderBy($field, $order->getDirection()); + } + } +} diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 74f5df2a5b1..1a3bda58e6a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -1,35 +1,16 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Storage\IStorage; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; /** * Handle the mapping between the string and numeric storage ids @@ -38,7 +19,7 @@ use OCP\Files\Storage\IStorage; * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') * and a numeric storage id which is referenced in the file cache * - * A mapping between the two storage ids is stored in the database and accessible trough this class + * A mapping between the two storage ids is stored in the database and accessible through this class * * @package OC\Files\Cache */ @@ -63,7 +44,7 @@ class Storage { * @param bool $isAvailable * @throws \RuntimeException */ - public function __construct($storage, $isAvailable = true) { + public function __construct($storage, $isAvailable, IDBConnection $connection) { if ($storage instanceof IStorage) { $this->storageId = $storage->getId(); } else { @@ -74,10 +55,9 @@ class Storage { if ($row = self::getStorageById($this->storageId)) { $this->numericId = (int)$row['numeric_id']; } else { - $connection = \OC::$server->getDatabaseConnection(); $available = $isAvailable ? 1 : 0; if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { - $this->numericId = (int)$connection->lastInsertId('*PREFIX*storages'); + $this->numericId = $connection->lastInsertId('*PREFIX*storages'); } else { if ($row = self::getStorageById($this->storageId)) { $this->numericId = (int)$row['numeric_id']; @@ -100,7 +80,7 @@ class Storage { * Adjusts the storage id to use md5 if too long * @param string $storageId storage id * @return string unchanged $storageId if its length is less than 64 characters, - * else returns the md5 of $storageId + * else returns the md5 of $storageId */ public static function adjustStorageId($storageId) { if (strlen($storageId) > 64) { @@ -124,26 +104,16 @@ class Storage { * @param int $numericId * @return string|null either the storage id string or null if the numeric id is not known */ - public static function getStorageId($numericId) { - $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); - $query->select('id') - ->from('storages') - ->where($query->expr()->eq('numeric_id', $query->createNamedParameter($numericId))); - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - if ($row) { - return $row['id']; - } else { - return null; - } + public static function getStorageId(int $numericId): ?string { + $storage = self::getGlobalCache()->getStorageInfoByNumericId($numericId); + return $storage['id'] ?? null; } /** * Get the numeric of the storage with the provided string id * * @param $storageId - * @return int|null either the numeric storage id or null if the storage id is not knwon + * @return int|null either the numeric storage id or null if the storage id is not known */ public static function getNumericStorageId($storageId) { $storageId = self::adjustStorageId($storageId); @@ -156,7 +126,7 @@ class Storage { } /** - * @return array|null [ available, last_checked ] + * @return array [ available, last_checked ] */ public function getAvailability() { if ($row = self::getStorageById($this->storageId)) { @@ -165,7 +135,10 @@ class Storage { 'last_checked' => $row['last_checked'] ]; } else { - return null; + return [ + 'available' => true, + 'last_checked' => time(), + ]; } } @@ -175,13 +148,16 @@ class Storage { */ public function setAvailability($isAvailable, int $delay = 0) { $available = $isAvailable ? 1 : 0; + if (!$isAvailable) { + \OCP\Server::get(LoggerInterface::class)->info('Storage with ' . $this->storageId . ' marked as unavailable', ['app' => 'lib']); + } $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->update('storages') ->set('available', $query->createNamedParameter($available)) ->set('last_checked', $query->createNamedParameter(time() + $delay)) ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); - $query->execute(); + $query->executeStatement(); } /** @@ -206,13 +182,54 @@ class Storage { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($storageId))); - $query->execute(); + $query->executeStatement(); if (!is_null($numericId)) { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->eq('storage', $query->createNamedParameter($numericId))); - $query->execute(); + $query->executeStatement(); + } + } + + /** + * remove the entry for the storage by the mount id + * + * @param int $mountId + */ + public static function cleanByMountId(int $mountId) { + $db = \OC::$server->getDatabaseConnection(); + + try { + $db->beginTransaction(); + + $query = $db->getQueryBuilder(); + $query->select('storage_id') + ->from('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $storageIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + $storageIds = array_unique($storageIds); + + $query = $db->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->in('numeric_id', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $query->executeStatement(); + + $db->commit(); + } catch (\Exception $e) { + $db->rollBack(); + throw $e; } } } diff --git a/lib/private/Files/Cache/StorageGlobal.php b/lib/private/Files/Cache/StorageGlobal.php index 23d6035084d..bab31b1db91 100644 --- a/lib/private/Files/Cache/StorageGlobal.php +++ b/lib/private/Files/Cache/StorageGlobal.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -34,19 +16,19 @@ use OCP\IDBConnection; * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') * and a numeric storage id which is referenced in the file cache * - * A mapping between the two storage ids is stored in the database and accessible trough this class + * A mapping between the two storage ids is stored in the database and accessible through this class * * @package OC\Files\Cache */ class StorageGlobal { - /** @var IDBConnection */ - private $connection; - - /** @var array[] */ + /** @var array<string, array> */ private $cache = []; + /** @var array<int, array> */ + private $numericIdCache = []; - public function __construct(IDBConnection $connection) { - $this->connection = $connection; + public function __construct( + private IDBConnection $connection, + ) { } /** @@ -58,7 +40,7 @@ class StorageGlobal { ->from('storages') ->where($builder->expr()->in('id', $builder->createNamedParameter(array_values($storageIds), IQueryBuilder::PARAM_STR_ARRAY))); - $result = $query->execute(); + $result = $query->executeQuery(); while ($row = $result->fetch()) { $this->cache[$row['id']] = $row; } @@ -69,22 +51,46 @@ class StorageGlobal { * @param string $storageId * @return array|null */ - public function getStorageInfo($storageId) { + public function getStorageInfo(string $storageId): ?array { if (!isset($this->cache[$storageId])) { $builder = $this->connection->getQueryBuilder(); $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) ->from('storages') ->where($builder->expr()->eq('id', $builder->createNamedParameter($storageId))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); if ($row) { $this->cache[$storageId] = $row; + $this->numericIdCache[(int)$row['numeric_id']] = $row; + } + } + return $this->cache[$storageId] ?? null; + } + + /** + * @param int $numericId + * @return array|null + */ + public function getStorageInfoByNumericId(int $numericId): ?array { + if (!isset($this->numericIdCache[$numericId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->eq('numeric_id', $builder->createNamedParameter($numericId))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $this->numericIdCache[$numericId] = $row; + $this->cache[$row['id']] = $row; } } - return isset($this->cache[$storageId]) ? $this->cache[$storageId] : null; + return $this->numericIdCache[$numericId] ?? null; } public function clearCache() { diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php index 04179ba0aaf..03681036aa2 100644 --- a/lib/private/Files/Cache/Updater.php +++ b/lib/private/Files/Cache/Updater.php @@ -1,37 +1,20 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Michael Gapczynski <GapczynskiM@gmail.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; +use Doctrine\DBAL\Exception\DeadlockException; use OC\Files\FileInfo; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Cache\IUpdater; use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; /** * Update the cache and propagate changes @@ -63,6 +46,8 @@ class Updater implements IUpdater { */ protected $cache; + private LoggerInterface $logger; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -71,17 +56,18 @@ class Updater implements IUpdater { $this->propagator = $storage->getPropagator(); $this->scanner = $storage->getScanner(); $this->cache = $storage->getCache(); + $this->logger = \OC::$server->get(LoggerInterface::class); } /** - * Disable updating the cache trough this updater + * Disable updating the cache through this updater */ public function disable() { $this->enabled = false; } /** - * Re-enable the updating of the cache trough this updater + * Re-enable the updating of the cache through this updater */ public function enable() { $this->enabled = true; @@ -115,7 +101,7 @@ class Updater implements IUpdater { * @param string $path * @param int $time */ - public function update($path, $time = null) { + public function update($path, $time = null, ?int $sizeDifference = null) { if (!$this->enabled or Scanner::isPartialFile($path)) { return; } @@ -124,20 +110,22 @@ class Updater implements IUpdater { } $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); - if ( - isset($data['oldSize']) && isset($data['size']) && - !$data['encrypted'] // encryption is a pita and touches the cache itself - ) { + + if (isset($data['oldSize']) && isset($data['size'])) { $sizeDifference = $data['size'] - $data['oldSize']; - } else { - // scanner didn't provide size info, fallback to full size calculation - $sizeDifference = 0; - if ($this->cache instanceof Cache) { - $this->cache->correctFolderSize($path, $data); - } + } + + // encryption is a pita and touches the cache itself + if (isset($data['encrypted']) && (bool)$data['encrypted']) { + $sizeDifference = null; + } + + // scanner didn't provide size info, fallback to full size calculation + if ($this->cache instanceof Cache && $sizeDifference === null) { + $this->cache->correctFolderSize($path, $data); } $this->correctParentStorageMtime($path); - $this->propagator->propagateChange($path, $time, $sizeDifference); + $this->propagator->propagateChange($path, $time, $sizeDifference ?? 0); } /** @@ -171,13 +159,51 @@ class Updater implements IUpdater { } /** - * Rename a file or folder in the cache and update the size, etag and mtime of the parent folders + * Rename a file or folder in the cache. * * @param IStorage $sourceStorage * @param string $source * @param string $target */ public function renameFromStorage(IStorage $sourceStorage, $source, $target) { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache) use ($sourceStorage, $source, $target) { + // Remove existing cache entry to no reuse the fileId. + if ($this->cache->inCache($target)) { + $this->cache->remove($target); + } + + if ($sourceStorage === $this->storage) { + $this->cache->move($source, $target); + } else { + $this->cache->moveFromCache($sourceCache, $source, $target); + } + }); + } + + /** + * Copy a file or folder in the cache. + */ + public function copyFromStorage(IStorage $sourceStorage, string $source, string $target): void { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache, ICacheEntry $sourceInfo) use ($target) { + $parent = dirname($target); + if ($parent === '.') { + $parent = ''; + } + $parentInCache = $this->cache->inCache($parent); + if (!$parentInCache) { + $parentData = $this->scanner->scan($parent, Scanner::SCAN_SHALLOW, -1, false); + $parentInCache = $parentData !== null; + } + if ($parentInCache) { + $this->cache->copyFromCache($sourceCache, $sourceInfo, $target); + } + }); + } + + /** + * Utility to copy or rename a file or folder in the cache and update the size, etag and mtime of the parent folders + */ + private function copyOrRenameFromStorage(IStorage $sourceStorage, string $source, string $target, callable $operation): void { if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) { return; } @@ -190,27 +216,25 @@ class Updater implements IUpdater { $sourceInfo = $sourceCache->get($source); - if ($sourceInfo !== false) { - if ($this->cache->inCache($target)) { - $this->cache->remove($target); - } + $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); + $targetExtension = pathinfo($target, PATHINFO_EXTENSION); + $targetIsTrash = preg_match("/^d\d+$/", $targetExtension); - if ($sourceStorage === $this->storage) { - $this->cache->move($source, $target); - } else { - $this->cache->moveFromCache($sourceCache, $source, $target); + if ($sourceInfo !== false) { + if (!$this->storage->instanceOfStorage(ObjectStoreStorage::class)) { + $operation($sourceCache, $sourceInfo); } - $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); - $targetExtension = pathinfo($target, PATHINFO_EXTENSION); - $targetIsTrash = preg_match("/d\d+/", $targetExtension); + $isDir = $sourceInfo->getMimeType() === FileInfo::MIMETYPE_FOLDER; + } else { + $isDir = $this->storage->is_dir($target); + } - if ($sourceExtension !== $targetExtension && $sourceInfo->getMimeType() !== FileInfo::MIMETYPE_FOLDER && !$targetIsTrash) { - // handle mime type change - $mimeType = $this->storage->getMimeType($target); - $fileId = $this->cache->getId($target); - $this->cache->update($fileId, ['mimetype' => $mimeType]); - } + if ($sourceExtension !== $targetExtension && !$isDir && !$targetIsTrash) { + // handle mime type change + $mimeType = $this->storage->getMimeType($target); + $fileId = $this->cache->getId($target); + $this->cache->update($fileId, ['mimetype' => $mimeType]); } if ($sourceCache instanceof Cache) { @@ -254,7 +278,14 @@ class Updater implements IUpdater { if ($parentId != -1) { $mtime = $this->storage->filemtime($parent); if ($mtime !== false) { - $this->cache->update($parentId, ['storage_mtime' => $mtime]); + try { + $this->cache->update($parentId, ['storage_mtime' => $mtime]); + } catch (DeadlockException $e) { + // ignore the failure. + // with failures concurrent updates, someone else would have already done it. + // in the worst case the `storage_mtime` isn't updated, which should at most only trigger an extra rescan + $this->logger->warning('Error while updating parent storage_mtime, should be safe to ignore', ['exception' => $e]); + } } } } diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php index 19e17e60959..f1de5d3cfb8 100644 --- a/lib/private/Files/Cache/Watcher.php +++ b/lib/private/Files/Cache/Watcher.php @@ -1,30 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use OCP\Files\Cache\ICacheEntry; @@ -53,6 +33,9 @@ class Watcher implements IWatcher { */ protected $scanner; + /** @var callable[] */ + protected $onUpdate = []; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -89,7 +72,14 @@ class Watcher implements IWatcher { } if ($cachedEntry === false || $this->needsUpdate($path, $cachedEntry)) { $this->update($path, $cachedEntry); - return true; + + if ($cachedEntry === false) { + return true; + } else { + // storage backends can sometimes return false positives, only return true if the scanner actually found a change + $newEntry = $this->cache->get($path); + return $newEntry->getStorageMTime() > $cachedEntry->getStorageMTime(); + } } else { return false; } @@ -113,6 +103,9 @@ class Watcher implements IWatcher { if ($this->cache instanceof Cache) { $this->cache->correctFolderSize($path); } + foreach ($this->onUpdate as $callback) { + $callback($path); + } } /** @@ -123,9 +116,9 @@ class Watcher implements IWatcher { * @return bool */ public function needsUpdate($path, $cachedData) { - if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and array_search($path, $this->checkedPaths) === false)) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { $this->checkedPaths[] = $path; - return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); } return false; } @@ -143,4 +136,11 @@ class Watcher implements IWatcher { } } } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index 6c1c17be028..5bc4ee8529d 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -1,61 +1,66 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; -use OC\Files\Search\SearchQuery; +use OC\Files\Cache\CacheDependencies; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; -use OCP\Files\Search\ISearchQuery; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; /** * Jail to a subdirectory of the wrapped cache */ class CacheJail extends CacheWrapper { - /** - * @var string - */ - protected $root; - /** - * @param \OCP\Files\Cache\ICache $cache - * @param string $root - */ - public function __construct($cache, $root) { - parent::__construct($cache); - $this->root = $root; + protected string $unjailedRoot; + + public function __construct( + ?ICache $cache, + protected string $root, + ?CacheDependencies $dependencies = null, + ) { + parent::__construct($cache, $dependencies); + + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); + } } + /** + * @return string + */ protected function getRoot() { return $this->root; } - protected function getSourcePath($path) { + /** + * Get the root path with any nested jails resolved + * + * @return string + */ + public function getGetUnjailedRoot() { + return $this->unjailedRoot; + } + + /** + * @return string + */ + protected function getSourcePath(string $path) { if ($path === '') { return $this->getRoot(); } else { @@ -65,26 +70,26 @@ class CacheJail extends CacheWrapper { /** * @param string $path + * @param null|string $root * @return null|string the jailed path or null if the path is outside the jail */ - protected function getJailedPath($path) { - if ($this->getRoot() === '') { + protected function getJailedPath(string $path, ?string $root = null) { + if ($root === null) { + $root = $this->getRoot(); + } + if ($root === '') { return $path; } - $rootLength = strlen($this->getRoot()) + 1; - if ($path === $this->getRoot()) { + $rootLength = strlen($root) + 1; + if ($path === $root) { return ''; - } elseif (substr($path, 0, $rootLength) === $this->getRoot() . '/') { + } elseif (substr($path, 0, $rootLength) === $root . '/') { return substr($path, $rootLength); } else { return null; } } - /** - * @param ICacheEntry|array $entry - * @return array - */ protected function formatCacheEntry($entry) { if (isset($entry['path'])) { $entry['path'] = $this->getJailedPath($entry['path']); @@ -92,15 +97,10 @@ class CacheJail extends CacheWrapper { return $entry; } - protected function filterCacheEntry($entry) { - $rootLength = strlen($this->getRoot()) + 1; - return $rootLength === 1 || ($entry['path'] === $this->getRoot()) || (substr($entry['path'], 0, $rootLength) === $this->getRoot() . '/'); - } - /** * get the stored metadata of a file or folder * - * @param string /int $file + * @param string|int $file * @return ICacheEntry|false */ public function get($file) { @@ -208,54 +208,15 @@ class CacheJail extends CacheWrapper { return $this->getCache()->getStatus($this->getSourcePath($file)); } - private function formatSearchResults($results) { - $results = array_filter($results, [$this, 'filterCacheEntry']); - $results = array_values($results); - return array_map([$this, 'formatCacheEntry'], $results); - } - - /** - * search for files matching $pattern - * - * @param string $pattern - * @return array an array of file data - */ - public function search($pattern) { - $results = $this->getCache()->search($pattern); - return $this->formatSearchResults($results); - } - - /** - * search for files by mimetype - * - * @param string $mimetype - * @return array - */ - public function searchByMime($mimetype) { - $results = $this->getCache()->searchByMime($mimetype); - return $this->formatSearchResults($results); - } - - public function searchQuery(ISearchQuery $query) { - $simpleQuery = new SearchQuery($query->getSearchOperation(), 0, 0, $query->getOrder(), $query->getUser()); - $results = $this->getCache()->searchQuery($simpleQuery); - $results = $this->formatSearchResults($results); - - $limit = $query->getLimit() === 0 ? null : $query->getLimit(); - $results = array_slice($results, $query->getOffset(), $limit); - - return $results; - } - /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); } } @@ -263,12 +224,13 @@ class CacheJail extends CacheWrapper { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($this->getSourcePath($path), $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); } else { return 0; } @@ -291,7 +253,7 @@ class CacheJail extends CacheWrapper { * use the one with the highest id gives the best result with the background scanner, since that is most * likely the folder where we stopped scanning previously * - * @return string|bool the path of the folder or false when no folder matched + * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { // not supported @@ -328,4 +290,40 @@ class CacheJail extends CacheWrapper { } return $this->getCache()->moveFromCache($sourceCache, $sourcePath, $this->getSourcePath($targetPath)); } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->addJailFilterQuery($this->getCache()->getQueryFilterForStorage()); + } + + protected function addJailFilterQuery(ISearchOperator $filter): ISearchOperator { + if ($this->getGetUnjailedRoot() !== '' && $this->getGetUnjailedRoot() !== '/') { + return new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, + [ + $filter, + new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, + [ + new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'path', $this->getGetUnjailedRoot()), + new SearchComparison(ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE, 'path', SearchComparison::escapeLikeParameter($this->getGetUnjailedRoot()) . '/%'), + ], + ) + ] + ); + } else { + return $filter; + } + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($this->getGetUnjailedRoot() === '' || str_starts_with($rawEntry->getPath(), $this->getGetUnjailedRoot())) { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $jailedPath = $this->getJailedPath($rawEntry->getPath()); + if ($jailedPath !== null) { + return $this->formatCacheEntry(clone $rawEntry) ?: null; + } + } + } + + return null; + } } diff --git a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php index 706f5d832f4..ff17cb79ac7 100644 --- a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php +++ b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache\Wrapper; class CachePermissionsMask extends CacheWrapper { diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index cac6cfed87e..f2f1036d6a3 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -1,62 +1,60 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Ari Selseng <ari@selseng.net> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; +use OCP\Server; class CacheWrapper extends Cache { /** - * @var \OCP\Files\Cache\ICache + * @var ?ICache */ protected $cache; - /** - * @param \OCP\Files\Cache\ICache $cache - */ - public function __construct($cache) { + public function __construct(?ICache $cache, ?CacheDependencies $dependencies = null) { $this->cache = $cache; + if (!$dependencies && $cache instanceof Cache) { + $this->mimetypeLoader = $cache->mimetypeLoader; + $this->connection = $cache->connection; + $this->querySearchHelper = $cache->querySearchHelper; + } else { + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + } } protected function getCache() { return $this->cache; } + protected function hasEncryptionWrapper(): bool { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->hasEncryptionWrapper(); + } else { + return false; + } + } + /** * Make it easy for wrappers to modify every returned cache entry * * @param ICacheEntry $entry - * @return ICacheEntry + * @return ICacheEntry|false */ protected function formatCacheEntry($entry) { return $entry; @@ -70,7 +68,7 @@ class CacheWrapper extends Cache { */ public function get($file) { $result = $this->getCache()->get($file); - if ($result) { + if ($result instanceof ICacheEntry) { $result = $this->formatCacheEntry($result); } return $result; @@ -216,42 +214,19 @@ class CacheWrapper extends Cache { return $this->getCache()->getStatus($file); } - /** - * search for files matching $pattern - * - * @param string $pattern - * @return ICacheEntry[] an array of file data - */ - public function search($pattern) { - $results = $this->getCache()->search($pattern); - return array_map([$this, 'formatCacheEntry'], $results); - } - - /** - * search for files by mimetype - * - * @param string $mimetype - * @return ICacheEntry[] - */ - public function searchByMime($mimetype) { - $results = $this->getCache()->searchByMime($mimetype); - return array_map([$this, 'formatCacheEntry'], $results); - } - public function searchQuery(ISearchQuery $query) { - $results = $this->getCache()->searchQuery($query); - return array_map([$this, 'formatCacheEntry'], $results); + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($path, $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); } } @@ -259,12 +234,13 @@ class CacheWrapper extends Cache { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($path, $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); } else { return 0; } @@ -286,7 +262,7 @@ class CacheWrapper extends Cache { * use the one with the highest id gives the best result with the background scanner, since that is most * likely the folder where we stopped scanning previously * - * @return string|bool the path of the folder or false when no folder matched + * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { return $this->getCache()->getIncomplete(); @@ -322,4 +298,18 @@ class CacheWrapper extends Cache { public static function getById($id) { return parent::getById($id); } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->getCache()->getQueryFilterForStorage(); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $entry = $this->formatCacheEntry(clone $rawEntry); + return $entry ?: null; + } + + return null; + } } diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php index 956d187783a..d6409b7875e 100644 --- a/lib/private/Files/Cache/Wrapper/JailPropagator.php +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Propagator; @@ -40,7 +22,7 @@ class JailPropagator extends Propagator { */ public function propagateChange($internalPath, $time, $sizeDifference = 0) { /** @var \OC\Files\Storage\Storage $storage */ - list($storage, $sourceInternalPath) = $this->storage->resolvePath($internalPath); + [$storage, $sourceInternalPath] = $this->storage->resolvePath($internalPath); $storage->getPropagator()->propagateChange($sourceInternalPath, $time, $sizeDifference); } } diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php new file mode 100644 index 00000000000..b1ae516654a --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Watcher; + +class JailWatcher extends Watcher { + private string $root; + private Watcher $watcher; + + public function __construct(Watcher $watcher, string $root) { + $this->watcher = $watcher; + $this->root = $root; + } + + protected function getRoot(): string { + return $this->root; + } + + protected function getSourcePath($path): string { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + public function setPolicy($policy) { + $this->watcher->setPolicy($policy); + } + + public function getPolicy() { + return $this->watcher->getPolicy(); + } + + + public function checkUpdate($path, $cachedEntry = null) { + return $this->watcher->checkUpdate($this->getSourcePath($path), $cachedEntry); + } + + public function update($path, $cachedData) { + $this->watcher->update($this->getSourcePath($path), $cachedData); + } + + public function needsUpdate($path, $cachedData) { + return $this->watcher->needsUpdate($this->getSourcePath($path), $cachedData); + } + + public function cleanFolder($path) { + $this->watcher->cleanFolder($this->getSourcePath($path)); + } + + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } +} |