diff options
Diffstat (limited to 'lib/private/Files/Cache')
24 files changed, 1615 insertions, 1065 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 6440bf05a1d..329466e682d 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -1,54 +1,25 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Ari Selseng <ari@selseng.net> - * @author Artem Kochnev <MrJeos@gmail.com> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Florin Peter <github@florin-peter.de> - * @author Frédéric Fortier <frederic.fortier@oronospolytechnique.com> - * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch> - * @author Joas Schilling <coding@schilljs.com> - * @author John Molakvoæ <skjnldsv@protonmail.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Michael Gapczynski <GapczynskiM@gmail.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; +use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchQuery; +use OC\Files\Storage\Wrapper\Encryption; +use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Cache\CacheEntryInsertedEvent; +use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheEntryUpdatedEvent; use OCP\Files\Cache\CacheInsertEvent; -use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheUpdateEvent; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; @@ -58,7 +29,9 @@ use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; +use OCP\Util; use Psr\Log\LoggerInterface; /** @@ -79,61 +52,51 @@ class Cache implements ICache { /** * @var array partial data for the cache */ - protected $partial = []; - - /** - * @var string - */ - protected $storageId; - - private $storage; - - /** - * @var Storage $storageCache - */ - protected $storageCache; - - /** @var IMimeTypeLoader */ - protected $mimetypeLoader; - - /** - * @var IDBConnection - */ - protected $connection; - - /** - * @var IEventDispatcher - */ - protected $eventDispatcher; - - /** @var QuerySearchHelper */ - protected $querySearchHelper; - - /** - * @param IStorage $storage - */ - public function __construct(IStorage $storage) { + protected array $partial = []; + protected string $storageId; + protected Storage $storageCache; + protected IMimeTypeLoader $mimetypeLoader; + protected IDBConnection $connection; + protected SystemConfig $systemConfig; + protected LoggerInterface $logger; + protected QuerySearchHelper $querySearchHelper; + protected IEventDispatcher $eventDispatcher; + protected IFilesMetadataManager $metadataManager; + + public function __construct( + private IStorage $storage, + // this constructor is used in to many pleases to easily do proper di + // so instead we group it all together + ?CacheDependencies $dependencies = null, + ) { $this->storageId = $storage->getId(); - $this->storage = $storage; if (strlen($this->storageId) > 64) { $this->storageId = md5($this->storageId); } - - $this->storageCache = new Storage($storage); - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); - $this->connection = \OC::$server->getDatabaseConnection(); - $this->eventDispatcher = \OC::$server->get(IEventDispatcher::class); - $this->querySearchHelper = \OC::$server->query(QuerySearchHelper::class); + if (!$dependencies) { + $dependencies = \OCP\Server::get(CacheDependencies::class); + } + $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->systemConfig = $dependencies->getSystemConfig(); + $this->logger = $dependencies->getLogger(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + $this->eventDispatcher = $dependencies->getEventDispatcher(); + $this->metadataManager = $dependencies->getMetadataManager(); } protected function getQueryBuilder() { return new CacheQueryBuilder( - $this->connection, - \OC::$server->getSystemConfig(), - \OC::$server->get(LoggerInterface::class) + $this->connection->getQueryBuilder(), + $this->metadataManager, ); } + public function getStorageCache(): Storage { + return $this->storageCache; + } + /** * Get the numeric storage id for this cache's storage * @@ -146,35 +109,39 @@ class Cache implements ICache { /** * get the stored metadata of a file or folder * - * @param string | int $file either the path of a file or folder or the file id for a file or folder - * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache + * @param string|int $file either the path of a file or folder or the file id for a file or folder + * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache */ public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); if (is_string($file) || $file == '') { // normalize file $file = $this->normalize($file); - $query->whereStorageId($this->getNumericStorageId()) - ->wherePath($file); + $query->wherePath($file); } else { //file id $query->whereFileId($file); } + $query->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $data = $result->fetch(); $result->closeCursor(); - //merge partial data - if (!$data && is_string($file) && isset($this->partial[$file])) { - return $this->partial[$file]; - } elseif (!$data) { - return $data; - } else { + if ($data !== false) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } } + + return false; } /** @@ -190,8 +157,8 @@ class Cache implements ICache { $data['path'] = (string)$data['path']; $data['fileid'] = (int)$data['fileid']; $data['parent'] = (int)$data['parent']; - $data['size'] = 0 + $data['size']; - $data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0); + $data['size'] = Util::numericToNumber($data['size']); + $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0); $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; $data['encryptedVersion'] = (int)$data['encrypted']; @@ -203,6 +170,9 @@ class Cache implements ICache { if ($data['storage_mtime'] == 0) { $data['storage_mtime'] = $data['mtime']; } + if (isset($data['f_permissions'])) { + $data['scan_permissions'] = $data['f_permissions']; + } $data['permissions'] = (int)$data['permissions']; if (isset($data['creation_time'])) { $data['creation_time'] = (int)$data['creation_time']; @@ -235,13 +205,17 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->selectFileCache() ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) ->orderBy('name', 'ASC'); - $result = $query->execute(); + $metadataQuery = $query->selectMetadata(); + + $result = $query->executeQuery(); $files = $result->fetchAll(); $result->closeCursor(); - return array_map(function (array $data) { + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); } @@ -280,14 +254,14 @@ class Cache implements ICache { $file = $this->normalize($file); if (isset($this->partial[$file])) { //add any saved partial data - $data = array_merge($this->partial[$file], $data); + $data = array_merge($this->partial[$file]->getData(), $data); unset($this->partial[$file]); } $requiredFields = ['size', 'mtime', 'mimetype']; foreach ($requiredFields as $field) { if (!isset($data[$field])) { //data not complete save as partial and return - $this->partial[$file] = $data; + $this->partial[$file] = new CacheEntry($data); return -1; } } @@ -296,6 +270,9 @@ class Cache implements ICache { if (!isset($data['parent'])) { $data['parent'] = $this->getParentId($file); } + if ($data['parent'] === -1 && $file !== '') { + throw new \Exception('Parent folder not in filecache for ' . $file); + } $data['name'] = basename($file); [$values, $extensionValues] = $this->normalizeData($data); @@ -316,12 +293,13 @@ class Cache implements ICache { if (count($extensionValues)) { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $storageId); $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { $query->setValue($column, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId); @@ -370,6 +348,7 @@ class Cache implements ICache { $query->update('filecache') ->whereFileId($id) + ->whereStorageId($this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -381,13 +360,14 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } if (count($extensionValues)) { try { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $this->getNumericStorageId()); $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { @@ -399,6 +379,7 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->update('filecache_extended') ->whereFileId($id) + ->hintShardKey('storage', $this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -410,7 +391,7 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } } @@ -445,7 +426,7 @@ class Cache implements ICache { $params = []; $extensionParams = []; foreach ($data as $name => $value) { - if (array_search($name, $fields) !== false) { + if (in_array($name, $fields)) { if ($name === 'path') { $params['path_hash'] = md5($value); } elseif ($name === 'mimetype') { @@ -465,7 +446,7 @@ class Cache implements ICache { } $params[$name] = $value; } - if (array_search($name, $extensionFields) !== false) { + if (in_array($name, $extensionFields)) { $extensionParams[$name] = $value; } } @@ -492,7 +473,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $id = $result->fetchOne(); $result->closeCursor(); @@ -545,13 +526,15 @@ class Cache implements ICache { if ($entry instanceof ICacheEntry) { $query = $this->getQueryBuilder(); $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereFileId($entry->getId()); - $query->execute(); + $query->executeStatement(); $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->whereFileId($entry->getId()); - $query->execute(); + ->whereFileId($entry->getId()) + ->hintShardKey('storage', $this->getNumericStorageId()); + $query->executeStatement(); if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) { $this->removeChildren($entry); @@ -562,20 +545,7 @@ class Cache implements ICache { } /** - * Get all sub folders of a folder - * - * @param ICacheEntry $entry the cache entry of the folder to get the subfolders for - * @return ICacheEntry[] the cache entries for the subfolders - */ - private function getSubFolders(ICacheEntry $entry) { - $children = $this->getFolderContentsById($entry->getId()); - return array_filter($children, function ($child) { - return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER; - }); - } - - /** - * Recursively remove all children of a folder + * Remove all children of a folder * * @param ICacheEntry $entry the cache entry of the folder to remove the children of * @throws \OC\DatabaseException @@ -583,6 +553,8 @@ class Cache implements ICache { private function removeChildren(ICacheEntry $entry) { $parentIds = [$entry->getId()]; $queue = [$entry->getId()]; + $deletedIds = []; + $deletedPaths = []; // we walk depth first through the file tree, removing all filecache_extended attributes while we walk // and collecting all folder ids to later use to delete the filecache entries @@ -591,20 +563,35 @@ class Cache implements ICache { $childIds = array_map(function (ICacheEntry $cacheEntry) { return $cacheEntry->getId(); }, $children); + $childPaths = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getPath(); + }, $children); + + foreach ($childIds as $childId) { + $deletedIds[] = $childId; + } + + foreach ($childPaths as $childPath) { + $deletedPaths[] = $childPath; + } $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->where($query->expr()->in('fileid', $query->createParameter('childIds'))); + ->where($query->expr()->in('fileid', $query->createParameter('childIds'))) + ->hintShardKey('storage', $this->getNumericStorageId()); foreach (array_chunk($childIds, 1000) as $childIdChunk) { $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY); - $query->execute(); + $query->executeStatement(); } /** @var ICacheEntry[] $childFolders */ - $childFolders = array_filter($children, function ($child) { - return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER; - }); + $childFolders = []; + foreach ($children as $child) { + if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $childFolders[] = $child; + } + } foreach ($childFolders as $folder) { $parentIds[] = $folder->getId(); $queue[] = $folder->getId(); @@ -613,11 +600,25 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereParentInParameter('parentIds'); + // Sorting before chunking allows the db to find the entries close to each + // other in the index + sort($parentIds, SORT_NUMERIC); foreach (array_chunk($parentIds, 1000) as $parentIdChunk) { $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY); - $query->execute(); + $query->executeStatement(); + } + + foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { + $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( + $this->storage, + $filePath, + $fileId, + $this->getNumericStorageId() + ); + $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); } } @@ -641,6 +642,10 @@ class Cache implements ICache { return [$this->getNumericStorageId(), $path]; } + protected function hasEncryptionWrapper(): bool { + return $this->storage->instanceOfStorage(Encryption::class); + } + /** * Move a file or folder in the cache * @@ -657,8 +662,17 @@ class Cache implements ICache { $targetPath = $this->normalize($targetPath); $sourceData = $sourceCache->get($sourcePath); - if ($sourceData === false) { - throw new \Exception('Invalid source storage path: ' . $sourcePath); + if (!$sourceData) { + throw new \Exception('Source path not found in cache: ' . $sourcePath); + } + + $shardDefinition = $this->connection->getShardDefinition('filecache'); + if ( + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + ) { + $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); + return; } $sourceId = $sourceData['fileid']; @@ -674,11 +688,15 @@ class Cache implements ICache { throw new \Exception('Invalid target storage id: ' . $targetStorageId); } - $this->connection->beginTransaction(); if ($sourceData['mimetype'] === 'httpd/unix-directory') { //update all child entries $sourceLength = mb_strlen($sourcePath); - $query = $this->connection->getQueryBuilder(); + + $childIds = $this->getChildIds($sourceStorageId, $sourcePath); + + $childChunks = array_chunk($childIds, 1000); + + $query = $this->getQueryBuilder(); $fun = $query->func(); $newPathFunction = $fun->concat( @@ -686,29 +704,73 @@ class Cache implements ICache { $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash ); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT)) ->set('path_hash', $fun->md5($newPathFunction)) ->set('path', $newPathFunction) - ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%'))); - - try { - $query->execute(); - } catch (\OC\DatabaseException $e) { - $this->connection->rollBack(); - throw $e; + ->whereStorageId($sourceStorageId) + ->andWhere($query->expr()->in('fileid', $query->createParameter('files'))); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); } + + // Retry transaction in case of RetryableException like deadlocks. + // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend + $retryLimit = 4; + for ($i = 1; $i <= $retryLimit; $i++) { + try { + $this->connection->beginTransaction(); + foreach ($childChunks as $chunk) { + $query->setParameter('files', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + break; + } catch (\OC\DatabaseException $e) { + $this->connection->rollBack(); + throw $e; + } catch (DbalException $e) { + $this->connection->rollBack(); + + if (!$e->isRetryable()) { + throw $e; + } + + // Simply throw if we already retried 4 times. + if ($i === $retryLimit) { + throw $e; + } + + // Sleep a bit to give some time to the other transaction to finish. + usleep(100 * 1000 * $i); + } + } + } else { + $this->connection->beginTransaction(); } $query = $this->getQueryBuilder(); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId)) ->set('path', $query->createNamedParameter($targetPath)) ->set('path_hash', $query->createNamedParameter(md5($targetPath))) ->set('name', $query->createNamedParameter(basename($targetPath))) ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT)) + ->whereStorageId($sourceStorageId) ->whereFileId($sourceId); - $query->execute(); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + $query->executeStatement(); $this->connection->commit(); @@ -727,6 +789,15 @@ class Cache implements ICache { } } + private function getChildIds(int $storageId, string $path): array { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($path) . '/%'))); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } + /** * remove all entries for files that are stored on the storage from the cache */ @@ -734,12 +805,12 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache') ->whereStorageId($this->getNumericStorageId()); - $query->execute(); + $query->executeStatement(); $query = $this->connection->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); - $query->execute(); + $query->executeStatement(); } /** @@ -764,7 +835,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $size = $result->fetchOne(); $result->closeCursor(); @@ -798,11 +869,11 @@ class Cache implements ICache { * search for files by mimetype * * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') - * where it will search for all mimetypes in the group ('image/*') + * where it will search for all mimetypes in the group ('image/*') * @return ICacheEntry[] an array of cache entries where the mimetype matches the search */ public function searchByMime($mimetype) { - if (strpos($mimetype, '/') === false) { + if (!str_contains($mimetype, '/')) { $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'); } else { $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype); @@ -810,26 +881,30 @@ class Cache implements ICache { return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** * Re-calculate the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { $this->calculateFolderSize($path, $data); + if ($path !== '') { $parent = dirname($path); if ($parent === '.' || $parent === '/') { $parent = ''; } + if ($isBackgroundScan) { $parentData = $this->get($parent); - if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) { + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { $this->correctFolderSize($parent, $parentData, $isBackgroundScan); } } else { @@ -850,9 +925,10 @@ class Cache implements ICache { $query->select($query->func()->count()) ->from('filecache') ->whereParent($fileId) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $size = (int)$result->fetchOne(); $result->closeCursor(); @@ -865,10 +941,23 @@ class Cache implements ICache { * calculate the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { + return $this->calculateFolderSizeInner($path, $entry); + } + + + /** + * inner function because we can't add new params to the public function without breaking any child classes + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown + * @return int|float + */ + protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) { $totalSize = 0; if (is_null($entry) || !isset($entry['fileid'])) { $entry = $this->get($path); @@ -879,21 +968,25 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('size', 'unencrypted_size') ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereParent($id); + if ($ignoreUnknown) { + $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); + } - $result = $query->execute(); + $result = $query->executeQuery(); $rows = $result->fetchAll(); $result->closeCursor(); if ($rows) { $sizes = array_map(function (array $row) { - return (int)$row['size']; + return Util::numericToNumber($row['size']); }, $rows); $unencryptedOnlySizes = array_map(function (array $row) { - return (int)$row['unencrypted_size']; + return Util::numericToNumber($row['unencrypted_size']); }, $rows); $unencryptedSizes = array_map(function (array $row) { - return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']); + return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']); }, $rows); $sum = array_sum($sizes); @@ -920,9 +1013,16 @@ class Cache implements ICache { $unencryptedTotal = 0; $unencryptedMax = 0; } - if ($entry['size'] !== $totalSize) { - // only set unencrypted size for a folder if any child entries have it set, or the folder is empty - if ($unencryptedMax > 0 || $totalSize === 0) { + + // only set unencrypted size for a folder if any child entries have it set, or the folder is empty + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + if ($shouldWriteUnEncryptedSize) { + // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes + if ($unencryptedMax === 0) { + $unencryptedTotal = 0; + } + $this->update($id, [ 'size' => $totalSize, 'unencrypted_size' => $unencryptedTotal, @@ -948,7 +1048,7 @@ class Cache implements ICache { ->from('filecache') ->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $files = $result->fetchAll(\PDO::FETCH_COLUMN); $result->closeCursor(); @@ -971,20 +1071,15 @@ class Cache implements ICache { $query->select('path') ->from('filecache') ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('fileid', 'DESC') ->setMaxResults(1); - $result = $query->execute(); + $result = $query->executeQuery(); $path = $result->fetchOne(); $result->closeCursor(); - if ($path === false) { - return false; - } - - // Make sure Oracle does not continue with null for empty strings - return (string)$path; + return $path === false ? false : (string)$path; } /** @@ -1000,7 +1095,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->whereFileId($id); - $result = $query->execute(); + $result = $query->executeQuery(); $path = $result->fetchOne(); $result->closeCursor(); @@ -1018,7 +1113,7 @@ class Cache implements ICache { * * @param int $id * @return array first element holding the storage id, second the path - * @deprecated use getPathById() instead + * @deprecated 17.0.0 use getPathById() instead */ public static function getById($id) { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); @@ -1026,7 +1121,7 @@ class Cache implements ICache { ->from('filecache') ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); @@ -1064,12 +1159,18 @@ class Cache implements ICache { */ public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { if ($sourceEntry->getId() < 0) { - throw new \RuntimeException("Invalid source cache entry on copyFromCache"); + throw new \RuntimeException('Invalid source cache entry on copyFromCache'); } $data = $this->cacheEntryToArray($sourceEntry); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $data['encrypted'] = 0; + } + $fileId = $this->put($targetPath, $data); if ($fileId <= 0) { - throw new \RuntimeException("Failed to copy to " . $targetPath . " from cache with source data " . json_encode($data) . " "); + throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' '); } if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); @@ -1082,7 +1183,7 @@ class Cache implements ICache { } private function cacheEntryToArray(ICacheEntry $entry): array { - return [ + $data = [ 'size' => $entry->getSize(), 'mtime' => $entry->getMTime(), 'storage_mtime' => $entry->getStorageMTime(), @@ -1095,6 +1196,10 @@ class Cache implements ICache { 'upload_time' => $entry->getUploadTime(), 'metadata_etag' => $entry->getMetadataEtag(), ]; + if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) { + $data['permissions'] = $entry['scan_permissions']; + } + return $data; } public function getQueryFilterForStorage(): ISearchOperator { @@ -1108,4 +1213,72 @@ class Cache implements ICache { return null; } } + + private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void { + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath()); + } else { + $fileIds = []; + } + $fileIds[] = $sourceEntry->getId(); + + $helper = $this->connection->getCrossShardMoveHelper(); + + $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId()); + $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId()); + + $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper(); + + $sourcePathLength = strlen($sourceEntry->getPath()); + foreach ($cacheItems as &$cacheItem) { + if ($cacheItem['path'] === $sourceEntry->getPath()) { + $cacheItem['path'] = $targetPath; + $cacheItem['parent'] = $this->getParentId($targetPath); + $cacheItem['name'] = basename($cacheItem['path']); + } else { + $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash + } + $cacheItem['path_hash'] = md5($cacheItem['path']); + $cacheItem['storage'] = $this->getNumericStorageId(); + if ($removeEncryptedFlag) { + $cacheItem['encrypted'] = 0; + } + } + + $targetConnection->beginTransaction(); + + try { + $helper->saveItems($targetConnection, 'filecache', $cacheItems); + $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems); + $helper->saveItems($targetConnection, 'files_metadata', $metadataItems); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + + $sourceConnection->beginTransaction(); + + try { + $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + } catch (\Exception $e) { + $targetConnection->rollback(); + $sourceConnection->rollBack(); + throw $e; + } + + try { + $sourceConnection->commit(); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + $targetConnection->commit(); + } } diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php new file mode 100644 index 00000000000..61d2a2f9646 --- /dev/null +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -0,0 +1,61 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDependencies { + public function __construct( + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $metadataManager, + private DisplayNameCache $displayNameCache, + ) { + } + + public function getMimeTypeLoader(): IMimeTypeLoader { + return $this->mimeTypeLoader; + } + + public function getConnection(): IDBConnection { + return $this->connection; + } + + public function getEventDispatcher(): IEventDispatcher { + return $this->eventDispatcher; + } + + public function getQuerySearchHelper(): QuerySearchHelper { + return $this->querySearchHelper; + } + + public function getSystemConfig(): SystemConfig { + return $this->systemConfig; + } + + public function getLogger(): LoggerInterface { + return $this->logger; + } + + public function getDisplayNameCache(): DisplayNameCache { + return $this->displayNameCache; + } + + public function getMetadataManager(): IFilesMetadataManager { + return $this->metadataManager; + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index 3c93296ff62..ab5bae316f4 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -1,24 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; @@ -114,15 +99,19 @@ class CacheEntry implements ICacheEntry { } public function getMetadataEtag(): ?string { - return $this->data['metadata_etag']; + return $this->data['metadata_etag'] ?? null; } public function getCreationTime(): ?int { - return $this->data['creation_time']; + return $this->data['creation_time'] ?? null; } public function getUploadTime(): ?int { - return $this->data['upload_time']; + return $this->data['upload_time'] ?? null; + } + + public function getParentId(): int { + return $this->data['parent']; } public function getData() { @@ -134,7 +123,7 @@ class CacheEntry implements ICacheEntry { } public function getUnencryptedSize(): int { - if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + if ($this->data['encrypted'] && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { return $this->data['unencrypted_size']; } else { return $this->data['size'] ?? 0; diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 496a8361d77..5492452273b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -3,48 +3,52 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2019 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; -use OC\DB\QueryBuilder\QueryBuilder; -use OC\SystemConfig; +use OC\DB\QueryBuilder\ExtendedQueryBuilder; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\IDBConnection; -use Psr\Log\LoggerInterface; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; /** * Query builder with commonly used helpers for filecache queries */ -class CacheQueryBuilder extends QueryBuilder { - private $alias = null; +class CacheQueryBuilder extends ExtendedQueryBuilder { + private ?string $alias = null; + + public function __construct( + IQueryBuilder $queryBuilder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($queryBuilder); + } + + public function selectTagUsage(): self { + $this + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') + ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') + ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') + ->from('filecache', 'filecache') + ->leftJoin('filecache', 'systemtag_object_mapping', 'systemtagmap', $this->expr()->andX( + $this->expr()->eq('filecache.fileid', $this->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $this->expr()->eq('systemtagmap.objecttype', $this->createNamedParameter('files')) + )) + ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $this->expr()->andX( + $this->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'), + $this->expr()->eq('systemtag.visibility', $this->createNamedParameter(true)) + )) + ->groupBy('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable'); - public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) { - parent::__construct($connection, $systemConfig, $logger); + return $this; } - public function selectFileCache(string $alias = null, bool $joinExtendedCache = true) { - $name = $alias ? $alias : 'filecache'; + public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true) { + $name = $alias ?: 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', - 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'unencrypted_size') + 'storage_mtime', 'encrypted', "$name.etag", "$name.permissions", 'checksum', 'unencrypted_size') ->from('filecache', $name); if ($joinExtendedCache) { @@ -107,4 +111,15 @@ class CacheQueryBuilder extends QueryBuilder { return $this; } + + /** + * join metadata to current query builder and returns an helper + * + * @return IMetadataQuery + */ + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } } diff --git a/lib/private/Files/Cache/FailedCache.php b/lib/private/Files/Cache/FailedCache.php index d60e09ea329..44c1016ca8e 100644 --- a/lib/private/Files/Cache/FailedCache.php +++ b/lib/private/Files/Cache/FailedCache.php @@ -1,23 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; @@ -139,7 +125,7 @@ class FailedCache implements ICache { } public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { - throw new \Exception("Invalid cache"); + throw new \Exception('Invalid cache'); } public function getQueryFilterForStorage(): ISearchOperator { diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php new file mode 100644 index 00000000000..c3f3614f3ca --- /dev/null +++ b/lib/private/Files/Cache/FileAccess.php @@ -0,0 +1,222 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\SystemConfig; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IFileAccess; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Low level access to the file cache + */ +class FileAccess implements IFileAccess { + public function __construct( + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private FilesMetadataManager $metadataManager, + private IMimeTypeLoader $mimeTypeLoader, + ) { + } + + private function getQuery(): CacheQueryBuilder { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getByFileIdInStorage(int $fileId, int $storageId): ?CacheEntry { + $items = array_values($this->getByFileIdsInStorage([$fileId], $storageId)); + return $items[0] ?? null; + } + + public function getByPathInStorage(string $path, int $storageId): ?CacheEntry { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->eq('filecache.path_hash', $query->createNamedParameter(md5($path)))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $row = $query->executeQuery()->fetch(); + return $row ? Cache::cacheEntryFromData($row, $this->mimeTypeLoader) : null; + } + + public function getByFileId(int $fileId): ?CacheEntry { + $items = array_values($this->getByFileIds([$fileId])); + return $items[0] ?? null; + } + + /** + * @param array[] $rows + * @return array<int, CacheEntry> + */ + private function rowsToEntries(array $rows): array { + $result = []; + foreach ($rows as $row) { + $entry = Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + $result[$entry->getId()] = $entry; + } + return $result; + } + + /** + * @param int[] $fileIds + * @return array<int, CacheEntry> + */ + public function getByFileIds(array $fileIds): array { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + /** + * @param int[] $fileIds + * @param int $storageId + * @return array<int, CacheEntry> + */ + public function getByFileIdsInStorage(array $fileIds, int $storageId): array { + $fileIds = array_values($fileIds); + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } +} diff --git a/lib/private/Files/Cache/HomeCache.php b/lib/private/Files/Cache/HomeCache.php index 6b6b94c3f20..248cdc818f0 100644 --- a/lib/private/Files/Cache/HomeCache.php +++ b/lib/private/Files/Cache/HomeCache.php @@ -1,30 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; @@ -35,8 +14,8 @@ class HomeCache extends Cache { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { if ($path !== '/' and $path !== '' and $path !== 'files' and $path !== 'files_trashbin' and $path !== 'files_versions') { @@ -44,46 +23,18 @@ class HomeCache extends Cache { } elseif ($path === '' or $path === '/') { // since the size of / isn't used (the size of /files is used instead) there is no use in calculating it return 0; + } else { + return $this->calculateFolderSizeInner($path, $entry, true); } - - $totalSize = 0; - if (is_null($entry)) { - $entry = $this->get($path); - } - if ($entry && $entry['mimetype'] === 'httpd/unix-directory') { - $id = $entry['fileid']; - - $query = $this->connection->getQueryBuilder(); - $query->selectAlias($query->func()->sum('size'), 'f1') - ->from('filecache') - ->where($query->expr()->eq('parent', $query->createNamedParameter($id))) - ->andWhere($query->expr()->eq('storage', $query->createNamedParameter($this->getNumericStorageId()))) - ->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); - - $result = $query->execute(); - $row = $result->fetch(); - $result->closeCursor(); - - if ($row) { - [$sum] = array_values($row); - $totalSize = 0 + $sum; - $entry['size'] += 0; - if ($entry['size'] !== $totalSize) { - $this->update($id, ['size' => $totalSize]); - } - } - $result->closeCursor(); - } - return $totalSize; } /** - * @param string $path + * @param string $file * @return ICacheEntry */ - public function get($path) { - $data = parent::get($path); - if ($path === '' or $path === '/') { + public function get($file) { + $data = parent::get($file); + if ($file === '' or $file === '/') { // only the size of the "files" dir counts $filesData = parent::get('files'); diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php index 6dba09d756b..d4ac8a7c8e3 100644 --- a/lib/private/Files/Cache/HomePropagator.php +++ b/lib/private/Files/Cache/HomePropagator.php @@ -1,23 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 0b6bc497ea3..3f4f70b865b 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -3,25 +3,8 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; @@ -44,6 +27,6 @@ class LocalRootScanner extends Scanner { private function shouldScanPath(string $path): bool { $path = trim($path, '/'); - return $path === '' || strpos($path, 'appdata_') === 0 || strpos($path, '__groupfolders') === 0; + return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Files/Cache/MoveFromCacheTrait.php b/lib/private/Files/Cache/MoveFromCacheTrait.php index 77cd7ea6d8c..db35c6bb7f8 100644 --- a/lib/private/Files/Cache/MoveFromCacheTrait.php +++ b/lib/private/Files/Cache/MoveFromCacheTrait.php @@ -1,23 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; diff --git a/lib/private/Files/Cache/NullWatcher.php b/lib/private/Files/Cache/NullWatcher.php index 2e83c1006bb..e3659214849 100644 --- a/lib/private/Files/Cache/NullWatcher.php +++ b/lib/private/Files/Cache/NullWatcher.php @@ -3,25 +3,8 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index 4bf88a60843..a6ba87896f4 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -1,35 +1,20 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\Cache; -use Doctrine\DBAL\Exception\RetryableException; +use OC\DB\Exceptions\DbalException; use OC\Files\Storage\Wrapper\Encryption; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IPropagator; use OCP\Files\Storage\IReliableEtagStorage; use OCP\IDBConnection; +use OCP\Server; +use Psr\Clock\ClockInterface; use Psr\Log\LoggerInterface; /** @@ -56,13 +41,15 @@ class Propagator implements IPropagator { */ private $ignore = []; + private ClockInterface $clock; + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { $this->storage = $storage; $this->connection = $connection; $this->ignore = $ignore; + $this->clock = Server::get(ClockInterface::class); } - /** * @param string $internalPath * @param int $time @@ -71,12 +58,14 @@ class Propagator implements IPropagator { public function propagateChange($internalPath, $time, $sizeDifference = 0) { // Do not propagate changes in ignored paths foreach ($this->ignore as $ignore) { - if (strpos($internalPath, $ignore) === 0) { + if (str_starts_with($internalPath, $ignore)) { return; } } - $storageId = (int)$this->storage->getStorageCache()->getNumericId(); + $time = min((int)$time, $this->clock->now()->getTimestamp()); + + $storageId = $this->storage->getStorageCache()->getNumericId(); $parents = $this->getParents($internalPath); @@ -96,7 +85,7 @@ class Propagator implements IPropagator { }, $parentHashes); $builder->update('filecache') - ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT))) + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT))) ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($builder->expr()->in('path_hash', $hashParams)); if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { @@ -136,7 +125,11 @@ class Propagator implements IPropagator { try { $builder->executeStatement(); break; - } catch (RetryableException $e) { + } catch (DbalException $e) { + if (!$e->isRetryable()) { + throw $e; + } + /** @var LoggerInterface $loggerInterface */ $loggerInterface = \OCP\Server::get(LoggerInterface::class); $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); @@ -199,27 +192,27 @@ class Propagator implements IPropagator { $query->update('filecache') ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) ->set('etag', $query->expr()->literal(uniqid())) - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); $sizeQuery = $this->connection->getQueryBuilder(); $sizeQuery->update('filecache') ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) - ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) - ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + ->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash'))) + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); foreach ($this->batch as $item) { $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); $query->setParameter('hash', $item['hash']); - $query->execute(); + $query->executeStatement(); if ($item['size']) { $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); $sizeQuery->setParameter('hash', $item['hash']); - $sizeQuery->execute(); + $sizeQuery->executeStatement(); } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index eba2aac927b..3ddcf1ca4e6 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -1,30 +1,12 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tobias Kaminsky <tobias@kaminsky.me> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; +use OC\Files\Cache\Wrapper\CacheJail; use OC\Files\Search\QueryOptimizer\QueryOptimizer; use OC\Files\Search\SearchBinaryOperator; use OC\SystemConfig; @@ -32,49 +14,115 @@ use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\IMimeTypeLoader; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; use Psr\Log\LoggerInterface; class QuerySearchHelper { - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - /** @var IDBConnection */ - private $connection; - /** @var SystemConfig */ - private $systemConfig; - private LoggerInterface $logger; - /** @var SearchBuilder */ - private $searchBuilder; - /** @var QueryOptimizer */ - private $queryOptimizer; - public function __construct( - IMimeTypeLoader $mimetypeLoader, - IDBConnection $connection, - SystemConfig $systemConfig, - LoggerInterface $logger, - SearchBuilder $searchBuilder, - QueryOptimizer $queryOptimizer + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; - $this->connection = $connection; - $this->systemConfig = $systemConfig; - $this->logger = $logger; - $this->searchBuilder = $searchBuilder; - $this->queryOptimizer = $queryOptimizer; } protected function getQueryBuilder() { return new CacheQueryBuilder( - $this->connection, - $this->systemConfig, - $this->logger + $this->connection->getQueryBuilder(), + $this->filesMetadataManager, ); } /** + * @param CacheQueryBuilder $query + * @param ISearchQuery $searchQuery + * @param array $caches + * @param IMetadataQuery|null $metadataQuery + */ + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null, + ): void { + $storageFilters = array_values(array_map(function (ICache $cache) { + return $cache->getQueryFilterForStorage(); + }, $caches)); + $storageFilter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters); + $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); + $this->queryOptimizer->processOperator($filter); + + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); + if ($searchExpr) { + $query->andWhere($searchExpr); + } + + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); + + if ($searchQuery->getLimit()) { + $query->setMaxResults($searchQuery->getLimit()); + } + if ($searchQuery->getOffset()) { + $query->setFirstResult($searchQuery->getOffset()); + } + } + + + /** + * @return array<array-key, array{id: int, name: string, visibility: int, editable: int, ref_file_id: int, number_files: int}> + */ + public function findUsedTagsInCaches(ISearchQuery $searchQuery, array $caches): array { + $query = $this->getQueryBuilder(); + $query->selectTagUsage(); + + $this->applySearchConstraints($query, $searchQuery, $caches); + + $result = $query->executeQuery(); + $tags = $result->fetchAll(); + $result->closeCursor(); + return $tags; + } + + protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user): void { + $query->leftJoin('file', 'systemtag_object_mapping', 'systemtagmap', $query->expr()->andX( + $query->expr()->eq('file.fileid', $query->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')) + )); + $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid')); + if (!$this->groupManager->isAdmin($user->getUID())) { + $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true))); + } + $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on); + } + + protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void { + $query + ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid')) + ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( + $query->expr()->eq('tagmap.categoryid', 'tag.id'), + $query->expr()->eq('tag.type', $query->createNamedParameter('files')), + $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID())) + )); + } + + + protected function equipQueryForShares(CacheQueryBuilder $query): void { + $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); + } + + /** * Perform a file system search in multiple caches * * the results will be grouped by the same array keys as the $caches argument to allow @@ -104,61 +152,34 @@ class QuerySearchHelper { $query = $builder->selectFileCache('file', false); - if ($this->searchBuilder->shouldJoinTags($searchQuery->getSearchOperation())) { - $user = $searchQuery->getUser(); - if ($user === null) { - throw new \InvalidArgumentException("Searching by tag requires the user to be set in the query"); - } - $query - ->leftJoin('file', 'vcategory_to_object', 'tagmap', $builder->expr()->eq('file.fileid', 'tagmap.objid')) - ->leftJoin('tagmap', 'vcategory', 'tag', $builder->expr()->andX( - $builder->expr()->eq('tagmap.type', 'tag.type'), - $builder->expr()->eq('tagmap.categoryid', 'tag.id'), - $builder->expr()->eq('tag.type', $builder->createNamedParameter('files')), - $builder->expr()->eq('tag.uid', $builder->createNamedParameter($user->getUID())) - )) - ->leftJoin('file', 'systemtag_object_mapping', 'systemtagmap', $builder->expr()->andX( - $builder->expr()->eq('file.fileid', $builder->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), - $builder->expr()->eq('systemtagmap.objecttype', $builder->createNamedParameter('files')) - )) - ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $builder->expr()->andX( - $builder->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'), - $builder->expr()->eq('systemtag.visibility', $builder->createNamedParameter(true)) - )); - } - - $storageFilters = array_values(array_map(function (ICache $cache) { - return $cache->getQueryFilterForStorage(); - }, $caches)); - $storageFilter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters); - $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); - $this->queryOptimizer->processOperator($filter); + $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); - $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($builder, $filter); - if ($searchExpr) { - $query->andWhere($searchExpr); + if (in_array('systemtag', $requestedFields)) { + $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } - - $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder()); - - if ($searchQuery->getLimit()) { - $query->setMaxResults($searchQuery->getLimit()); + if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { + $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } - if ($searchQuery->getOffset()) { - $query->setFirstResult($searchQuery->getOffset()); + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); } - $result = $query->execute(); + $metadataQuery = $query->selectMetadata(); + + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); + + $result = $query->executeQuery(); $files = $result->fetchAll(); - $rawEntries = array_map(function (array $data) { + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return Cache::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); $result->closeCursor(); // loop through all caches for each result to see if the result matches that storage - // results are grouped by the same array keys as the caches argument to allow the caller to distringuish the source of the results + // results are grouped by the same array keys as the caches argument to allow the caller to distinguish the source of the results $results = array_fill_keys(array_keys($caches), []); foreach ($rawEntries as $rawEntry) { foreach ($caches as $cacheKey => $cache) { @@ -170,4 +191,50 @@ class QuerySearchHelper { } return $results; } + + protected function requireUser(ISearchQuery $searchQuery): IUser { + $user = $searchQuery->getUser(); + if ($user === null) { + throw new \InvalidArgumentException('This search operation requires the user to be set in the query'); + } + return $user; + } + + /** + * @return list{0?: array<array-key, ICache>, 1?: array<array-key, IMountPoint>} + */ + public function getCachesAndMountPointsForSearch(IRootFolder $root, string $path, bool $limitToHome = false): array { + $rootLength = strlen($path); + $mount = $root->getMount($path); + $storage = $mount->getStorage(); + if ($storage === null) { + return []; + } + $internalPath = $mount->getInternalPath($path); + + if ($internalPath !== '') { + // a temporary CacheJail is used to handle filtering down the results to within this folder + /** @var ICache[] $caches */ + $caches = ['' => new CacheJail($storage->getCache(''), $internalPath)]; + } else { + /** @var ICache[] $caches */ + $caches = ['' => $storage->getCache('')]; + } + /** @var IMountPoint[] $mountByMountPoint */ + $mountByMountPoint = ['' => $mount]; + + if (!$limitToHome) { + $mounts = $root->getMountsIn($path); + foreach ($mounts as $mount) { + $storage = $mount->getStorage(); + if ($storage) { + $relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/'); + $caches[$relativeMountPoint] = $storage->getCache(''); + $mountByMountPoint[$relativeMountPoint] = $mount; + } + } + } + + return [$caches, $mountByMountPoint]; + } } diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index f7d1d105d83..b067f70b8cb 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -1,48 +1,24 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Ari Selseng <ari@selseng.net> - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Martin Mattel <martin.mattel@diemattels.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Owen Winkler <a_github@midnightcircus.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; use Doctrine\DBAL\Exception; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OC\Hooks\BasicEmitter; +use OC\SystemConfig; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\ILockingStorage; use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; use OCP\Lock\ILockingProvider; -use OC\Files\Storage\Wrapper\Encoding; -use OC\Files\Storage\Wrapper\Jail; -use OC\Hooks\BasicEmitter; use Psr\Log\LoggerInterface; /** @@ -87,12 +63,18 @@ class Scanner extends BasicEmitter implements IScanner { */ protected $lockingProvider; + protected IDBConnection $connection; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); - $this->cacheActive = !\OC::$server->getConfig()->getSystemValue('filesystem_cache_readonly', false); - $this->lockingProvider = \OC::$server->getLockingProvider(); + /** @var SystemConfig $config */ + $config = \OC::$server->get(SystemConfig::class); + $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); + $this->lockingProvider = \OC::$server->get(ILockingProvider::class); + $this->connection = \OC::$server->get(IDBConnection::class); } /** @@ -101,7 +83,7 @@ class Scanner extends BasicEmitter implements IScanner { * * @param bool $useTransactions */ - public function setUseTransactions($useTransactions) { + public function setUseTransactions($useTransactions): void { $this->useTransactions = $useTransactions; } @@ -126,9 +108,9 @@ class Scanner extends BasicEmitter implements IScanner { * @param string $file * @param int $reuseExisting * @param int $parentId - * @param array|null|false $cacheData existing data in the cache for the file to be scanned + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param null $data the metadata for the file, as returned by the storage + * @param array|null $data the metadata for the file, as returned by the storage * @return array|null an array of metadata of the scanned file * @throws \OCP\Lock\LockedException */ @@ -140,123 +122,130 @@ class Scanner extends BasicEmitter implements IScanner { return null; } } + // only proceed if $file is not a partial file, blacklist is handled by the storage - if (!self::isPartialFile($file)) { - // acquire a lock + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } } - try { - $data = $data ?? $this->getData($file); - } catch (ForbiddenException $e) { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + return null; + } + + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); } - return null; - } + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } - try { - if ($data) { - // pre-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; } - $parent = dirname($file); - if ($parent === '.' || $parent === '/') { - $parent = ''; - } - if ($parentId === -1) { - $parentId = $this->cache->getParentId($file); - } + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } - // scan the parent if it's not in the cache (id -1) and the current file is not the root folder - if ($file && $parentId === -1) { - $parentData = $this->scanFile($parent); - if (!$parentData) { - return null; + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; } - $parentId = $parentData['fileid']; - } - if ($parent) { - $data['parent'] = $parentId; - } - if (is_null($cacheData)) { - /** @var CacheEntry $cacheData */ - $cacheData = $this->cache->get($file); - } - if ($cacheData && $reuseExisting && isset($cacheData['fileid'])) { - // prevent empty etag - $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; - $fileId = $cacheData['fileid']; - $data['fileid'] = $fileId; - // only reuse data if the file hasn't explicitly changed - if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { - $data['mtime'] = $cacheData['mtime']; - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { - $data['size'] = $cacheData['size']; - } - if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { - $data['etag'] = $etag; - } + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; } - // Only update metadata that has changed - $newData = array_diff_assoc($data, $cacheData->getData()); - } else { - $newData = $data; - $fileId = -1; - } - if (!empty($newData)) { - // Reset the checksum if the data has changed - $newData['checksum'] = ''; - $newData['parent'] = $parentId; - $data['fileid'] = $this->addToCache($file, $newData, $fileId); } - - $data['oldSize'] = ($cacheData && isset($cacheData['size'])) ? $cacheData['size'] : 0; - if ($cacheData && isset($cacheData['encrypted'])) { - $data['encrypted'] = $cacheData['encrypted']; + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); } - // post-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; } } else { - $this->removeFromCache($file); + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; } - } catch (\Exception $e) { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); } - throw $e; - } - // release the acquired lock - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; } - } - if ($data && !isset($data['encrypted'])) { - $data['encrypted'] = false; + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } - return $data; } - return null; + return $data; } protected function removeFromCache($path) { @@ -278,7 +267,7 @@ class Scanner extends BasicEmitter implements IScanner { $data['permissions'] = $data['scan_permissions']; } \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); - $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data]); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data, $fileId]); if ($this->cacheActive) { if ($fileId !== -1) { $this->cache->update($fileId, $data); @@ -321,36 +310,82 @@ class Scanner extends BasicEmitter implements IScanner { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } + try { - $data = $this->scanFile($path, $reuse, -1, null, $lock); - if ($data && $data['mimetype'] === 'httpd/unix-directory') { - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data); + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; } finally { - if ($lock) { - if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { - $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - } + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); } } return $data; } /** + * Compares $array1 against $array2 and returns all the values in $array1 that are not in $array2 + * Note this is a one-way check - i.e. we don't care about things that are in $array2 that aren't in $array1 + * + * Supports multi-dimensional arrays + * Also checks keys/indexes + * Comparisons are strict just like array_diff_assoc + * Order of keys/values does not matter + * + * @param array $array1 + * @param array $array2 + * @return array with the differences between $array1 and $array1 + * @throws \InvalidArgumentException if $array1 isn't an actual array + * + */ + protected function array_diff_assoc_multi(array $array1, array $array2) { + + $result = []; + + foreach ($array1 as $key => $value) { + + // if $array2 doesn't have the same key, that's a result + if (!array_key_exists($key, $array2)) { + $result[$key] = $value; + continue; + } + + // if $array2's value for the same key is different, that's a result + if ($array2[$key] !== $value && !is_array($value)) { + $result[$key] = $value; + continue; + } + + if (is_array($value)) { + $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); + if (!empty($nestedDiff)) { + $result[$key] = $nestedDiff; + continue; + } + } + } + return $result; + } + + /** * Get the children currently in the cache * * @param int $folderId - * @return array[] + * @return array<string, \OCP\Files\Cache\ICacheEntry> */ - protected function getExistingChildren($folderId) { + protected function getExistingChildren($folderId): array { $existingChildren = []; $children = $this->cache->getFolderContentsById($folderId); foreach ($children as $child) { @@ -363,41 +398,60 @@ class Scanner extends BasicEmitter implements IScanner { * scan all the files and folders in a folder * * @param string $path - * @param bool $recursive - * @param int $reuse + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + * @param int $reuse a combination of self::REUSE_* * @param int $folderId id for the folder to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param array $data the data of the folder before (re)scanning the children - * @return int the size of the scanned folder or -1 if the size is unknown at this stage + * @param int|float $oldSize the size of the folder before (re)scanning the children + * @return int|float the size of the scanned folder or -1 if the size is unknown at this stage */ - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) { + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]); $size = 0; - if (!is_null($folderId)) { - $folderId = $this->cache->getId($path); - } - $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + + foreach ($childQueue as $child => [$childId, $childSize]) { + // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed + $childEtagChanged = false; + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize, $childEtagChanged); + $etagChanged |= $childEtagChanged; - foreach ($childQueue as $child => $childId) { - $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock); if ($childSize === -1) { $size = -1; } elseif ($size !== -1) { $size += $childSize; } } - $oldSize = $data['size'] ?? null; - if ($this->cacheActive && $oldSize !== $size) { - $this->cache->update($folderId, ['size' => $size]); + + // for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size + // to make sure we also updated the unencrypted-size where applicable + if ($this->storage->instanceOfStorage(Encryption::class)) { + $this->cache->calculateFolderSize($path); + } else { + if ($this->cacheActive) { + $updatedData = []; + if ($oldSize !== $size) { + $updatedData['size'] = $size; + } + if ($etagChanged) { + $updatedData['etag'] = uniqid(); + } + if ($updatedData) { + $this->cache->update($folderId, $updatedData); + } + } } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); return $size; } - private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { + /** + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + */ + private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array { // we put this in it's own function so it cleans up the memory before we start recursing $existingChildren = $this->getExistingChildren($folderId); $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); @@ -408,14 +462,14 @@ class Scanner extends BasicEmitter implements IScanner { } if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->beginTransaction(); + $this->connection->beginTransaction(); } $exceptionOccurred = false; $childQueue = []; $newChildNames = []; foreach ($newChildren as $fileMeta) { - $permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions']; + $permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions']; if ($permissions === 0) { continue; } @@ -423,7 +477,7 @@ class Scanner extends BasicEmitter implements IScanner { $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/'); if (trim($originalFile, '/') !== $file) { // encoding mismatch, might require compatibility wrapper - \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']); + \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".', ['app' => 'core']); $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]); // skip this entry continue; @@ -432,27 +486,31 @@ class Scanner extends BasicEmitter implements IScanner { $newChildNames[] = $file; $child = $path ? $path . '/' . $file : $file; try { - $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false; + $existingData = $existingChildren[$file] ?? false; $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta); if ($data) { if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) { - $childQueue[$child] = $data['fileid']; + $childQueue[$child] = [$data['fileid'], $data['size']]; } elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) { // only recurse into folders which aren't fully scanned - $childQueue[$child] = $data['fileid']; + $childQueue[$child] = [$data['fileid'], $data['size']]; } elseif ($data['size'] === -1) { $size = -1; } elseif ($size !== -1) { $size += $data['size']; } + + if (isset($data['etag_changed']) && $data['etag_changed']) { + $etagChanged = true; + } } } catch (Exception $ex) { // might happen if inserting duplicate while a scanning // process is running in parallel // log and ignore if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->rollback(); - \OC::$server->getDatabaseConnection()->beginTransaction(); + $this->connection->rollback(); + $this->connection->beginTransaction(); } \OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [ 'app' => 'core', @@ -461,7 +519,7 @@ class Scanner extends BasicEmitter implements IScanner { $exceptionOccurred = true; } catch (\OCP\Lock\LockedException $e) { if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->rollback(); + $this->connection->rollback(); } throw $e; } @@ -472,7 +530,7 @@ class Scanner extends BasicEmitter implements IScanner { $this->removeFromCache($child); } if ($this->useTransactions) { - \OC::$server->getDatabaseConnection()->commit(); + $this->connection->commit(); } if ($exceptionOccurred) { // It might happen that the parallel scan process has already @@ -496,7 +554,7 @@ class Scanner extends BasicEmitter implements IScanner { if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { return true; } - if (strpos($file, '.part/') !== false) { + if (str_contains($file, '.part/')) { return true; } @@ -536,7 +594,7 @@ class Scanner extends BasicEmitter implements IScanner { } } - private function runBackgroundScanJob(callable $callback, $path) { + protected function runBackgroundScanJob(callable $callback, $path) { try { $callback(); \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]); diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index 63dc4b9cd0e..e1d3c42a8a2 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -1,27 +1,8 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tobias Kaminsky <tobias@kaminsky.me> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; @@ -32,11 +13,17 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; /** * Tools for transforming search queries into database queries + * + * @psalm-import-type ParamSingleValue from ISearchComparison + * @psalm-import-type ParamValue from ISearchComparison */ class SearchBuilder { + /** @var array<string, string> */ protected static $searchOperatorMap = [ ISearchComparison::COMPARE_LIKE => 'iLike', ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like', @@ -45,8 +32,11 @@ class SearchBuilder { ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', ISearchComparison::COMPARE_LESS_THAN => 'lt', ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte', + ISearchComparison::COMPARE_DEFINED => 'isNotNull', + ISearchComparison::COMPARE_IN => 'in', ]; + /** @var array<string, string> */ protected static $searchOperatorNegativeMap = [ ISearchComparison::COMPARE_LIKE => 'notLike', ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike', @@ -55,47 +45,82 @@ class SearchBuilder { ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', ISearchComparison::COMPARE_LESS_THAN => 'gte', ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt', + ISearchComparison::COMPARE_DEFINED => 'isNull', + ISearchComparison::COMPARE_IN => 'notIn', ]; - public const TAG_FAVORITE = '_$!<Favorite>!$_'; + /** @var array<string, string> */ + protected static $fieldTypes = [ + 'mimetype' => 'string', + 'mtime' => 'integer', + 'name' => 'string', + 'path' => 'string', + 'size' => 'integer', + 'tagname' => 'string', + 'systemtag' => 'string', + 'favorite' => 'boolean', + 'fileid' => 'integer', + 'storage' => 'integer', + 'share_with' => 'string', + 'share_type' => 'integer', + 'owner' => 'string', + ]; - /** @var IMimeTypeLoader */ - private $mimetypeLoader; + /** @var array<string, int|string> */ + protected static $paramTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR, + 'integer' => IQueryBuilder::PARAM_INT, + 'boolean' => IQueryBuilder::PARAM_BOOL, + ]; + + /** @var array<string, int> */ + protected static $paramArrayTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR_ARRAY, + 'integer' => IQueryBuilder::PARAM_INT_ARRAY, + 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, + ]; + + public const TAG_FAVORITE = '_$!<Favorite>!$_'; public function __construct( - IMimeTypeLoader $mimetypeLoader + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; } /** - * Whether or not the tag tables should be joined to complete the search - * - * @param ISearchOperator $operator - * @return boolean + * @return string[] */ - public function shouldJoinTags(ISearchOperator $operator) { + public function extractRequestedFields(ISearchOperator $operator): array { if ($operator instanceof ISearchBinaryOperator) { - return array_reduce($operator->getArguments(), function ($shouldJoin, ISearchOperator $operator) { - return $shouldJoin || $this->shouldJoinTags($operator); - }, false); - } elseif ($operator instanceof ISearchComparison) { - return $operator->getField() === 'tagname' || $operator->getField() === 'favorite' || $operator->getField() === 'systemtag'; + return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { + return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); + }, []); + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { + return [$operator->getField()]; } - return false; + return []; } /** * @param IQueryBuilder $builder * @param ISearchOperator[] $operators */ - public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) { - return array_filter(array_map(function ($operator) use ($builder) { - return $this->searchOperatorToDBExpr($builder, $operator); + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null, + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); }, $operators)); } - public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) { + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null, + ) { $expr = $builder->expr(); if ($operator instanceof ISearchBinaryOperator) { @@ -107,62 +132,99 @@ class SearchBuilder { case ISearchBinaryOperator::OPERATOR_NOT: $negativeOperator = $operator->getArguments()[0]; if ($negativeOperator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap); + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); } else { throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); } // no break case ISearchBinaryOperator::OPERATOR_AND: - return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); case ISearchBinaryOperator::OPERATOR_OR: - return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); default: throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); } } elseif ($operator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap); + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); } else { throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); } } - private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) { - $this->validateComparison($comparison); + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null, + ) { + if ($comparison->getExtra()) { + [$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison); + } - [$field, $value, $type] = $this->getOperatorFieldAndValue($comparison); if (isset($operatorMap[$type])) { $queryOperator = $operatorMap[$type]; - return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value)); + return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType)); } else { throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); } } - private function getOperatorFieldAndValue(ISearchComparison $operator) { + /** + * @param ISearchComparison $operator + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValue(ISearchComparison $operator): array { + $this->validateComparison($operator); $field = $operator->getField(); $value = $operator->getValue(); $type = $operator->getType(); + $pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true); + return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash); + } + + /** + * @param string $field + * @param ParamValue $value + * @param string $type + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array { + $paramType = self::$fieldTypes[$field]; + if ($type === ISearchComparison::COMPARE_IN) { + $resultField = $field; + $values = []; + foreach ($value as $arrayValue) { + /** @var ParamSingleValue $arrayValue */ + [$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash); + $resultField = $arrayField; + $values[] = $arrayValue; + } + return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType]; + } if ($field === 'mimetype') { $value = (string)$value; - if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) { - $value = (int)$this->mimetypeLoader->getId($value); - } elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) { + if ($type === ISearchComparison::COMPARE_EQUAL) { + $value = $this->mimetypeLoader->getId($value); + } elseif ($type === ISearchComparison::COMPARE_LIKE) { // transform "mimetype='foo/%'" to "mimepart='foo'" if (preg_match('|(.+)/%|', $value, $matches)) { $field = 'mimepart'; - $value = (int)$this->mimetypeLoader->getId($matches[1]); + $value = $this->mimetypeLoader->getId($matches[1]); $type = ISearchComparison::COMPARE_EQUAL; - } elseif (strpos($value, '%') !== false) { + } elseif (str_contains($value, '%')) { throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); } else { $field = 'mimetype'; - $value = (int)$this->mimetypeLoader->getId($value); + $value = $this->mimetypeLoader->getId($value); $type = ISearchComparison::COMPARE_EQUAL; } } } elseif ($field === 'favorite') { $field = 'tag.category'; $value = self::TAG_FAVORITE; + $paramType = 'string'; } elseif ($field === 'name') { $field = 'file.name'; } elseif ($field === 'tagname') { @@ -171,59 +233,89 @@ class SearchBuilder { $field = 'systemtag.name'; } elseif ($field === 'fileid') { $field = 'file.fileid'; - } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true)) { + } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { $field = 'path_hash'; $value = md5((string)$value); + } elseif ($field === 'owner') { + $field = 'uid_owner'; } - return [$field, $value, $type]; + return [$field, $value, $type, $paramType]; } private function validateComparison(ISearchComparison $operator) { - $types = [ - 'mimetype' => 'string', - 'mtime' => 'integer', - 'name' => 'string', - 'path' => 'string', - 'size' => 'integer', - 'tagname' => 'string', - 'systemtag' => 'string', - 'favorite' => 'boolean', - 'fileid' => 'integer', - 'storage' => 'integer', - ]; $comparisons = [ - 'mimetype' => ['eq', 'like'], + 'mimetype' => ['eq', 'like', 'in'], 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], - 'name' => ['eq', 'like', 'clike'], - 'path' => ['eq', 'like', 'clike'], + 'name' => ['eq', 'like', 'clike', 'in'], + 'path' => ['eq', 'like', 'clike', 'in'], 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], 'tagname' => ['eq', 'like'], 'systemtag' => ['eq', 'like'], 'favorite' => ['eq'], - 'fileid' => ['eq'], - 'storage' => ['eq'], + 'fileid' => ['eq', 'in'], + 'storage' => ['eq', 'in'], + 'share_with' => ['eq'], + 'share_type' => ['eq'], + 'owner' => ['eq'], ]; - if (!isset($types[$operator->getField()])) { + if (!isset(self::$fieldTypes[$operator->getField()])) { throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); } - $type = $types[$operator->getField()]; - if (gettype($operator->getValue()) !== $type) { - throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + $type = self::$fieldTypes[$operator->getField()]; + if ($operator->getType() === ISearchComparison::COMPARE_IN) { + if (!is_array($operator->getValue())) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + foreach ($operator->getValue() as $arrayValue) { + if (gettype($arrayValue) !== $type) { + throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField()); + } + } + } else { + if (gettype($operator->getValue()) !== $type) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } } if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); } } - private function getParameterForValue(IQueryBuilder $builder, $value) { + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + + switch ($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type, $paramType]; + } + + private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) { if ($value instanceof \DateTime) { $value = $value->getTimestamp(); } - if (is_numeric($value)) { - $type = IQueryBuilder::PARAM_INT; + if (is_array($value)) { + $type = self::$paramArrayTypeMap[$paramType]; } else { - $type = IQueryBuilder::PARAM_STR; + $type = self::$paramTypeMap[$paramType]; } return $builder->createNamedParameter($value, $type); } @@ -231,24 +323,32 @@ class SearchBuilder { /** * @param IQueryBuilder $query * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery */ - public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) { + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { foreach ($orders as $order) { $field = $order->getField(); - if ($field === 'fileid') { - $field = 'file.fileid'; - } + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; - // Mysql really likes to pick an index for sorting if it can't fully satisfy the where - // filter with an index, since search queries pretty much never are fully filtered by index - // mysql often picks an index for sorting instead of the much more useful index for filtering. - // - // By changing the order by to an expression, mysql isn't smart enough to see that it could still - // use the index, so it instead picks an index for the filtering - if ($field === 'mtime') { - $field = $query->func()->add($field, $query->createNamedParameter(0)); - } + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } $query->addOrderBy($field, $order->getDirection()); } } diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 01fc638cef8..1a3bda58e6a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -1,36 +1,15 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Storage\IStorage; +use OCP\IDBConnection; use Psr\Log\LoggerInterface; /** @@ -65,7 +44,7 @@ class Storage { * @param bool $isAvailable * @throws \RuntimeException */ - public function __construct($storage, $isAvailable = true) { + public function __construct($storage, $isAvailable, IDBConnection $connection) { if ($storage instanceof IStorage) { $this->storageId = $storage->getId(); } else { @@ -76,7 +55,6 @@ class Storage { if ($row = self::getStorageById($this->storageId)) { $this->numericId = (int)$row['numeric_id']; } else { - $connection = \OC::$server->getDatabaseConnection(); $available = $isAvailable ? 1 : 0; if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { $this->numericId = $connection->lastInsertId('*PREFIX*storages'); @@ -102,7 +80,7 @@ class Storage { * Adjusts the storage id to use md5 if too long * @param string $storageId storage id * @return string unchanged $storageId if its length is less than 64 characters, - * else returns the md5 of $storageId + * else returns the md5 of $storageId */ public static function adjustStorageId($storageId) { if (strlen($storageId) > 64) { @@ -171,7 +149,7 @@ class Storage { public function setAvailability($isAvailable, int $delay = 0) { $available = $isAvailable ? 1 : 0; if (!$isAvailable) { - \OC::$server->get(LoggerInterface::class)->info('Storage with ' . $this->storageId . ' marked as unavailable', ['app' => 'lib']); + \OCP\Server::get(LoggerInterface::class)->info('Storage with ' . $this->storageId . ' marked as unavailable', ['app' => 'lib']); } $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); @@ -179,7 +157,7 @@ class Storage { ->set('available', $query->createNamedParameter($available)) ->set('last_checked', $query->createNamedParameter(time() + $delay)) ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); - $query->execute(); + $query->executeStatement(); } /** @@ -204,13 +182,13 @@ class Storage { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($storageId))); - $query->execute(); + $query->executeStatement(); if (!is_null($numericId)) { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->eq('storage', $query->createNamedParameter($numericId))); - $query->execute(); + $query->executeStatement(); } } @@ -235,6 +213,7 @@ class Storage { $query = $db->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); $query->executeStatement(); $query = $db->getQueryBuilder(); diff --git a/lib/private/Files/Cache/StorageGlobal.php b/lib/private/Files/Cache/StorageGlobal.php index 74cbd5abdb2..bab31b1db91 100644 --- a/lib/private/Files/Cache/StorageGlobal.php +++ b/lib/private/Files/Cache/StorageGlobal.php @@ -1,25 +1,8 @@ <?php + /** - * @copyright Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache; @@ -38,16 +21,14 @@ use OCP\IDBConnection; * @package OC\Files\Cache */ class StorageGlobal { - /** @var IDBConnection */ - private $connection; - /** @var array<string, array> */ private $cache = []; /** @var array<int, array> */ private $numericIdCache = []; - public function __construct(IDBConnection $connection) { - $this->connection = $connection; + public function __construct( + private IDBConnection $connection, + ) { } /** @@ -59,7 +40,7 @@ class StorageGlobal { ->from('storages') ->where($builder->expr()->in('id', $builder->createNamedParameter(array_values($storageIds), IQueryBuilder::PARAM_STR_ARRAY))); - $result = $query->execute(); + $result = $query->executeQuery(); while ($row = $result->fetch()) { $this->cache[$row['id']] = $row; } @@ -77,7 +58,7 @@ class StorageGlobal { ->from('storages') ->where($builder->expr()->eq('id', $builder->createNamedParameter($storageId))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); @@ -100,7 +81,7 @@ class StorageGlobal { ->from('storages') ->where($builder->expr()->eq('numeric_id', $builder->createNamedParameter($numericId))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php index f8c187996e6..03681036aa2 100644 --- a/lib/private/Files/Cache/Updater.php +++ b/lib/private/Files/Cache/Updater.php @@ -1,36 +1,20 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Michael Gapczynski <GapczynskiM@gmail.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; +use Doctrine\DBAL\Exception\DeadlockException; use OC\Files\FileInfo; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Cache\IUpdater; use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; /** * Update the cache and propagate changes @@ -62,6 +46,8 @@ class Updater implements IUpdater { */ protected $cache; + private LoggerInterface $logger; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -70,6 +56,7 @@ class Updater implements IUpdater { $this->propagator = $storage->getPropagator(); $this->scanner = $storage->getScanner(); $this->cache = $storage->getCache(); + $this->logger = \OC::$server->get(LoggerInterface::class); } /** @@ -114,7 +101,7 @@ class Updater implements IUpdater { * @param string $path * @param int $time */ - public function update($path, $time = null) { + public function update($path, $time = null, ?int $sizeDifference = null) { if (!$this->enabled or Scanner::isPartialFile($path)) { return; } @@ -123,20 +110,22 @@ class Updater implements IUpdater { } $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); - if ( - isset($data['oldSize']) && isset($data['size']) && - !$data['encrypted'] // encryption is a pita and touches the cache itself - ) { + + if (isset($data['oldSize']) && isset($data['size'])) { $sizeDifference = $data['size'] - $data['oldSize']; - } else { - // scanner didn't provide size info, fallback to full size calculation - $sizeDifference = 0; - if ($this->cache instanceof Cache) { - $this->cache->correctFolderSize($path, $data); - } + } + + // encryption is a pita and touches the cache itself + if (isset($data['encrypted']) && (bool)$data['encrypted']) { + $sizeDifference = null; + } + + // scanner didn't provide size info, fallback to full size calculation + if ($this->cache instanceof Cache && $sizeDifference === null) { + $this->cache->correctFolderSize($path, $data); } $this->correctParentStorageMtime($path); - $this->propagator->propagateChange($path, $time, $sizeDifference); + $this->propagator->propagateChange($path, $time, $sizeDifference ?? 0); } /** @@ -170,13 +159,51 @@ class Updater implements IUpdater { } /** - * Rename a file or folder in the cache and update the size, etag and mtime of the parent folders + * Rename a file or folder in the cache. * * @param IStorage $sourceStorage * @param string $source * @param string $target */ public function renameFromStorage(IStorage $sourceStorage, $source, $target) { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache) use ($sourceStorage, $source, $target) { + // Remove existing cache entry to no reuse the fileId. + if ($this->cache->inCache($target)) { + $this->cache->remove($target); + } + + if ($sourceStorage === $this->storage) { + $this->cache->move($source, $target); + } else { + $this->cache->moveFromCache($sourceCache, $source, $target); + } + }); + } + + /** + * Copy a file or folder in the cache. + */ + public function copyFromStorage(IStorage $sourceStorage, string $source, string $target): void { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache, ICacheEntry $sourceInfo) use ($target) { + $parent = dirname($target); + if ($parent === '.') { + $parent = ''; + } + $parentInCache = $this->cache->inCache($parent); + if (!$parentInCache) { + $parentData = $this->scanner->scan($parent, Scanner::SCAN_SHALLOW, -1, false); + $parentInCache = $parentData !== null; + } + if ($parentInCache) { + $this->cache->copyFromCache($sourceCache, $sourceInfo, $target); + } + }); + } + + /** + * Utility to copy or rename a file or folder in the cache and update the size, etag and mtime of the parent folders + */ + private function copyOrRenameFromStorage(IStorage $sourceStorage, string $source, string $target, callable $operation): void { if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) { return; } @@ -189,27 +216,25 @@ class Updater implements IUpdater { $sourceInfo = $sourceCache->get($source); - if ($sourceInfo !== false) { - if ($this->cache->inCache($target)) { - $this->cache->remove($target); - } + $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); + $targetExtension = pathinfo($target, PATHINFO_EXTENSION); + $targetIsTrash = preg_match("/^d\d+$/", $targetExtension); - if ($sourceStorage === $this->storage) { - $this->cache->move($source, $target); - } else { - $this->cache->moveFromCache($sourceCache, $source, $target); + if ($sourceInfo !== false) { + if (!$this->storage->instanceOfStorage(ObjectStoreStorage::class)) { + $operation($sourceCache, $sourceInfo); } - $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); - $targetExtension = pathinfo($target, PATHINFO_EXTENSION); - $targetIsTrash = preg_match("/d\d+/", $targetExtension); + $isDir = $sourceInfo->getMimeType() === FileInfo::MIMETYPE_FOLDER; + } else { + $isDir = $this->storage->is_dir($target); + } - if ($sourceExtension !== $targetExtension && $sourceInfo->getMimeType() !== FileInfo::MIMETYPE_FOLDER && !$targetIsTrash) { - // handle mime type change - $mimeType = $this->storage->getMimeType($target); - $fileId = $this->cache->getId($target); - $this->cache->update($fileId, ['mimetype' => $mimeType]); - } + if ($sourceExtension !== $targetExtension && !$isDir && !$targetIsTrash) { + // handle mime type change + $mimeType = $this->storage->getMimeType($target); + $fileId = $this->cache->getId($target); + $this->cache->update($fileId, ['mimetype' => $mimeType]); } if ($sourceCache instanceof Cache) { @@ -253,7 +278,14 @@ class Updater implements IUpdater { if ($parentId != -1) { $mtime = $this->storage->filemtime($parent); if ($mtime !== false) { - $this->cache->update($parentId, ['storage_mtime' => $mtime]); + try { + $this->cache->update($parentId, ['storage_mtime' => $mtime]); + } catch (DeadlockException $e) { + // ignore the failure. + // with failures concurrent updates, someone else would have already done it. + // in the worst case the `storage_mtime` isn't updated, which should at most only trigger an extra rescan + $this->logger->warning('Error while updating parent storage_mtime, should be safe to ignore', ['exception' => $e]); + } } } } diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php index acc76f263dc..f1de5d3cfb8 100644 --- a/lib/private/Files/Cache/Watcher.php +++ b/lib/private/Files/Cache/Watcher.php @@ -1,28 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache; @@ -52,6 +33,9 @@ class Watcher implements IWatcher { */ protected $scanner; + /** @var callable[] */ + protected $onUpdate = []; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -119,6 +103,9 @@ class Watcher implements IWatcher { if ($this->cache instanceof Cache) { $this->cache->correctFolderSize($path); } + foreach ($this->onUpdate as $callback) { + $callback($path); + } } /** @@ -129,9 +116,9 @@ class Watcher implements IWatcher { * @return bool */ public function needsUpdate($path, $cachedData) { - if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and array_search($path, $this->checkedPaths) === false)) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { $this->checkedPaths[] = $path; - return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); } return false; } @@ -149,4 +136,11 @@ class Watcher implements IWatcher { } } } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index 628ca3ee0e0..5bc4ee8529d 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -1,35 +1,17 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; @@ -39,29 +21,29 @@ use OCP\Files\Search\ISearchOperator; * Jail to a subdirectory of the wrapped cache */ class CacheJail extends CacheWrapper { - /** - * @var string - */ - protected $root; - protected $unjailedRoot; - /** - * @param ?\OCP\Files\Cache\ICache $cache - * @param string $root - */ - public function __construct($cache, $root) { - parent::__construct($cache); - $this->root = $root; - $this->connection = \OC::$server->getDatabaseConnection(); - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); - - if ($cache instanceof CacheJail) { - $this->unjailedRoot = $cache->getSourcePath($root); - } else { - $this->unjailedRoot = $root; + protected string $unjailedRoot; + + public function __construct( + ?ICache $cache, + protected string $root, + ?CacheDependencies $dependencies = null, + ) { + parent::__construct($cache, $dependencies); + + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); } } + /** + * @return string + */ protected function getRoot() { return $this->root; } @@ -71,11 +53,14 @@ class CacheJail extends CacheWrapper { * * @return string */ - protected function getGetUnjailedRoot() { + public function getGetUnjailedRoot() { return $this->unjailedRoot; } - protected function getSourcePath($path) { + /** + * @return string + */ + protected function getSourcePath(string $path) { if ($path === '') { return $this->getRoot(); } else { @@ -88,7 +73,7 @@ class CacheJail extends CacheWrapper { * @param null|string $root * @return null|string the jailed path or null if the path is outside the jail */ - protected function getJailedPath(string $path, string $root = null) { + protected function getJailedPath(string $path, ?string $root = null) { if ($root === null) { $root = $this->getRoot(); } @@ -115,7 +100,7 @@ class CacheJail extends CacheWrapper { /** * get the stored metadata of a file or folder * - * @param string /int $file + * @param string|int $file * @return ICacheEntry|false */ public function get($file) { @@ -226,12 +211,12 @@ class CacheJail extends CacheWrapper { /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); } } @@ -239,12 +224,13 @@ class CacheJail extends CacheWrapper { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($this->getSourcePath($path), $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); } else { return 0; } @@ -328,7 +314,7 @@ class CacheJail extends CacheWrapper { } public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { - if ($this->getGetUnjailedRoot() === '' || strpos($rawEntry->getPath(), $this->getGetUnjailedRoot()) === 0) { + if ($this->getGetUnjailedRoot() === '' || str_starts_with($rawEntry->getPath(), $this->getGetUnjailedRoot())) { $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); if ($rawEntry) { $jailedPath = $this->getJailedPath($rawEntry->getPath()); diff --git a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php index cbf16a909ff..ff17cb79ac7 100644 --- a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php +++ b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php @@ -1,24 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache\Wrapper; diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index 66ae83fd144..f2f1036d6a3 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -1,61 +1,55 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Ari Selseng <ari@selseng.net> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Jagszent <daniel@jagszent.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; -use OC\Files\Cache\QuerySearchHelper; +use OC\Files\Cache\CacheDependencies; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; +use OCP\Server; class CacheWrapper extends Cache { /** - * @var \OCP\Files\Cache\ICache + * @var ?ICache */ protected $cache; - /** - * @param \OCP\Files\Cache\ICache $cache - */ - public function __construct($cache) { + public function __construct(?ICache $cache, ?CacheDependencies $dependencies = null) { $this->cache = $cache; - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); - $this->connection = \OC::$server->getDatabaseConnection(); - $this->querySearchHelper = \OC::$server->get(QuerySearchHelper::class); + if (!$dependencies && $cache instanceof Cache) { + $this->mimetypeLoader = $cache->mimetypeLoader; + $this->connection = $cache->connection; + $this->querySearchHelper = $cache->querySearchHelper; + } else { + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + } } protected function getCache() { return $this->cache; } + protected function hasEncryptionWrapper(): bool { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->hasEncryptionWrapper(); + } else { + return false; + } + } + /** * Make it easy for wrappers to modify every returned cache entry * @@ -74,7 +68,7 @@ class CacheWrapper extends Cache { */ public function get($file) { $result = $this->getCache()->get($file); - if ($result) { + if ($result instanceof ICacheEntry) { $result = $this->formatCacheEntry($result); } return $result; @@ -220,19 +214,19 @@ class CacheWrapper extends Cache { return $this->getCache()->getStatus($file); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($path, $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); } } @@ -240,12 +234,13 @@ class CacheWrapper extends Cache { * get the size of a folder and set it in the cache * * @param string $path - * @param array $entry (optional) meta data of the folder - * @return int + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($path, $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); } else { return 0; } diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php index 25e53ded39d..d6409b7875e 100644 --- a/lib/private/Files/Cache/Wrapper/JailPropagator.php +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -1,24 +1,8 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\Cache\Wrapper; diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php new file mode 100644 index 00000000000..b1ae516654a --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Watcher; + +class JailWatcher extends Watcher { + private string $root; + private Watcher $watcher; + + public function __construct(Watcher $watcher, string $root) { + $this->watcher = $watcher; + $this->root = $root; + } + + protected function getRoot(): string { + return $this->root; + } + + protected function getSourcePath($path): string { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + public function setPolicy($policy) { + $this->watcher->setPolicy($policy); + } + + public function getPolicy() { + return $this->watcher->getPolicy(); + } + + + public function checkUpdate($path, $cachedEntry = null) { + return $this->watcher->checkUpdate($this->getSourcePath($path), $cachedEntry); + } + + public function update($path, $cachedData) { + $this->watcher->update($this->getSourcePath($path), $cachedData); + } + + public function needsUpdate($path, $cachedData) { + return $this->watcher->needsUpdate($this->getSourcePath($path), $cachedData); + } + + public function cleanFolder($path) { + $this->watcher->cleanFolder($this->getSourcePath($path)); + } + + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } +} |