diff options
Diffstat (limited to 'lib/private/Files/Cache')
| -rw-r--r-- | lib/private/Files/Cache/Cache.php | 57 | ||||
| -rw-r--r-- | lib/private/Files/Cache/CacheEntry.php | 4 | ||||
| -rw-r--r-- | lib/private/Files/Cache/CacheQueryBuilder.php | 2 | ||||
| -rw-r--r-- | lib/private/Files/Cache/FileAccess.php | 125 | ||||
| -rw-r--r-- | lib/private/Files/Cache/QuerySearchHelper.php | 1 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Scanner.php | 254 | ||||
| -rw-r--r-- | lib/private/Files/Cache/SearchBuilder.php | 18 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Storage.php | 1 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Watcher.php | 15 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/CacheJail.php | 48 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/CacheWrapper.php | 15 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/JailPropagator.php | 1 | ||||
| -rw-r--r-- | lib/private/Files/Cache/Wrapper/JailWatcher.php | 3 |
13 files changed, 345 insertions, 199 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index cb841755efd..329466e682d 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -109,7 +109,7 @@ class Cache implements ICache { /** * get the stored metadata of a file or folder * - * @param string | int $file either the path of a file or folder or the file id for a file or folder + * @param string|int $file either the path of a file or folder or the file id for a file or folder * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache */ public function get($file) { @@ -131,15 +131,17 @@ class Cache implements ICache { $data = $result->fetch(); $result->closeCursor(); - //merge partial data - if (!$data && is_string($file) && isset($this->partial[$file])) { - return $this->partial[$file]; - } elseif (!$data) { - return $data; - } else { + if ($data !== false) { $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } } + + return false; } /** @@ -661,13 +663,13 @@ class Cache implements ICache { $sourceData = $sourceCache->get($sourcePath); if (!$sourceData) { - throw new \Exception('Invalid source storage path: ' . $sourcePath); + throw new \Exception('Source path not found in cache: ' . $sourcePath); } $shardDefinition = $this->connection->getShardDefinition('filecache'); if ( - $shardDefinition && - $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) ) { $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); return; @@ -886,19 +888,23 @@ class Cache implements ICache { /** * Re-calculate the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { $this->calculateFolderSize($path, $data); + if ($path !== '') { $parent = dirname($path); if ($parent === '.' || $parent === '/') { $parent = ''; } + if ($isBackgroundScan) { $parentData = $this->get($parent); - if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) { + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { $this->correctFolderSize($parent, $parentData, $isBackgroundScan); } } else { @@ -920,7 +926,7 @@ class Cache implements ICache { ->from('filecache') ->whereParent($fileId) ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); $result = $query->executeQuery(); $size = (int)$result->fetchOne(); @@ -1009,8 +1015,8 @@ class Cache implements ICache { } // only set unencrypted size for a folder if any child entries have it set, or the folder is empty - $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || $entry['unencrypted_size'] > 0; - if ($entry['size'] !== $totalSize || ($entry['unencrypted_size'] !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { if ($shouldWriteUnEncryptedSize) { // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes if ($unencryptedMax === 0) { @@ -1061,28 +1067,19 @@ class Cache implements ICache { * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { - // we select the fileid here first instead of directly selecting the path since this helps mariadb/mysql - // to use the correct index. - // The overhead of this should be minimal since the cost of selecting the path by id should be much lower - // than the cost of finding an item with size < 0 $query = $this->getQueryBuilder(); - $query->select('fileid') + $query->select('path') ->from('filecache') ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('fileid', 'DESC') ->setMaxResults(1); $result = $query->executeQuery(); - $id = $result->fetchOne(); + $path = $result->fetchOne(); $result->closeCursor(); - if ($id === false) { - return false; - } - - $path = $this->getPathById($id); - return $path ?? false; + return $path === false ? false : (string)$path; } /** diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index e9417c8012a..ab5bae316f4 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -110,6 +110,10 @@ class CacheEntry implements ICacheEntry { return $this->data['upload_time'] ?? null; } + public function getParentId(): int { + return $this->data['parent']; + } + public function getData() { return $this->data; } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 5ae60ee80b6..5492452273b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -28,7 +28,7 @@ class CacheQueryBuilder extends ExtendedQueryBuilder { public function selectTagUsage(): self { $this - ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag') + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') ->from('filecache', 'filecache') diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 11a95b5d897..c3f3614f3ca 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -10,6 +10,7 @@ namespace OC\Files\Cache; use OC\FilesMetadata\FilesMetadataManager; use OC\SystemConfig; +use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IFileAccess; use OCP\Files\IMimeTypeLoader; @@ -94,4 +95,128 @@ class FileAccess implements IFileAccess { $rows = $query->executeQuery()->fetchAll(); return $this->rowsToEntries($rows); } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index ff2d6766893..3ddcf1ca4e6 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 3bd674f79e2..b067f70b8cb 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -83,7 +83,7 @@ class Scanner extends BasicEmitter implements IScanner { * * @param bool $useTransactions */ - public function setUseTransactions($useTransactions) { + public function setUseTransactions($useTransactions): void { $this->useTransactions = $useTransactions; } @@ -108,9 +108,9 @@ class Scanner extends BasicEmitter implements IScanner { * @param string $file * @param int $reuseExisting * @param int $parentId - * @param array|null|false $cacheData existing data in the cache for the file to be scanned + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param null $data the metadata for the file, as returned by the storage + * @param array|null $data the metadata for the file, as returned by the storage * @return array|null an array of metadata of the scanned file * @throws \OCP\Lock\LockedException */ @@ -122,139 +122,130 @@ class Scanner extends BasicEmitter implements IScanner { return null; } } + // only proceed if $file is not a partial file, blacklist is handled by the storage - if (!self::isPartialFile($file)) { - // acquire a lock + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { if ($lock) { if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } } - try { - $data = $data ?? $this->getData($file); - } catch (ForbiddenException $e) { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } - } + return null; + } - return null; - } + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + } - try { - if ($data) { - // pre-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); - } + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } - $parent = dirname($file); - if ($parent === '.' || $parent === '/') { - $parent = ''; - } - if ($parentId === -1) { - $parentId = $this->cache->getParentId($file); + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; } - // scan the parent if it's not in the cache (id -1) and the current file is not the root folder - if ($file && $parentId === -1) { - $parentData = $this->scanFile($parent); - if (!$parentData) { - return null; - } - $parentId = $parentData['fileid']; - } - if ($parent) { - $data['parent'] = $parentId; - } - if (is_null($cacheData)) { - /** @var CacheEntry $cacheData */ - $cacheData = $this->cache->get($file); - } - if ($cacheData && $reuseExisting && isset($cacheData['fileid'])) { - // prevent empty etag - $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; - $fileId = $cacheData['fileid']; - $data['fileid'] = $fileId; - // only reuse data if the file hasn't explicitly changed - $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; - // if the folder is marked as unscanned, never reuse etags - if ($mtimeUnchanged && $cacheData['size'] !== -1) { - $data['mtime'] = $cacheData['mtime']; - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { - $data['size'] = $cacheData['size']; - } - if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { - $data['etag'] = $etag; - } - } + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } - // we only updated unencrypted_size if it's already set - if ($cacheData['unencrypted_size'] === 0) { - unset($data['unencrypted_size']); + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; } - - // Only update metadata that has changed - // i.e. get all the values in $data that are not present in the cache already - $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); - - // make it known to the caller that etag has been changed and needs propagation - if (isset($newData['etag'])) { - $data['etag_changed'] = true; + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; } - } else { - // we only updated unencrypted_size if it's already set - unset($data['unencrypted_size']); - $newData = $data; - $fileId = -1; } - if (!empty($newData)) { - // Reset the checksum if the data has changed - $newData['checksum'] = ''; - $newData['parent'] = $parentId; - $data['fileid'] = $this->addToCache($file, $newData, $fileId); - } - - $data['oldSize'] = ($cacheData && isset($cacheData['size'])) ? $cacheData['size'] : 0; - if ($cacheData && isset($cacheData['encrypted'])) { - $data['encrypted'] = $cacheData['encrypted']; + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); } - // post-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; } } else { - $this->removeFromCache($file); + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; } - } catch (\Exception $e) { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); } - throw $e; - } - // release the acquired lock - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; } - } - if ($data && !isset($data['encrypted'])) { - $data['encrypted'] = false; + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } - return $data; } - return null; + return $data; } protected function removeFromCache($path) { @@ -319,29 +310,26 @@ class Scanner extends BasicEmitter implements IScanner { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } + try { - try { - $data = $this->scanFile($path, $reuse, -1, null, $lock); - if ($data && $data['mimetype'] === 'httpd/unix-directory') { - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); - $data['size'] = $size; - } - } catch (NotFoundException $e) { - $this->removeFromCache($path); - return null; + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); + $data['size'] = $size; } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; } finally { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - } + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); } } return $data; @@ -363,23 +351,23 @@ class Scanner extends BasicEmitter implements IScanner { * */ protected function array_diff_assoc_multi(array $array1, array $array2) { - + $result = []; foreach ($array1 as $key => $value) { - + // if $array2 doesn't have the same key, that's a result if (!array_key_exists($key, $array2)) { $result[$key] = $value; continue; } - + // if $array2's value for the same key is different, that's a result if ($array2[$key] !== $value && !is_array($value)) { $result[$key] = $value; continue; } - + if (is_array($value)) { $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); if (!empty($nestedDiff)) { @@ -395,9 +383,9 @@ class Scanner extends BasicEmitter implements IScanner { * Get the children currently in the cache * * @param int $folderId - * @return array[] + * @return array<string, \OCP\Files\Cache\ICacheEntry> */ - protected function getExistingChildren($folderId) { + protected function getExistingChildren($folderId): array { $existingChildren = []; $children = $this->cache->getFolderContentsById($folderId); foreach ($children as $child) { diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index e89b95b756f..e1d3c42a8a2 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -12,6 +13,7 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FilesMetadata\IMetadataQuery; /** @@ -80,13 +82,10 @@ class SearchBuilder { public const TAG_FAVORITE = '_$!<Favorite>!$_'; - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - public function __construct( - IMimeTypeLoader $mimetypeLoader, + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; } /** @@ -285,11 +284,18 @@ class SearchBuilder { private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { - $paramType = self::$fieldTypes[$operator->getField()]; $field = $operator->getField(); $value = $operator->getValue(); $type = $operator->getType(); + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + switch ($operator->getExtra()) { case IMetadataQuery::EXTRA: $metadataQuery->joinIndex($field); // join index table if not joined yet diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 2b49e65f0b4..1a3bda58e6a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -213,6 +213,7 @@ class Storage { $query = $db->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); $query->executeStatement(); $query = $db->getQueryBuilder(); diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php index 2e42b716695..f1de5d3cfb8 100644 --- a/lib/private/Files/Cache/Watcher.php +++ b/lib/private/Files/Cache/Watcher.php @@ -33,6 +33,9 @@ class Watcher implements IWatcher { */ protected $scanner; + /** @var callable[] */ + protected $onUpdate = []; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -100,6 +103,9 @@ class Watcher implements IWatcher { if ($this->cache instanceof Cache) { $this->cache->correctFolderSize($path); } + foreach ($this->onUpdate as $callback) { + $callback($path); + } } /** @@ -112,7 +118,7 @@ class Watcher implements IWatcher { public function needsUpdate($path, $cachedData) { if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { $this->checkedPaths[] = $path; - return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); } return false; } @@ -130,4 +136,11 @@ class Watcher implements IWatcher { } } } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index ea0f992114a..5bc4ee8529d 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -21,27 +21,29 @@ use OCP\Files\Search\ISearchOperator; * Jail to a subdirectory of the wrapped cache */ class CacheJail extends CacheWrapper { - /** - * @var string - */ - protected $root; - protected $unjailedRoot; + + protected string $unjailedRoot; public function __construct( ?ICache $cache, - string $root, + protected string $root, ?CacheDependencies $dependencies = null, ) { parent::__construct($cache, $dependencies); - $this->root = $root; - if ($cache instanceof CacheJail) { - $this->unjailedRoot = $cache->getSourcePath($root); - } else { - $this->unjailedRoot = $root; + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); } } + /** + * @return string + */ protected function getRoot() { return $this->root; } @@ -51,11 +53,14 @@ class CacheJail extends CacheWrapper { * * @return string */ - protected function getGetUnjailedRoot() { + public function getGetUnjailedRoot() { return $this->unjailedRoot; } - protected function getSourcePath($path) { + /** + * @return string + */ + protected function getSourcePath(string $path) { if ($path === '') { return $this->getRoot(); } else { @@ -95,7 +100,7 @@ class CacheJail extends CacheWrapper { /** * get the stored metadata of a file or folder * - * @param string /int $file + * @param string|int $file * @return ICacheEntry|false */ public function get($file) { @@ -206,12 +211,12 @@ class CacheJail extends CacheWrapper { /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); } } @@ -223,8 +228,9 @@ class CacheJail extends CacheWrapper { * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($this->getSourcePath($path), $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); } else { return 0; } diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index fdaa2cf4b7a..f2f1036d6a3 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -221,12 +221,12 @@ class CacheWrapper extends Cache { /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($path, $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); } } @@ -238,8 +238,9 @@ class CacheWrapper extends Cache { * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($path, $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); } else { return 0; } diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php index 19ca4a13ece..d6409b7875e 100644 --- a/lib/private/Files/Cache/Wrapper/JailPropagator.php +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php index 9bd7da57233..b1ae516654a 100644 --- a/lib/private/Files/Cache/Wrapper/JailWatcher.php +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -55,4 +55,7 @@ class JailWatcher extends Watcher { $this->watcher->cleanFolder($this->getSourcePath($path)); } + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } } |