diff options
Diffstat (limited to 'lib/private/Files/Cache')
24 files changed, 5138 insertions, 0 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php new file mode 100644 index 00000000000..329466e682d --- /dev/null +++ b/lib/private/Files/Cache/Cache.php @@ -0,0 +1,1284 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception\UniqueConstraintViolationException; +use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery; +use OC\Files\Storage\Wrapper\Encryption; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\CacheEntryInsertedEvent; +use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntryUpdatedEvent; +use OCP\Files\Cache\CacheInsertEvent; +use OCP\Files\Cache\CacheUpdateEvent; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\FileInfo; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use OCP\Util; +use Psr\Log\LoggerInterface; + +/** + * Metadata cache for a storage + * + * The cache stores the metadata for all files and folders in a storage and is kept up to date through the following mechanisms: + * + * - Scanner: scans the storage and updates the cache where needed + * - Watcher: checks for changes made to the filesystem outside of the Nextcloud instance and rescans files and folder when a change is detected + * - Updater: listens to changes made to the filesystem inside of the Nextcloud instance and updates the cache where needed + * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater + */ +class Cache implements ICache { + use MoveFromCacheTrait { + MoveFromCacheTrait::moveFromCache as moveFromCacheFallback; + } + + /** + * @var array partial data for the cache + */ + protected array $partial = []; + protected string $storageId; + protected Storage $storageCache; + protected IMimeTypeLoader $mimetypeLoader; + protected IDBConnection $connection; + protected SystemConfig $systemConfig; + protected LoggerInterface $logger; + protected QuerySearchHelper $querySearchHelper; + protected IEventDispatcher $eventDispatcher; + protected IFilesMetadataManager $metadataManager; + + public function __construct( + private IStorage $storage, + // this constructor is used in to many pleases to easily do proper di + // so instead we group it all together + ?CacheDependencies $dependencies = null, + ) { + $this->storageId = $storage->getId(); + if (strlen($this->storageId) > 64) { + $this->storageId = md5($this->storageId); + } + if (!$dependencies) { + $dependencies = \OCP\Server::get(CacheDependencies::class); + } + $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->systemConfig = $dependencies->getSystemConfig(); + $this->logger = $dependencies->getLogger(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + $this->eventDispatcher = $dependencies->getEventDispatcher(); + $this->metadataManager = $dependencies->getMetadataManager(); + } + + protected function getQueryBuilder() { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getStorageCache(): Storage { + return $this->storageCache; + } + + /** + * Get the numeric storage id for this cache's storage + * + * @return int + */ + public function getNumericStorageId() { + return $this->storageCache->getNumericId(); + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file either the path of a file or folder or the file id for a file or folder + * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache + */ + public function get($file) { + $query = $this->getQueryBuilder(); + $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); + + if (is_string($file) || $file == '') { + // normalize file + $file = $this->normalize($file); + + $query->wherePath($file); + } else { //file id + $query->whereFileId($file); + } + $query->whereStorageId($this->getNumericStorageId()); + + $result = $query->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + + if ($data !== false) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } + } + + return false; + } + + /** + * Create a CacheEntry from database row + * + * @param array $data + * @param IMimeTypeLoader $mimetypeLoader + * @return CacheEntry + */ + public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) { + //fix types + $data['name'] = (string)$data['name']; + $data['path'] = (string)$data['path']; + $data['fileid'] = (int)$data['fileid']; + $data['parent'] = (int)$data['parent']; + $data['size'] = Util::numericToNumber($data['size']); + $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0); + $data['mtime'] = (int)$data['mtime']; + $data['storage_mtime'] = (int)$data['storage_mtime']; + $data['encryptedVersion'] = (int)$data['encrypted']; + $data['encrypted'] = (bool)$data['encrypted']; + $data['storage_id'] = $data['storage']; + $data['storage'] = (int)$data['storage']; + $data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']); + $data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']); + if ($data['storage_mtime'] == 0) { + $data['storage_mtime'] = $data['mtime']; + } + if (isset($data['f_permissions'])) { + $data['scan_permissions'] = $data['f_permissions']; + } + $data['permissions'] = (int)$data['permissions']; + if (isset($data['creation_time'])) { + $data['creation_time'] = (int)$data['creation_time']; + } + if (isset($data['upload_time'])) { + $data['upload_time'] = (int)$data['upload_time']; + } + return new CacheEntry($data); + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return ICacheEntry[] + */ + public function getFolderContentsById($fileId) { + if ($fileId > -1) { + $query = $this->getQueryBuilder(); + $query->selectFileCache() + ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) + ->orderBy('name', 'ASC'); + + $metadataQuery = $query->selectMetadata(); + + $result = $query->executeQuery(); + $files = $result->fetchAll(); + $result->closeCursor(); + + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return self::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); + } + return []; + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + // normalize file + $file = $this->normalize($file); + + if (isset($this->partial[$file])) { //add any saved partial data + $data = array_merge($this->partial[$file]->getData(), $data); + unset($this->partial[$file]); + } + + $requiredFields = ['size', 'mtime', 'mimetype']; + foreach ($requiredFields as $field) { + if (!isset($data[$field])) { //data not complete save as partial and return + $this->partial[$file] = new CacheEntry($data); + return -1; + } + } + + $data['path'] = $file; + if (!isset($data['parent'])) { + $data['parent'] = $this->getParentId($file); + } + if ($data['parent'] === -1 && $file !== '') { + throw new \Exception('Parent folder not in filecache for ' . $file); + } + $data['name'] = basename($file); + + [$values, $extensionValues] = $this->normalizeData($data); + $storageId = $this->getNumericStorageId(); + $values['storage'] = $storageId; + + try { + $builder = $this->connection->getQueryBuilder(); + $builder->insert('filecache'); + + foreach ($values as $column => $value) { + $builder->setValue($column, $builder->createNamedParameter($value)); + } + + if ($builder->execute()) { + $fileId = $builder->getLastInsertId(); + + if (count($extensionValues)) { + $query = $this->getQueryBuilder(); + $query->insert('filecache_extended'); + $query->hintShardKey('storage', $storageId); + + $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + foreach ($extensionValues as $column => $value) { + $query->setValue($column, $query->createNamedParameter($value)); + } + $query->executeStatement(); + } + + $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId); + $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + return $fileId; + } + } catch (UniqueConstraintViolationException $e) { + // entry exists already + if ($this->connection->inTransaction()) { + $this->connection->commit(); + $this->connection->beginTransaction(); + } + } + + // The file was created in the mean time + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + throw new \RuntimeException('File entry could not be inserted but could also not be selected with getId() in order to perform an update. Please try again.'); + } + } + + /** + * update the metadata of an existing file or folder in the cache + * + * @param int $id the fileid of the existing file or folder + * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged + */ + public function update($id, array $data) { + if (isset($data['path'])) { + // normalize path + $data['path'] = $this->normalize($data['path']); + } + + if (isset($data['name'])) { + // normalize path + $data['name'] = $this->normalize($data['name']); + } + + [$values, $extensionValues] = $this->normalizeData($data); + + if (count($values)) { + $query = $this->getQueryBuilder(); + + $query->update('filecache') + ->whereFileId($id) + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { + return $query->expr()->orX( + $query->expr()->neq($key, $query->createNamedParameter($value)), + $query->expr()->isNull($key) + ); + }, array_keys($values), array_values($values)))); + + foreach ($values as $key => $value) { + $query->set($key, $query->createNamedParameter($value)); + } + + $query->executeStatement(); + } + + if (count($extensionValues)) { + try { + $query = $this->getQueryBuilder(); + $query->insert('filecache_extended'); + $query->hintShardKey('storage', $this->getNumericStorageId()); + + $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); + foreach ($extensionValues as $column => $value) { + $query->setValue($column, $query->createNamedParameter($value)); + } + + $query->execute(); + } catch (UniqueConstraintViolationException $e) { + $query = $this->getQueryBuilder(); + $query->update('filecache_extended') + ->whereFileId($id) + ->hintShardKey('storage', $this->getNumericStorageId()) + ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { + return $query->expr()->orX( + $query->expr()->neq($key, $query->createNamedParameter($value)), + $query->expr()->isNull($key) + ); + }, array_keys($extensionValues), array_values($extensionValues)))); + + foreach ($extensionValues as $key => $value) { + $query->set($key, $query->createNamedParameter($value)); + } + + $query->executeStatement(); + } + } + + $path = $this->getPathById($id); + // path can still be null if the file doesn't exist + if ($path !== null) { + $event = new CacheEntryUpdatedEvent($this->storage, $path, $id, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } + } + + /** + * extract query parts and params array from data array + * + * @param array $data + * @return array + */ + protected function normalizeData(array $data): array { + $fields = [ + 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', + 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; + $extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; + + $doNotCopyStorageMTime = false; + if (array_key_exists('mtime', $data) && $data['mtime'] === null) { + // this horrific magic tells it to not copy storage_mtime to mtime + unset($data['mtime']); + $doNotCopyStorageMTime = true; + } + + $params = []; + $extensionParams = []; + foreach ($data as $name => $value) { + if (in_array($name, $fields)) { + if ($name === 'path') { + $params['path_hash'] = md5($value); + } elseif ($name === 'mimetype') { + $params['mimepart'] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/'))); + $value = $this->mimetypeLoader->getId($value); + } elseif ($name === 'storage_mtime') { + if (!$doNotCopyStorageMTime && !isset($data['mtime'])) { + $params['mtime'] = $value; + } + } elseif ($name === 'encrypted') { + if (isset($data['encryptedVersion'])) { + $value = $data['encryptedVersion']; + } else { + // Boolean to integer conversion + $value = $value ? 1 : 0; + } + } + $params[$name] = $value; + } + if (in_array($name, $extensionFields)) { + $extensionParams[$name] = $value; + } + } + return [$params, array_filter($extensionParams)]; + } + + /** + * get the file id for a file + * + * A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file + * + * File ids are easiest way for apps to store references to a file since unlike paths they are not affected by renames or sharing + * + * @param string $file + * @return int + */ + public function getId($file) { + // normalize file + $file = $this->normalize($file); + + $query = $this->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->wherePath($file); + + $result = $query->executeQuery(); + $id = $result->fetchOne(); + $result->closeCursor(); + + return $id === false ? -1 : (int)$id; + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + if ($file === '') { + return -1; + } else { + $parent = $this->getParentPath($file); + return (int)$this->getId($parent); + } + } + + private function getParentPath($path) { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + return $parent; + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getId($file) != -1; + } + + /** + * remove a file or folder from the cache + * + * when removing a folder from the cache all files and folders inside the folder will be removed as well + * + * @param string $file + */ + public function remove($file) { + $entry = $this->get($file); + + if ($entry instanceof ICacheEntry) { + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereFileId($entry->getId()); + $query->executeStatement(); + + $query = $this->getQueryBuilder(); + $query->delete('filecache_extended') + ->whereFileId($entry->getId()) + ->hintShardKey('storage', $this->getNumericStorageId()); + $query->executeStatement(); + + if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $this->removeChildren($entry); + } + + $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $entry->getPath(), $entry->getId(), $this->getNumericStorageId())); + } + } + + /** + * Remove all children of a folder + * + * @param ICacheEntry $entry the cache entry of the folder to remove the children of + * @throws \OC\DatabaseException + */ + private function removeChildren(ICacheEntry $entry) { + $parentIds = [$entry->getId()]; + $queue = [$entry->getId()]; + $deletedIds = []; + $deletedPaths = []; + + // we walk depth first through the file tree, removing all filecache_extended attributes while we walk + // and collecting all folder ids to later use to delete the filecache entries + while ($entryId = array_pop($queue)) { + $children = $this->getFolderContentsById($entryId); + $childIds = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getId(); + }, $children); + $childPaths = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getPath(); + }, $children); + + foreach ($childIds as $childId) { + $deletedIds[] = $childId; + } + + foreach ($childPaths as $childPath) { + $deletedPaths[] = $childPath; + } + + $query = $this->getQueryBuilder(); + $query->delete('filecache_extended') + ->where($query->expr()->in('fileid', $query->createParameter('childIds'))) + ->hintShardKey('storage', $this->getNumericStorageId()); + + foreach (array_chunk($childIds, 1000) as $childIdChunk) { + $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + + /** @var ICacheEntry[] $childFolders */ + $childFolders = []; + foreach ($children as $child) { + if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $childFolders[] = $child; + } + } + foreach ($childFolders as $folder) { + $parentIds[] = $folder->getId(); + $queue[] = $folder->getId(); + } + } + + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereParentInParameter('parentIds'); + + // Sorting before chunking allows the db to find the entries close to each + // other in the index + sort($parentIds, SORT_NUMERIC); + foreach (array_chunk($parentIds, 1000) as $parentIdChunk) { + $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + + foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { + $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( + $this->storage, + $filePath, + $fileId, + $this->getNumericStorageId() + ); + $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); + } + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->moveFromCache($this, $source, $target); + } + + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $path]; + } + + protected function hasEncryptionWrapper(): bool { + return $this->storage->instanceOfStorage(Encryption::class); + } + + /** + * Move a file or folder in the cache + * + * @param ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + * @throws \OC\DatabaseException + * @throws \Exception if the given storages have an invalid id + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache instanceof Cache) { + // normalize source and target + $sourcePath = $this->normalize($sourcePath); + $targetPath = $this->normalize($targetPath); + + $sourceData = $sourceCache->get($sourcePath); + if (!$sourceData) { + throw new \Exception('Source path not found in cache: ' . $sourcePath); + } + + $shardDefinition = $this->connection->getShardDefinition('filecache'); + if ( + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + ) { + $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); + return; + } + + $sourceId = $sourceData['fileid']; + $newParentId = $this->getParentId($targetPath); + + [$sourceStorageId, $sourcePath] = $sourceCache->getMoveInfo($sourcePath); + [$targetStorageId, $targetPath] = $this->getMoveInfo($targetPath); + + if (is_null($sourceStorageId) || $sourceStorageId === false) { + throw new \Exception('Invalid source storage id: ' . $sourceStorageId); + } + if (is_null($targetStorageId) || $targetStorageId === false) { + throw new \Exception('Invalid target storage id: ' . $targetStorageId); + } + + if ($sourceData['mimetype'] === 'httpd/unix-directory') { + //update all child entries + $sourceLength = mb_strlen($sourcePath); + + $childIds = $this->getChildIds($sourceStorageId, $sourcePath); + + $childChunks = array_chunk($childIds, 1000); + + $query = $this->getQueryBuilder(); + + $fun = $query->func(); + $newPathFunction = $fun->concat( + $query->createNamedParameter($targetPath), + $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash + ); + $query->update('filecache') + ->set('path_hash', $fun->md5($newPathFunction)) + ->set('path', $newPathFunction) + ->whereStorageId($sourceStorageId) + ->andWhere($query->expr()->in('fileid', $query->createParameter('files'))); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + // Retry transaction in case of RetryableException like deadlocks. + // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend + $retryLimit = 4; + for ($i = 1; $i <= $retryLimit; $i++) { + try { + $this->connection->beginTransaction(); + foreach ($childChunks as $chunk) { + $query->setParameter('files', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + break; + } catch (\OC\DatabaseException $e) { + $this->connection->rollBack(); + throw $e; + } catch (DbalException $e) { + $this->connection->rollBack(); + + if (!$e->isRetryable()) { + throw $e; + } + + // Simply throw if we already retried 4 times. + if ($i === $retryLimit) { + throw $e; + } + + // Sleep a bit to give some time to the other transaction to finish. + usleep(100 * 1000 * $i); + } + } + } else { + $this->connection->beginTransaction(); + } + + $query = $this->getQueryBuilder(); + $query->update('filecache') + ->set('path', $query->createNamedParameter($targetPath)) + ->set('path_hash', $query->createNamedParameter(md5($targetPath))) + ->set('name', $query->createNamedParameter(basename($targetPath))) + ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT)) + ->whereStorageId($sourceStorageId) + ->whereFileId($sourceId); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + $query->executeStatement(); + + $this->connection->commit(); + + if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) { + $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId())); + $event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } else { + $event = new CacheEntryUpdatedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } + } else { + $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath); + } + } + + private function getChildIds(int $storageId, string $path): array { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($path) . '/%'))); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()); + $query->executeStatement(); + + $query = $this->connection->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); + $query->executeStatement(); + } + + /** + * Get the scan status of a file + * + * - Cache::NOT_FOUND: File is not in the cache + * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known + * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned + * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned + * + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + // normalize file + $file = $this->normalize($file); + + $query = $this->getQueryBuilder(); + $query->select('size') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->wherePath($file); + + $result = $query->executeQuery(); + $size = $result->fetchOne(); + $result->closeCursor(); + + if ($size !== false) { + if ((int)$size === -1) { + return self::SHALLOW; + } else { + return self::COMPLETE; + } + } else { + if (isset($this->partial[$file])) { + return self::PARTIAL; + } else { + return self::NOT_FOUND; + } + } + } + + /** + * search for files matching $pattern + * + * @param string $pattern the search pattern using SQL search syntax (e.g. '%searchstring%') + * @return ICacheEntry[] an array of cache entries where the name matches the search pattern + */ + public function search($pattern) { + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', $pattern); + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); + } + + /** + * search for files by mimetype + * + * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') + * where it will search for all mimetypes in the group ('image/*') + * @return ICacheEntry[] an array of cache entries where the mimetype matches the search + */ + public function searchByMime($mimetype) { + if (!str_contains($mimetype, '/')) { + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'); + } else { + $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype); + } + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); + } + + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); + } + + /** + * Re-calculate the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $this->calculateFolderSize($path, $data); + + if ($path !== '') { + $parent = dirname($path); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + + if ($isBackgroundScan) { + $parentData = $this->get($parent); + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { + $this->correctFolderSize($parent, $parentData, $isBackgroundScan); + } + } else { + $this->correctFolderSize($parent); + } + } + } + + /** + * get the incomplete count that shares parent $folder + * + * @param int $fileId the file id of the folder + * @return int + */ + public function getIncompleteChildrenCount($fileId) { + if ($fileId > -1) { + $query = $this->getQueryBuilder(); + $query->select($query->func()->count()) + ->from('filecache') + ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $size = (int)$result->fetchOne(); + $result->closeCursor(); + + return $size; + } + return -1; + } + + /** + * calculate the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + return $this->calculateFolderSizeInner($path, $entry); + } + + + /** + * inner function because we can't add new params to the public function without breaking any child classes + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown + * @return int|float + */ + protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) { + $totalSize = 0; + if (is_null($entry) || !isset($entry['fileid'])) { + $entry = $this->get($path); + } + if (isset($entry['mimetype']) && $entry['mimetype'] === FileInfo::MIMETYPE_FOLDER) { + $id = $entry['fileid']; + + $query = $this->getQueryBuilder(); + $query->select('size', 'unencrypted_size') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereParent($id); + if ($ignoreUnknown) { + $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); + } + + $result = $query->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + + if ($rows) { + $sizes = array_map(function (array $row) { + return Util::numericToNumber($row['size']); + }, $rows); + $unencryptedOnlySizes = array_map(function (array $row) { + return Util::numericToNumber($row['unencrypted_size']); + }, $rows); + $unencryptedSizes = array_map(function (array $row) { + return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']); + }, $rows); + + $sum = array_sum($sizes); + $min = min($sizes); + + $unencryptedSum = array_sum($unencryptedSizes); + $unencryptedMin = min($unencryptedSizes); + $unencryptedMax = max($unencryptedOnlySizes); + + $sum = 0 + $sum; + $min = 0 + $min; + if ($min === -1) { + $totalSize = $min; + } else { + $totalSize = $sum; + } + if ($unencryptedMin === -1 || $min === -1) { + $unencryptedTotal = $unencryptedMin; + } else { + $unencryptedTotal = $unencryptedSum; + } + } else { + $totalSize = 0; + $unencryptedTotal = 0; + $unencryptedMax = 0; + } + + // only set unencrypted size for a folder if any child entries have it set, or the folder is empty + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + if ($shouldWriteUnEncryptedSize) { + // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes + if ($unencryptedMax === 0) { + $unencryptedTotal = 0; + } + + $this->update($id, [ + 'size' => $totalSize, + 'unencrypted_size' => $unencryptedTotal, + ]); + } else { + $this->update($id, [ + 'size' => $totalSize, + ]); + } + } + } + return $totalSize; + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + $query = $this->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()); + + $result = $query->executeQuery(); + $files = $result->fetchAll(\PDO::FETCH_COLUMN); + $result->closeCursor(); + + return array_map(function ($id) { + return (int)$id; + }, $files); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + $query = $this->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->orderBy('fileid', 'DESC') + ->setMaxResults(1); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + return $path === false ? false : (string)$path; + } + + /** + * get the path of a file on this storage by it's file id + * + * @param int $id the file id of the file or folder to search + * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache + */ + public function getPathById($id) { + $query = $this->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereFileId($id); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + if ($path === false) { + return null; + } + + return (string)$path; + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @return array first element holding the storage id, second the path + * @deprecated 17.0.0 use getPathById() instead + */ + public static function getById($id) { + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->select('path', 'storage') + ->from('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $numericId = $row['storage']; + $path = $row['path']; + } else { + return null; + } + + if ($id = Storage::getStorageId($numericId)) { + return [$id, $path]; + } else { + return null; + } + } + + /** + * normalize the given path + * + * @param string $path + * @return string + */ + public function normalize($path) { + return trim(\OC_Util::normalizeUnicode($path), '/'); + } + + /** + * Copy a file or folder in the cache + * + * @param ICache $sourceCache + * @param ICacheEntry $sourceEntry + * @param string $targetPath + * @return int fileId of copied entry + */ + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + if ($sourceEntry->getId() < 0) { + throw new \RuntimeException('Invalid source cache entry on copyFromCache'); + } + $data = $this->cacheEntryToArray($sourceEntry); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $data['encrypted'] = 0; + } + + $fileId = $this->put($targetPath, $data); + if ($fileId <= 0) { + throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' '); + } + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); + foreach ($folderContent as $subEntry) { + $subTargetPath = $targetPath . '/' . $subEntry->getName(); + $this->copyFromCache($sourceCache, $subEntry, $subTargetPath); + } + } + return $fileId; + } + + private function cacheEntryToArray(ICacheEntry $entry): array { + $data = [ + 'size' => $entry->getSize(), + 'mtime' => $entry->getMTime(), + 'storage_mtime' => $entry->getStorageMTime(), + 'mimetype' => $entry->getMimeType(), + 'mimepart' => $entry->getMimePart(), + 'etag' => $entry->getEtag(), + 'permissions' => $entry->getPermissions(), + 'encrypted' => $entry->isEncrypted(), + 'creation_time' => $entry->getCreationTime(), + 'upload_time' => $entry->getUploadTime(), + 'metadata_etag' => $entry->getMetadataEtag(), + ]; + if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) { + $data['permissions'] = $entry['scan_permissions']; + } + return $data; + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', $this->getNumericStorageId()); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($rawEntry->getStorageId() === $this->getNumericStorageId()) { + return $rawEntry; + } else { + return null; + } + } + + private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void { + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath()); + } else { + $fileIds = []; + } + $fileIds[] = $sourceEntry->getId(); + + $helper = $this->connection->getCrossShardMoveHelper(); + + $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId()); + $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId()); + + $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper(); + + $sourcePathLength = strlen($sourceEntry->getPath()); + foreach ($cacheItems as &$cacheItem) { + if ($cacheItem['path'] === $sourceEntry->getPath()) { + $cacheItem['path'] = $targetPath; + $cacheItem['parent'] = $this->getParentId($targetPath); + $cacheItem['name'] = basename($cacheItem['path']); + } else { + $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash + } + $cacheItem['path_hash'] = md5($cacheItem['path']); + $cacheItem['storage'] = $this->getNumericStorageId(); + if ($removeEncryptedFlag) { + $cacheItem['encrypted'] = 0; + } + } + + $targetConnection->beginTransaction(); + + try { + $helper->saveItems($targetConnection, 'filecache', $cacheItems); + $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems); + $helper->saveItems($targetConnection, 'files_metadata', $metadataItems); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + + $sourceConnection->beginTransaction(); + + try { + $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + } catch (\Exception $e) { + $targetConnection->rollback(); + $sourceConnection->rollBack(); + throw $e; + } + + try { + $sourceConnection->commit(); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + $targetConnection->commit(); + } +} diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php new file mode 100644 index 00000000000..61d2a2f9646 --- /dev/null +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -0,0 +1,61 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDependencies { + public function __construct( + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $metadataManager, + private DisplayNameCache $displayNameCache, + ) { + } + + public function getMimeTypeLoader(): IMimeTypeLoader { + return $this->mimeTypeLoader; + } + + public function getConnection(): IDBConnection { + return $this->connection; + } + + public function getEventDispatcher(): IEventDispatcher { + return $this->eventDispatcher; + } + + public function getQuerySearchHelper(): QuerySearchHelper { + return $this->querySearchHelper; + } + + public function getSystemConfig(): SystemConfig { + return $this->systemConfig; + } + + public function getLogger(): LoggerInterface { + return $this->logger; + } + + public function getDisplayNameCache(): DisplayNameCache { + return $this->displayNameCache; + } + + public function getMetadataManager(): IFilesMetadataManager { + return $this->metadataManager; + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php new file mode 100644 index 00000000000..ab5bae316f4 --- /dev/null +++ b/lib/private/Files/Cache/CacheEntry.php @@ -0,0 +1,132 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +/** + * meta data for a file or folder + */ +class CacheEntry implements ICacheEntry { + /** + * @var array + */ + private $data; + + public function __construct(array $data) { + $this->data = $data; + } + + public function offsetSet($offset, $value): void { + $this->data[$offset] = $value; + } + + public function offsetExists($offset): bool { + return isset($this->data[$offset]); + } + + public function offsetUnset($offset): void { + unset($this->data[$offset]); + } + + /** + * @return mixed + */ + #[\ReturnTypeWillChange] + public function offsetGet($offset) { + if (isset($this->data[$offset])) { + return $this->data[$offset]; + } else { + return null; + } + } + + public function getId() { + return (int)$this->data['fileid']; + } + + public function getStorageId() { + return $this->data['storage']; + } + + + public function getPath() { + return (string)$this->data['path']; + } + + + public function getName() { + return $this->data['name']; + } + + + public function getMimeType() { + return $this->data['mimetype']; + } + + + public function getMimePart() { + return $this->data['mimepart']; + } + + public function getSize() { + return $this->data['size']; + } + + public function getMTime() { + return $this->data['mtime']; + } + + public function getStorageMTime() { + return $this->data['storage_mtime']; + } + + public function getEtag() { + return $this->data['etag']; + } + + public function getPermissions() { + return $this->data['permissions']; + } + + public function isEncrypted() { + return isset($this->data['encrypted']) && $this->data['encrypted']; + } + + public function getMetadataEtag(): ?string { + return $this->data['metadata_etag'] ?? null; + } + + public function getCreationTime(): ?int { + return $this->data['creation_time'] ?? null; + } + + public function getUploadTime(): ?int { + return $this->data['upload_time'] ?? null; + } + + public function getParentId(): int { + return $this->data['parent']; + } + + public function getData() { + return $this->data; + } + + public function __clone() { + $this->data = array_merge([], $this->data); + } + + public function getUnencryptedSize(): int { + if ($this->data['encrypted'] && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return $this->data['size'] ?? 0; + } + } +} diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php new file mode 100644 index 00000000000..5492452273b --- /dev/null +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -0,0 +1,125 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\DB\QueryBuilder\ExtendedQueryBuilder; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; + +/** + * Query builder with commonly used helpers for filecache queries + */ +class CacheQueryBuilder extends ExtendedQueryBuilder { + private ?string $alias = null; + + public function __construct( + IQueryBuilder $queryBuilder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($queryBuilder); + } + + public function selectTagUsage(): self { + $this + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') + ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') + ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') + ->from('filecache', 'filecache') + ->leftJoin('filecache', 'systemtag_object_mapping', 'systemtagmap', $this->expr()->andX( + $this->expr()->eq('filecache.fileid', $this->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $this->expr()->eq('systemtagmap.objecttype', $this->createNamedParameter('files')) + )) + ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $this->expr()->andX( + $this->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'), + $this->expr()->eq('systemtag.visibility', $this->createNamedParameter(true)) + )) + ->groupBy('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable'); + + return $this; + } + + public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true) { + $name = $alias ?: 'filecache'; + $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', + 'storage_mtime', 'encrypted', "$name.etag", "$name.permissions", 'checksum', 'unencrypted_size') + ->from('filecache', $name); + + if ($joinExtendedCache) { + $this->addSelect('metadata_etag', 'creation_time', 'upload_time'); + $this->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); + } + + $this->alias = $name; + + return $this; + } + + public function whereStorageId(int $storageId) { + $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function whereFileId(int $fileId) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->eq("{$alias}fileid", $this->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function wherePath(string $path) { + $this->andWhere($this->expr()->eq('path_hash', $this->createNamedParameter(md5($path)))); + + return $this; + } + + public function whereParent(int $parent) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->eq("{$alias}parent", $this->createNamedParameter($parent, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function whereParentInParameter(string $parameter) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->in("{$alias}parent", $this->createParameter($parameter))); + + return $this; + } + + /** + * join metadata to current query builder and returns an helper + * + * @return IMetadataQuery + */ + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } +} diff --git a/lib/private/Files/Cache/FailedCache.php b/lib/private/Files/Cache/FailedCache.php new file mode 100644 index 00000000000..44c1016ca8e --- /dev/null +++ b/lib/private/Files/Cache/FailedCache.php @@ -0,0 +1,138 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OC\Files\Search\SearchComparison; +use OCP\Constants; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; + +/** + * Storage placeholder to represent a missing precondition, storage unavailable + */ +class FailedCache implements ICache { + /** @var bool whether to show the failed storage in the ui */ + private $visible; + + /** + * FailedCache constructor. + * + * @param bool $visible + */ + public function __construct($visible = true) { + $this->visible = $visible; + } + + + public function getNumericStorageId() { + return -1; + } + + public function get($file) { + if ($file === '') { + return new CacheEntry([ + 'fileid' => -1, + 'size' => 0, + 'mimetype' => 'httpd/unix-directory', + 'mimepart' => 'httpd', + 'permissions' => $this->visible ? Constants::PERMISSION_READ : 0, + 'mtime' => time() + ]); + } else { + return false; + } + } + + public function getFolderContents($folder) { + return []; + } + + public function getFolderContentsById($fileId) { + return []; + } + + public function put($file, array $data) { + } + + public function insert($file, array $data) { + } + + public function update($id, array $data) { + } + + public function getId($file) { + return -1; + } + + public function getParentId($file) { + return -1; + } + + public function inCache($file) { + return false; + } + + public function remove($file) { + } + + public function move($source, $target) { + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + } + + public function clear() { + } + + public function getStatus($file) { + return ICache::NOT_FOUND; + } + + public function search($pattern) { + return []; + } + + public function searchByMime($mimetype) { + return []; + } + + public function searchQuery(ISearchQuery $query) { + return []; + } + + public function getAll() { + return []; + } + + public function getIncomplete() { + return []; + } + + public function getPathById($id) { + return null; + } + + public function normalize($path) { + return $path; + } + + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + throw new \Exception('Invalid cache'); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', -1); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + return null; + } +} diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php new file mode 100644 index 00000000000..c3f3614f3ca --- /dev/null +++ b/lib/private/Files/Cache/FileAccess.php @@ -0,0 +1,222 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\SystemConfig; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IFileAccess; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Low level access to the file cache + */ +class FileAccess implements IFileAccess { + public function __construct( + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private FilesMetadataManager $metadataManager, + private IMimeTypeLoader $mimeTypeLoader, + ) { + } + + private function getQuery(): CacheQueryBuilder { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getByFileIdInStorage(int $fileId, int $storageId): ?CacheEntry { + $items = array_values($this->getByFileIdsInStorage([$fileId], $storageId)); + return $items[0] ?? null; + } + + public function getByPathInStorage(string $path, int $storageId): ?CacheEntry { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->eq('filecache.path_hash', $query->createNamedParameter(md5($path)))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $row = $query->executeQuery()->fetch(); + return $row ? Cache::cacheEntryFromData($row, $this->mimeTypeLoader) : null; + } + + public function getByFileId(int $fileId): ?CacheEntry { + $items = array_values($this->getByFileIds([$fileId])); + return $items[0] ?? null; + } + + /** + * @param array[] $rows + * @return array<int, CacheEntry> + */ + private function rowsToEntries(array $rows): array { + $result = []; + foreach ($rows as $row) { + $entry = Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + $result[$entry->getId()] = $entry; + } + return $result; + } + + /** + * @param int[] $fileIds + * @return array<int, CacheEntry> + */ + public function getByFileIds(array $fileIds): array { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + /** + * @param int[] $fileIds + * @param int $storageId + * @return array<int, CacheEntry> + */ + public function getByFileIdsInStorage(array $fileIds, int $storageId): array { + $fileIds = array_values($fileIds); + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } +} diff --git a/lib/private/Files/Cache/HomeCache.php b/lib/private/Files/Cache/HomeCache.php new file mode 100644 index 00000000000..248cdc818f0 --- /dev/null +++ b/lib/private/Files/Cache/HomeCache.php @@ -0,0 +1,47 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +class HomeCache extends Cache { + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + if ($path !== '/' and $path !== '' and $path !== 'files' and $path !== 'files_trashbin' and $path !== 'files_versions') { + return parent::calculateFolderSize($path, $entry); + } elseif ($path === '' or $path === '/') { + // since the size of / isn't used (the size of /files is used instead) there is no use in calculating it + return 0; + } else { + return $this->calculateFolderSizeInner($path, $entry, true); + } + } + + /** + * @param string $file + * @return ICacheEntry + */ + public function get($file) { + $data = parent::get($file); + if ($file === '' or $file === '/') { + // only the size of the "files" dir counts + $filesData = parent::get('files'); + + if (isset($filesData['size'])) { + $data['size'] = $filesData['size']; + } + } + return $data; + } +} diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php new file mode 100644 index 00000000000..d4ac8a7c8e3 --- /dev/null +++ b/lib/private/Files/Cache/HomePropagator.php @@ -0,0 +1,37 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\IDBConnection; + +class HomePropagator extends Propagator { + private $ignoredBaseFolders; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection) { + parent::__construct($storage, $connection); + $this->ignoredBaseFolders = ['files_encryption']; + } + + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + [$baseFolder] = explode('/', $internalPath, 2); + if (in_array($baseFolder, $this->ignoredBaseFolders)) { + return []; + } else { + parent::propagateChange($internalPath, $time, $sizeDifference); + } + } +} diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php new file mode 100644 index 00000000000..3f4f70b865b --- /dev/null +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -0,0 +1,32 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +class LocalRootScanner extends Scanner { + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + if ($this->shouldScanPath($file)) { + return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); + } else { + return null; + } + } + + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + if ($this->shouldScanPath($path)) { + return parent::scan($path, $recursive, $reuse, $lock); + } else { + return null; + } + } + + private function shouldScanPath(string $path): bool { + $path = trim($path, '/'); + return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); + } +} diff --git a/lib/private/Files/Cache/MoveFromCacheTrait.php b/lib/private/Files/Cache/MoveFromCacheTrait.php new file mode 100644 index 00000000000..db35c6bb7f8 --- /dev/null +++ b/lib/private/Files/Cache/MoveFromCacheTrait.php @@ -0,0 +1,44 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; + +/** + * Fallback implementation for moveFromCache + */ +trait MoveFromCacheTrait { + /** + * store meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + abstract public function put($file, array $data); + + abstract public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int; + + /** + * Move a file or folder in the cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $sourceEntry = $sourceCache->get($sourcePath); + + $this->copyFromCache($sourceCache, $sourceEntry, $targetPath); + + $sourceCache->remove($sourcePath); + } +} diff --git a/lib/private/Files/Cache/NullWatcher.php b/lib/private/Files/Cache/NullWatcher.php new file mode 100644 index 00000000000..e3659214849 --- /dev/null +++ b/lib/private/Files/Cache/NullWatcher.php @@ -0,0 +1,38 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +class NullWatcher extends Watcher { + private $policy; + + public function __construct() { + } + + public function setPolicy($policy) { + $this->policy = $policy; + } + + public function getPolicy() { + return $this->policy; + } + + public function checkUpdate($path, $cachedEntry = null) { + return false; + } + + public function update($path, $cachedData) { + } + + public function needsUpdate($path, $cachedData) { + return false; + } + + public function cleanFolder($path) { + } +} diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php new file mode 100644 index 00000000000..a6ba87896f4 --- /dev/null +++ b/lib/private/Files/Cache/Propagator.php @@ -0,0 +1,223 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OC\DB\Exceptions\DbalException; +use OC\Files\Storage\Wrapper\Encryption; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; +use OCP\Server; +use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; + +/** + * Propagate etags and mtimes within the storage + */ +class Propagator implements IPropagator { + public const MAX_RETRIES = 3; + private $inBatch = false; + + private $batch = []; + + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @var IDBConnection + */ + private $connection; + + /** + * @var array + */ + private $ignore = []; + + private ClockInterface $clock; + + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { + $this->storage = $storage; + $this->connection = $connection; + $this->ignore = $ignore; + $this->clock = Server::get(ClockInterface::class); + } + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + // Do not propagate changes in ignored paths + foreach ($this->ignore as $ignore) { + if (str_starts_with($internalPath, $ignore)) { + return; + } + } + + $time = min((int)$time, $this->clock->now()->getTimestamp()); + + $storageId = $this->storage->getStorageCache()->getNumericId(); + + $parents = $this->getParents($internalPath); + + if ($this->inBatch) { + foreach ($parents as $parent) { + $this->addToBatch($parent, $time, $sizeDifference); + } + return; + } + + $parentHashes = array_map('md5', $parents); + $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag + + $builder = $this->connection->getQueryBuilder(); + $hashParams = array_map(function ($hash) use ($builder) { + return $builder->expr()->literal($hash); + }, $parentHashes); + + $builder->update('filecache') + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT))) + ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($builder->expr()->in('path_hash', $hashParams)); + if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); + } + + if ($sizeDifference !== 0) { + $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $newSize = $builder->func()->greatest( + $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + ); + + // Only update if row had a previously calculated size + $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); + + if ($this->storage->instanceOfStorage(Encryption::class)) { + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); + $newUnencryptedSize = $builder->func()->greatest( + $builder->func()->add( + $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), + $builder->createNamedParameter($sizeDifference) + ), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + ); + + // Only update if row had a previously calculated size + $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); + } + } + + for ($i = 0; $i < self::MAX_RETRIES; $i++) { + try { + $builder->executeStatement(); + break; + } catch (DbalException $e) { + if (!$e->isRetryable()) { + throw $e; + } + + /** @var LoggerInterface $loggerInterface */ + $loggerInterface = \OCP\Server::get(LoggerInterface::class); + $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); + } + } + } + + protected function getParents($path) { + $parts = explode('/', $path); + $parent = ''; + $parents = []; + foreach ($parts as $part) { + $parents[] = $parent; + $parent = trim($parent . '/' . $part, '/'); + } + return $parents; + } + + /** + * Mark the beginning of a propagation batch + * + * Note that not all cache setups support propagation in which case this will be a noop + * + * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent + * before the batch is committed. + */ + public function beginBatch() { + $this->inBatch = true; + } + + private function addToBatch($internalPath, $time, $sizeDifference) { + if (!isset($this->batch[$internalPath])) { + $this->batch[$internalPath] = [ + 'hash' => md5($internalPath), + 'time' => $time, + 'size' => $sizeDifference, + ]; + } else { + $this->batch[$internalPath]['size'] += $sizeDifference; + if ($time > $this->batch[$internalPath]['time']) { + $this->batch[$internalPath]['time'] = $time; + } + } + } + + /** + * Commit the active propagation batch + */ + public function commitBatch() { + if (!$this->inBatch) { + throw new \BadMethodCallException('Not in batch'); + } + $this->inBatch = false; + + $this->connection->beginTransaction(); + + $query = $this->connection->getQueryBuilder(); + $storageId = (int)$this->storage->getStorageCache()->getNumericId(); + + $query->update('filecache') + ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) + ->set('etag', $query->expr()->literal(uniqid())) + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); + + $sizeQuery = $this->connection->getQueryBuilder(); + $sizeQuery->update('filecache') + ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) + ->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash'))) + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); + + foreach ($this->batch as $item) { + $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); + $query->setParameter('hash', $item['hash']); + + $query->executeStatement(); + + if ($item['size']) { + $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); + $sizeQuery->setParameter('hash', $item['hash']); + + $sizeQuery->executeStatement(); + } + } + + $this->batch = []; + + $this->connection->commit(); + } +} diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php new file mode 100644 index 00000000000..3ddcf1ca4e6 --- /dev/null +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -0,0 +1,240 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\QueryOptimizer\QueryOptimizer; +use OC\Files\Search\SearchBinaryOperator; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; +use Psr\Log\LoggerInterface; + +class QuerySearchHelper { + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + protected function getQueryBuilder() { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->filesMetadataManager, + ); + } + + /** + * @param CacheQueryBuilder $query + * @param ISearchQuery $searchQuery + * @param array $caches + * @param IMetadataQuery|null $metadataQuery + */ + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null, + ): void { + $storageFilters = array_values(array_map(function (ICache $cache) { + return $cache->getQueryFilterForStorage(); + }, $caches)); + $storageFilter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters); + $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); + $this->queryOptimizer->processOperator($filter); + + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); + if ($searchExpr) { + $query->andWhere($searchExpr); + } + + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); + + if ($searchQuery->getLimit()) { + $query->setMaxResults($searchQuery->getLimit()); + } + if ($searchQuery->getOffset()) { + $query->setFirstResult($searchQuery->getOffset()); + } + } + + + /** + * @return array<array-key, array{id: int, name: string, visibility: int, editable: int, ref_file_id: int, number_files: int}> + */ + public function findUsedTagsInCaches(ISearchQuery $searchQuery, array $caches): array { + $query = $this->getQueryBuilder(); + $query->selectTagUsage(); + + $this->applySearchConstraints($query, $searchQuery, $caches); + + $result = $query->executeQuery(); + $tags = $result->fetchAll(); + $result->closeCursor(); + return $tags; + } + + protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user): void { + $query->leftJoin('file', 'systemtag_object_mapping', 'systemtagmap', $query->expr()->andX( + $query->expr()->eq('file.fileid', $query->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')) + )); + $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid')); + if (!$this->groupManager->isAdmin($user->getUID())) { + $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true))); + } + $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on); + } + + protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void { + $query + ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid')) + ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( + $query->expr()->eq('tagmap.categoryid', 'tag.id'), + $query->expr()->eq('tag.type', $query->createNamedParameter('files')), + $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID())) + )); + } + + + protected function equipQueryForShares(CacheQueryBuilder $query): void { + $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); + } + + /** + * Perform a file system search in multiple caches + * + * the results will be grouped by the same array keys as the $caches argument to allow + * post-processing based on which cache the result came from + * + * @template T of array-key + * @param ISearchQuery $searchQuery + * @param array<T, ICache> $caches + * @return array<T, ICacheEntry[]> + */ + public function searchInCaches(ISearchQuery $searchQuery, array $caches): array { + // search in multiple caches at once by creating one query in the following format + // SELECT ... FROM oc_filecache WHERE + // <filter expressions from the search query> + // AND ( + // <filter expression for storage1> OR + // <filter expression for storage2> OR + // ... + // ); + // + // This gives us all the files matching the search query from all caches + // + // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id) + // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result. + + $builder = $this->getQueryBuilder(); + + $query = $builder->selectFileCache('file', false); + + $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + + if (in_array('systemtag', $requestedFields)) { + $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); + } + if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { + $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); + } + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); + } + + $metadataQuery = $query->selectMetadata(); + + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); + + $result = $query->executeQuery(); + $files = $result->fetchAll(); + + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return Cache::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); + + $result->closeCursor(); + + // loop through all caches for each result to see if the result matches that storage + // results are grouped by the same array keys as the caches argument to allow the caller to distinguish the source of the results + $results = array_fill_keys(array_keys($caches), []); + foreach ($rawEntries as $rawEntry) { + foreach ($caches as $cacheKey => $cache) { + $entry = $cache->getCacheEntryFromSearchResult($rawEntry); + if ($entry) { + $results[$cacheKey][] = $entry; + } + } + } + return $results; + } + + protected function requireUser(ISearchQuery $searchQuery): IUser { + $user = $searchQuery->getUser(); + if ($user === null) { + throw new \InvalidArgumentException('This search operation requires the user to be set in the query'); + } + return $user; + } + + /** + * @return list{0?: array<array-key, ICache>, 1?: array<array-key, IMountPoint>} + */ + public function getCachesAndMountPointsForSearch(IRootFolder $root, string $path, bool $limitToHome = false): array { + $rootLength = strlen($path); + $mount = $root->getMount($path); + $storage = $mount->getStorage(); + if ($storage === null) { + return []; + } + $internalPath = $mount->getInternalPath($path); + + if ($internalPath !== '') { + // a temporary CacheJail is used to handle filtering down the results to within this folder + /** @var ICache[] $caches */ + $caches = ['' => new CacheJail($storage->getCache(''), $internalPath)]; + } else { + /** @var ICache[] $caches */ + $caches = ['' => $storage->getCache('')]; + } + /** @var IMountPoint[] $mountByMountPoint */ + $mountByMountPoint = ['' => $mount]; + + if (!$limitToHome) { + $mounts = $root->getMountsIn($path); + foreach ($mounts as $mount) { + $storage = $mount->getStorage(); + if ($storage) { + $relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/'); + $caches[$relativeMountPoint] = $storage->getCache(''); + $mountByMountPoint[$relativeMountPoint] = $mount; + } + } + } + + return [$caches, $mountByMountPoint]; + } +} diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php new file mode 100644 index 00000000000..b067f70b8cb --- /dev/null +++ b/lib/private/Files/Cache/Scanner.php @@ -0,0 +1,623 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OC\Hooks\BasicEmitter; +use OC\SystemConfig; +use OCP\Files\Cache\IScanner; +use OCP\Files\ForbiddenException; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; +use OCP\Lock\ILockingProvider; +use Psr\Log\LoggerInterface; + +/** + * Class Scanner + * + * Hooks available in scope \OC\Files\Cache\Scanner: + * - scanFile(string $path, string $storageId) + * - scanFolder(string $path, string $storageId) + * - postScanFile(string $path, string $storageId) + * - postScanFolder(string $path, string $storageId) + * + * @package OC\Files\Cache + */ +class Scanner extends BasicEmitter implements IScanner { + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var string $storageId + */ + protected $storageId; + + /** + * @var \OC\Files\Cache\Cache $cache + */ + protected $cache; + + /** + * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache + */ + protected $cacheActive; + + /** + * @var bool $useTransactions whether to use transactions + */ + protected $useTransactions = true; + + /** + * @var \OCP\Lock\ILockingProvider + */ + protected $lockingProvider; + + protected IDBConnection $connection; + + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->storageId = $this->storage->getId(); + $this->cache = $storage->getCache(); + /** @var SystemConfig $config */ + $config = \OC::$server->get(SystemConfig::class); + $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); + $this->lockingProvider = \OC::$server->get(ILockingProvider::class); + $this->connection = \OC::$server->get(IDBConnection::class); + } + + /** + * Whether to wrap the scanning of a folder in a database transaction + * On default transactions are used + * + * @param bool $useTransactions + */ + public function setUseTransactions($useTransactions): void { + $this->useTransactions = $useTransactions; + } + + /** + * get all the metadata of a file or folder + * * + * + * @param string $path + * @return array|null an array of metadata of the file + */ + protected function getData($path) { + $data = $this->storage->getMetaData($path); + if (is_null($data)) { + \OC::$server->get(LoggerInterface::class)->debug("!!! Path '$path' is not accessible or present !!!", ['app' => 'core']); + } + return $data; + } + + /** + * scan a single file and store it in the cache + * + * @param string $file + * @param int $reuseExisting + * @param int $parentId + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @param array|null $data the metadata for the file, as returned by the storage + * @return array|null an array of metadata of the scanned file + * @throws \OCP\Lock\LockedException + */ + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + if ($file !== '') { + try { + $this->storage->verifyPath(dirname($file), basename($file)); + } catch (\Exception $e) { + return null; + } + } + + // only proceed if $file is not a partial file, blacklist is handled by the storage + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + return null; + } + + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } + + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; + } + + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } + + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; + } + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; + } + } + + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); + } + + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; + } + } else { + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; + } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); + } + + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; + } + + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + return $data; + } + + protected function removeFromCache($path) { + \OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]); + $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]); + if ($this->cacheActive) { + $this->cache->remove($path); + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + * @return int the id of the added file + */ + protected function addToCache($path, $data, $fileId = -1) { + if (isset($data['scan_permissions'])) { + $data['permissions'] = $data['scan_permissions']; + } + \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data, $fileId]); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + return $fileId; + } else { + return $this->cache->insert($path, $data); + } + } else { + return -1; + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + */ + protected function updateCache($path, $data, $fileId = -1) { + \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); + $this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + } else { + $this->cache->put($path, $data); + } + } + } + + /** + * scan a folder and all it's children + * + * @param string $path + * @param bool $recursive + * @param int $reuse + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @return array|null an array of the meta data of the scanned file or folder + */ + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + + try { + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); + $data['size'] = $size; + } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; + } finally { + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + } + } + return $data; + } + + /** + * Compares $array1 against $array2 and returns all the values in $array1 that are not in $array2 + * Note this is a one-way check - i.e. we don't care about things that are in $array2 that aren't in $array1 + * + * Supports multi-dimensional arrays + * Also checks keys/indexes + * Comparisons are strict just like array_diff_assoc + * Order of keys/values does not matter + * + * @param array $array1 + * @param array $array2 + * @return array with the differences between $array1 and $array1 + * @throws \InvalidArgumentException if $array1 isn't an actual array + * + */ + protected function array_diff_assoc_multi(array $array1, array $array2) { + + $result = []; + + foreach ($array1 as $key => $value) { + + // if $array2 doesn't have the same key, that's a result + if (!array_key_exists($key, $array2)) { + $result[$key] = $value; + continue; + } + + // if $array2's value for the same key is different, that's a result + if ($array2[$key] !== $value && !is_array($value)) { + $result[$key] = $value; + continue; + } + + if (is_array($value)) { + $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); + if (!empty($nestedDiff)) { + $result[$key] = $nestedDiff; + continue; + } + } + } + return $result; + } + + /** + * Get the children currently in the cache + * + * @param int $folderId + * @return array<string, \OCP\Files\Cache\ICacheEntry> + */ + protected function getExistingChildren($folderId): array { + $existingChildren = []; + $children = $this->cache->getFolderContentsById($folderId); + foreach ($children as $child) { + $existingChildren[$child['name']] = $child; + } + return $existingChildren; + } + + /** + * scan all the files and folders in a folder + * + * @param string $path + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + * @param int $reuse a combination of self::REUSE_* + * @param int $folderId id for the folder to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @param int|float $oldSize the size of the folder before (re)scanning the children + * @return int|float the size of the scanned folder or -1 if the size is unknown at this stage + */ + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]); + $size = 0; + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + + foreach ($childQueue as $child => [$childId, $childSize]) { + // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed + $childEtagChanged = false; + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize, $childEtagChanged); + $etagChanged |= $childEtagChanged; + + if ($childSize === -1) { + $size = -1; + } elseif ($size !== -1) { + $size += $childSize; + } + } + + // for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size + // to make sure we also updated the unencrypted-size where applicable + if ($this->storage->instanceOfStorage(Encryption::class)) { + $this->cache->calculateFolderSize($path); + } else { + if ($this->cacheActive) { + $updatedData = []; + if ($oldSize !== $size) { + $updatedData['size'] = $size; + } + if ($etagChanged) { + $updatedData['etag'] = uniqid(); + } + if ($updatedData) { + $this->cache->update($folderId, $updatedData); + } + } + } + $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); + return $size; + } + + /** + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + */ + private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array { + // we put this in it's own function so it cleans up the memory before we start recursing + $existingChildren = $this->getExistingChildren($folderId); + $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); + + if (count($existingChildren) === 0 && count($newChildren) === 0) { + // no need to do a transaction + return []; + } + + if ($this->useTransactions) { + $this->connection->beginTransaction(); + } + + $exceptionOccurred = false; + $childQueue = []; + $newChildNames = []; + foreach ($newChildren as $fileMeta) { + $permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions']; + if ($permissions === 0) { + continue; + } + $originalFile = $fileMeta['name']; + $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/'); + if (trim($originalFile, '/') !== $file) { + // encoding mismatch, might require compatibility wrapper + \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".', ['app' => 'core']); + $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]); + // skip this entry + continue; + } + + $newChildNames[] = $file; + $child = $path ? $path . '/' . $file : $file; + try { + $existingData = $existingChildren[$file] ?? false; + $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta); + if ($data) { + if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) { + $childQueue[$child] = [$data['fileid'], $data['size']]; + } elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) { + // only recurse into folders which aren't fully scanned + $childQueue[$child] = [$data['fileid'], $data['size']]; + } elseif ($data['size'] === -1) { + $size = -1; + } elseif ($size !== -1) { + $size += $data['size']; + } + + if (isset($data['etag_changed']) && $data['etag_changed']) { + $etagChanged = true; + } + } + } catch (Exception $ex) { + // might happen if inserting duplicate while a scanning + // process is running in parallel + // log and ignore + if ($this->useTransactions) { + $this->connection->rollback(); + $this->connection->beginTransaction(); + } + \OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [ + 'app' => 'core', + 'exception' => $ex, + ]); + $exceptionOccurred = true; + } catch (\OCP\Lock\LockedException $e) { + if ($this->useTransactions) { + $this->connection->rollback(); + } + throw $e; + } + } + $removedChildren = \array_diff(array_keys($existingChildren), $newChildNames); + foreach ($removedChildren as $childName) { + $child = $path ? $path . '/' . $childName : $childName; + $this->removeFromCache($child); + } + if ($this->useTransactions) { + $this->connection->commit(); + } + if ($exceptionOccurred) { + // It might happen that the parallel scan process has already + // inserted mimetypes but those weren't available yet inside the transaction + // To make sure to have the updated mime types in such cases, + // we reload them here + \OC::$server->getMimeTypeLoader()->reset(); + } + return $childQueue; + } + + /** + * check if the file should be ignored when scanning + * NOTE: files with a '.part' extension are ignored as well! + * prevents unfinished put requests to be scanned + * + * @param string $file + * @return boolean + */ + public static function isPartialFile($file) { + if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { + return true; + } + if (str_contains($file, '.part/')) { + return true; + } + + return false; + } + + /** + * walk over any folders that are not fully scanned yet and scan them + */ + public function backgroundScan() { + if ($this->storage->instanceOfStorage(Jail::class)) { + // for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage + // this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient). + // + // Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will + // have to be scanned at some point anyway. + $unJailedScanner = $this->storage->getUnjailedStorage()->getScanner(); + $unJailedScanner->backgroundScan(); + } else { + if (!$this->cache->inCache('')) { + // if the storage isn't in the cache yet, just scan the root completely + $this->runBackgroundScanJob(function () { + $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); + }, ''); + } else { + $lastPath = null; + // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck) + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function () use ($path) { + $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } + } + } + } + + protected function runBackgroundScanJob(callable $callback, $path) { + try { + $callback(); + \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]); + if ($this->cacheActive && $this->cache instanceof Cache) { + $this->cache->correctFolderSize($path, null, true); + } + } catch (\OCP\Files\StorageInvalidException $e) { + // skip unavailable storages + } catch (\OCP\Files\StorageNotAvailableException $e) { + // skip unavailable storages + } catch (\OCP\Files\ForbiddenException $e) { + // skip forbidden storages + } catch (\OCP\Lock\LockedException $e) { + // skip unavailable storages + } + } + + /** + * Set whether the cache is affected by scan operations + * + * @param boolean $active The active state of the cache + */ + public function setCacheActive($active) { + $this->cacheActive = $active; + } +} diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php new file mode 100644 index 00000000000..e1d3c42a8a2 --- /dev/null +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -0,0 +1,355 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; + +/** + * Tools for transforming search queries into database queries + * + * @psalm-import-type ParamSingleValue from ISearchComparison + * @psalm-import-type ParamValue from ISearchComparison + */ +class SearchBuilder { + /** @var array<string, string> */ + protected static $searchOperatorMap = [ + ISearchComparison::COMPARE_LIKE => 'iLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like', + ISearchComparison::COMPARE_EQUAL => 'eq', + ISearchComparison::COMPARE_GREATER_THAN => 'gt', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', + ISearchComparison::COMPARE_LESS_THAN => 'lt', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte', + ISearchComparison::COMPARE_DEFINED => 'isNotNull', + ISearchComparison::COMPARE_IN => 'in', + ]; + + /** @var array<string, string> */ + protected static $searchOperatorNegativeMap = [ + ISearchComparison::COMPARE_LIKE => 'notLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike', + ISearchComparison::COMPARE_EQUAL => 'neq', + ISearchComparison::COMPARE_GREATER_THAN => 'lte', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', + ISearchComparison::COMPARE_LESS_THAN => 'gte', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt', + ISearchComparison::COMPARE_DEFINED => 'isNull', + ISearchComparison::COMPARE_IN => 'notIn', + ]; + + /** @var array<string, string> */ + protected static $fieldTypes = [ + 'mimetype' => 'string', + 'mtime' => 'integer', + 'name' => 'string', + 'path' => 'string', + 'size' => 'integer', + 'tagname' => 'string', + 'systemtag' => 'string', + 'favorite' => 'boolean', + 'fileid' => 'integer', + 'storage' => 'integer', + 'share_with' => 'string', + 'share_type' => 'integer', + 'owner' => 'string', + ]; + + /** @var array<string, int|string> */ + protected static $paramTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR, + 'integer' => IQueryBuilder::PARAM_INT, + 'boolean' => IQueryBuilder::PARAM_BOOL, + ]; + + /** @var array<string, int> */ + protected static $paramArrayTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR_ARRAY, + 'integer' => IQueryBuilder::PARAM_INT_ARRAY, + 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, + ]; + + public const TAG_FAVORITE = '_$!<Favorite>!$_'; + + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @return string[] + */ + public function extractRequestedFields(ISearchOperator $operator): array { + if ($operator instanceof ISearchBinaryOperator) { + return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { + return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); + }, []); + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { + return [$operator->getField()]; + } + return []; + } + + /** + * @param IQueryBuilder $builder + * @param ISearchOperator[] $operators + */ + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null, + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); + }, $operators)); + } + + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null, + ) { + $expr = $builder->expr(); + + if ($operator instanceof ISearchBinaryOperator) { + if (count($operator->getArguments()) === 0) { + return null; + } + + switch ($operator->getType()) { + case ISearchBinaryOperator::OPERATOR_NOT: + $negativeOperator = $operator->getArguments()[0]; + if ($negativeOperator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); + } + // no break + case ISearchBinaryOperator::OPERATOR_AND: + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + case ISearchBinaryOperator::OPERATOR_OR: + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + default: + throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); + } + } elseif ($operator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); + } + } + + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null, + ) { + if ($comparison->getExtra()) { + [$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison); + } + + if (isset($operatorMap[$type])) { + $queryOperator = $operatorMap[$type]; + return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType)); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); + } + } + + /** + * @param ISearchComparison $operator + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValue(ISearchComparison $operator): array { + $this->validateComparison($operator); + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + $pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true); + return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash); + } + + /** + * @param string $field + * @param ParamValue $value + * @param string $type + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array { + $paramType = self::$fieldTypes[$field]; + if ($type === ISearchComparison::COMPARE_IN) { + $resultField = $field; + $values = []; + foreach ($value as $arrayValue) { + /** @var ParamSingleValue $arrayValue */ + [$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash); + $resultField = $arrayField; + $values[] = $arrayValue; + } + return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType]; + } + if ($field === 'mimetype') { + $value = (string)$value; + if ($type === ISearchComparison::COMPARE_EQUAL) { + $value = $this->mimetypeLoader->getId($value); + } elseif ($type === ISearchComparison::COMPARE_LIKE) { + // transform "mimetype='foo/%'" to "mimepart='foo'" + if (preg_match('|(.+)/%|', $value, $matches)) { + $field = 'mimepart'; + $value = $this->mimetypeLoader->getId($matches[1]); + $type = ISearchComparison::COMPARE_EQUAL; + } elseif (str_contains($value, '%')) { + throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); + } else { + $field = 'mimetype'; + $value = $this->mimetypeLoader->getId($value); + $type = ISearchComparison::COMPARE_EQUAL; + } + } + } elseif ($field === 'favorite') { + $field = 'tag.category'; + $value = self::TAG_FAVORITE; + $paramType = 'string'; + } elseif ($field === 'name') { + $field = 'file.name'; + } elseif ($field === 'tagname') { + $field = 'tag.category'; + } elseif ($field === 'systemtag') { + $field = 'systemtag.name'; + } elseif ($field === 'fileid') { + $field = 'file.fileid'; + } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { + $field = 'path_hash'; + $value = md5((string)$value); + } elseif ($field === 'owner') { + $field = 'uid_owner'; + } + return [$field, $value, $type, $paramType]; + } + + private function validateComparison(ISearchComparison $operator) { + $comparisons = [ + 'mimetype' => ['eq', 'like', 'in'], + 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'name' => ['eq', 'like', 'clike', 'in'], + 'path' => ['eq', 'like', 'clike', 'in'], + 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'tagname' => ['eq', 'like'], + 'systemtag' => ['eq', 'like'], + 'favorite' => ['eq'], + 'fileid' => ['eq', 'in'], + 'storage' => ['eq', 'in'], + 'share_with' => ['eq'], + 'share_type' => ['eq'], + 'owner' => ['eq'], + ]; + + if (!isset(self::$fieldTypes[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); + } + $type = self::$fieldTypes[$operator->getField()]; + if ($operator->getType() === ISearchComparison::COMPARE_IN) { + if (!is_array($operator->getValue())) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + foreach ($operator->getValue() as $arrayValue) { + if (gettype($arrayValue) !== $type) { + throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField()); + } + } + } else { + if (gettype($operator->getValue()) !== $type) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + } + if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); + } + } + + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + + switch ($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type, $paramType]; + } + + private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) { + if ($value instanceof \DateTime) { + $value = $value->getTimestamp(); + } + if (is_array($value)) { + $type = self::$paramArrayTypeMap[$paramType]; + } else { + $type = self::$paramTypeMap[$paramType]; + } + return $builder->createNamedParameter($value, $type); + } + + /** + * @param IQueryBuilder $query + * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery + */ + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { + foreach ($orders as $order) { + $field = $order->getField(); + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; + + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } + $query->addOrderBy($field, $order->getDirection()); + } + } +} diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php new file mode 100644 index 00000000000..1a3bda58e6a --- /dev/null +++ b/lib/private/Files/Cache/Storage.php @@ -0,0 +1,235 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Storage\IStorage; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Handle the mapping between the string and numeric storage ids + * + * Each storage has 2 different ids + * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') + * and a numeric storage id which is referenced in the file cache + * + * A mapping between the two storage ids is stored in the database and accessible through this class + * + * @package OC\Files\Cache + */ +class Storage { + /** @var StorageGlobal|null */ + private static $globalCache = null; + private $storageId; + private $numericId; + + /** + * @return StorageGlobal + */ + public static function getGlobalCache() { + if (is_null(self::$globalCache)) { + self::$globalCache = new StorageGlobal(\OC::$server->getDatabaseConnection()); + } + return self::$globalCache; + } + + /** + * @param \OC\Files\Storage\Storage|string $storage + * @param bool $isAvailable + * @throws \RuntimeException + */ + public function __construct($storage, $isAvailable, IDBConnection $connection) { + if ($storage instanceof IStorage) { + $this->storageId = $storage->getId(); + } else { + $this->storageId = $storage; + } + $this->storageId = self::adjustStorageId($this->storageId); + + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = (int)$row['numeric_id']; + } else { + $available = $isAvailable ? 1 : 0; + if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { + $this->numericId = $connection->lastInsertId('*PREFIX*storages'); + } else { + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = (int)$row['numeric_id']; + } else { + throw new \RuntimeException('Storage could neither be inserted nor be selected from the database: ' . $this->storageId); + } + } + } + } + + /** + * @param string $storageId + * @return array + */ + public static function getStorageById($storageId) { + return self::getGlobalCache()->getStorageInfo($storageId); + } + + /** + * Adjusts the storage id to use md5 if too long + * @param string $storageId storage id + * @return string unchanged $storageId if its length is less than 64 characters, + * else returns the md5 of $storageId + */ + public static function adjustStorageId($storageId) { + if (strlen($storageId) > 64) { + return md5($storageId); + } + return $storageId; + } + + /** + * Get the numeric id for the storage + * + * @return int + */ + public function getNumericId() { + return $this->numericId; + } + + /** + * Get the string id for the storage + * + * @param int $numericId + * @return string|null either the storage id string or null if the numeric id is not known + */ + public static function getStorageId(int $numericId): ?string { + $storage = self::getGlobalCache()->getStorageInfoByNumericId($numericId); + return $storage['id'] ?? null; + } + + /** + * Get the numeric of the storage with the provided string id + * + * @param $storageId + * @return int|null either the numeric storage id or null if the storage id is not known + */ + public static function getNumericStorageId($storageId) { + $storageId = self::adjustStorageId($storageId); + + if ($row = self::getStorageById($storageId)) { + return (int)$row['numeric_id']; + } else { + return null; + } + } + + /** + * @return array [ available, last_checked ] + */ + public function getAvailability() { + if ($row = self::getStorageById($this->storageId)) { + return [ + 'available' => (int)$row['available'] === 1, + 'last_checked' => $row['last_checked'] + ]; + } else { + return [ + 'available' => true, + 'last_checked' => time(), + ]; + } + } + + /** + * @param bool $isAvailable + * @param int $delay amount of seconds to delay reconsidering that storage further + */ + public function setAvailability($isAvailable, int $delay = 0) { + $available = $isAvailable ? 1 : 0; + if (!$isAvailable) { + \OCP\Server::get(LoggerInterface::class)->info('Storage with ' . $this->storageId . ' marked as unavailable', ['app' => 'lib']); + } + + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->update('storages') + ->set('available', $query->createNamedParameter($available)) + ->set('last_checked', $query->createNamedParameter(time() + $delay)) + ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); + $query->executeStatement(); + } + + /** + * Check if a string storage id is known + * + * @param string $storageId + * @return bool + */ + public static function exists($storageId) { + return !is_null(self::getNumericStorageId($storageId)); + } + + /** + * remove the entry for the storage + * + * @param string $storageId + */ + public static function remove($storageId) { + $storageId = self::adjustStorageId($storageId); + $numericId = self::getNumericStorageId($storageId); + + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->eq('id', $query->createNamedParameter($storageId))); + $query->executeStatement(); + + if (!is_null($numericId)) { + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($numericId))); + $query->executeStatement(); + } + } + + /** + * remove the entry for the storage by the mount id + * + * @param int $mountId + */ + public static function cleanByMountId(int $mountId) { + $db = \OC::$server->getDatabaseConnection(); + + try { + $db->beginTransaction(); + + $query = $db->getQueryBuilder(); + $query->select('storage_id') + ->from('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $storageIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + $storageIds = array_unique($storageIds); + + $query = $db->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->in('numeric_id', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $query->executeStatement(); + + $db->commit(); + } catch (\Exception $e) { + $db->rollBack(); + throw $e; + } + } +} diff --git a/lib/private/Files/Cache/StorageGlobal.php b/lib/private/Files/Cache/StorageGlobal.php new file mode 100644 index 00000000000..bab31b1db91 --- /dev/null +++ b/lib/private/Files/Cache/StorageGlobal.php @@ -0,0 +1,99 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Handle the mapping between the string and numeric storage ids + * + * Each storage has 2 different ids + * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') + * and a numeric storage id which is referenced in the file cache + * + * A mapping between the two storage ids is stored in the database and accessible through this class + * + * @package OC\Files\Cache + */ +class StorageGlobal { + /** @var array<string, array> */ + private $cache = []; + /** @var array<int, array> */ + private $numericIdCache = []; + + public function __construct( + private IDBConnection $connection, + ) { + } + + /** + * @param string[] $storageIds + */ + public function loadForStorageIds(array $storageIds) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->in('id', $builder->createNamedParameter(array_values($storageIds), IQueryBuilder::PARAM_STR_ARRAY))); + + $result = $query->executeQuery(); + while ($row = $result->fetch()) { + $this->cache[$row['id']] = $row; + } + $result->closeCursor(); + } + + /** + * @param string $storageId + * @return array|null + */ + public function getStorageInfo(string $storageId): ?array { + if (!isset($this->cache[$storageId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->eq('id', $builder->createNamedParameter($storageId))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $this->cache[$storageId] = $row; + $this->numericIdCache[(int)$row['numeric_id']] = $row; + } + } + return $this->cache[$storageId] ?? null; + } + + /** + * @param int $numericId + * @return array|null + */ + public function getStorageInfoByNumericId(int $numericId): ?array { + if (!isset($this->numericIdCache[$numericId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->eq('numeric_id', $builder->createNamedParameter($numericId))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $this->numericIdCache[$numericId] = $row; + $this->cache[$row['id']] = $row; + } + } + return $this->numericIdCache[$numericId] ?? null; + } + + public function clearCache() { + $this->cache = []; + } +} diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php new file mode 100644 index 00000000000..03681036aa2 --- /dev/null +++ b/lib/private/Files/Cache/Updater.php @@ -0,0 +1,292 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception\DeadlockException; +use OC\Files\FileInfo; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; + +/** + * Update the cache and propagate changes + * + */ +class Updater implements IUpdater { + /** + * @var bool + */ + protected $enabled = true; + + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @var \OC\Files\Cache\Propagator + */ + protected $propagator; + + /** + * @var Scanner + */ + protected $scanner; + + /** + * @var Cache + */ + protected $cache; + + private LoggerInterface $logger; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->propagator = $storage->getPropagator(); + $this->scanner = $storage->getScanner(); + $this->cache = $storage->getCache(); + $this->logger = \OC::$server->get(LoggerInterface::class); + } + + /** + * Disable updating the cache through this updater + */ + public function disable() { + $this->enabled = false; + } + + /** + * Re-enable the updating of the cache through this updater + */ + public function enable() { + $this->enabled = true; + } + + /** + * Get the propagator for etags and mtime for the view the updater works on + * + * @return Propagator + */ + public function getPropagator() { + return $this->propagator; + } + + /** + * Propagate etag and mtime changes for the parent folders of $path up to the root of the filesystem + * + * @param string $path the path of the file to propagate the changes for + * @param int|null $time the timestamp to set as mtime for the parent folders, if left out the current time is used + */ + public function propagate($path, $time = null) { + if (Scanner::isPartialFile($path)) { + return; + } + $this->propagator->propagateChange($path, $time); + } + + /** + * Update the cache for $path and update the size, etag and mtime of the parent folders + * + * @param string $path + * @param int $time + */ + public function update($path, $time = null, ?int $sizeDifference = null) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + if (is_null($time)) { + $time = time(); + } + + $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); + + if (isset($data['oldSize']) && isset($data['size'])) { + $sizeDifference = $data['size'] - $data['oldSize']; + } + + // encryption is a pita and touches the cache itself + if (isset($data['encrypted']) && (bool)$data['encrypted']) { + $sizeDifference = null; + } + + // scanner didn't provide size info, fallback to full size calculation + if ($this->cache instanceof Cache && $sizeDifference === null) { + $this->cache->correctFolderSize($path, $data); + } + $this->correctParentStorageMtime($path); + $this->propagator->propagateChange($path, $time, $sizeDifference ?? 0); + } + + /** + * Remove $path from the cache and update the size, etag and mtime of the parent folders + * + * @param string $path + */ + public function remove($path) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + + $entry = $this->cache->get($path); + + $this->cache->remove($path); + + $this->correctParentStorageMtime($path); + if ($entry instanceof ICacheEntry) { + $this->propagator->propagateChange($path, time(), -$entry->getSize()); + } else { + $this->propagator->propagateChange($path, time()); + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($parent); + } + } + } + + /** + * Rename a file or folder in the cache. + * + * @param IStorage $sourceStorage + * @param string $source + * @param string $target + */ + public function renameFromStorage(IStorage $sourceStorage, $source, $target) { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache) use ($sourceStorage, $source, $target) { + // Remove existing cache entry to no reuse the fileId. + if ($this->cache->inCache($target)) { + $this->cache->remove($target); + } + + if ($sourceStorage === $this->storage) { + $this->cache->move($source, $target); + } else { + $this->cache->moveFromCache($sourceCache, $source, $target); + } + }); + } + + /** + * Copy a file or folder in the cache. + */ + public function copyFromStorage(IStorage $sourceStorage, string $source, string $target): void { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache, ICacheEntry $sourceInfo) use ($target) { + $parent = dirname($target); + if ($parent === '.') { + $parent = ''; + } + $parentInCache = $this->cache->inCache($parent); + if (!$parentInCache) { + $parentData = $this->scanner->scan($parent, Scanner::SCAN_SHALLOW, -1, false); + $parentInCache = $parentData !== null; + } + if ($parentInCache) { + $this->cache->copyFromCache($sourceCache, $sourceInfo, $target); + } + }); + } + + /** + * Utility to copy or rename a file or folder in the cache and update the size, etag and mtime of the parent folders + */ + private function copyOrRenameFromStorage(IStorage $sourceStorage, string $source, string $target, callable $operation): void { + if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) { + return; + } + + $time = time(); + + $sourceCache = $sourceStorage->getCache(); + $sourceUpdater = $sourceStorage->getUpdater(); + $sourcePropagator = $sourceStorage->getPropagator(); + + $sourceInfo = $sourceCache->get($source); + + $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); + $targetExtension = pathinfo($target, PATHINFO_EXTENSION); + $targetIsTrash = preg_match("/^d\d+$/", $targetExtension); + + if ($sourceInfo !== false) { + if (!$this->storage->instanceOfStorage(ObjectStoreStorage::class)) { + $operation($sourceCache, $sourceInfo); + } + + $isDir = $sourceInfo->getMimeType() === FileInfo::MIMETYPE_FOLDER; + } else { + $isDir = $this->storage->is_dir($target); + } + + if ($sourceExtension !== $targetExtension && !$isDir && !$targetIsTrash) { + // handle mime type change + $mimeType = $this->storage->getMimeType($target); + $fileId = $this->cache->getId($target); + $this->cache->update($fileId, ['mimetype' => $mimeType]); + } + + if ($sourceCache instanceof Cache) { + $sourceCache->correctFolderSize($source); + } + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($target); + } + if ($sourceUpdater instanceof Updater) { + $sourceUpdater->correctParentStorageMtime($source); + } + $this->correctParentStorageMtime($target); + $this->updateStorageMTimeOnly($target); + $sourcePropagator->propagateChange($source, $time); + $this->propagator->propagateChange($target, $time); + } + + private function updateStorageMTimeOnly($internalPath) { + $fileId = $this->cache->getId($internalPath); + if ($fileId !== -1) { + $mtime = $this->storage->filemtime($internalPath); + if ($mtime !== false) { + $this->cache->update( + $fileId, [ + 'mtime' => null, // this magic tells it to not overwrite mtime + 'storage_mtime' => $mtime + ] + ); + } + } + } + + /** + * update the storage_mtime of the direct parent in the cache to the mtime from the storage + * + * @param string $internalPath + */ + private function correctParentStorageMtime($internalPath) { + $parentId = $this->cache->getParentId($internalPath); + $parent = dirname($internalPath); + if ($parentId != -1) { + $mtime = $this->storage->filemtime($parent); + if ($mtime !== false) { + try { + $this->cache->update($parentId, ['storage_mtime' => $mtime]); + } catch (DeadlockException $e) { + // ignore the failure. + // with failures concurrent updates, someone else would have already done it. + // in the worst case the `storage_mtime` isn't updated, which should at most only trigger an extra rescan + $this->logger->warning('Error while updating parent storage_mtime, should be safe to ignore', ['exception' => $e]); + } + } + } + } +} diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php new file mode 100644 index 00000000000..f1de5d3cfb8 --- /dev/null +++ b/lib/private/Files/Cache/Watcher.php @@ -0,0 +1,146 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IWatcher; + +/** + * check the storage backends for updates and change the cache accordingly + */ +class Watcher implements IWatcher { + protected $watchPolicy = self::CHECK_ONCE; + + protected $checkedPaths = []; + + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var Cache $cache + */ + protected $cache; + + /** + * @var Scanner $scanner ; + */ + protected $scanner; + + /** @var callable[] */ + protected $onUpdate = []; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->cache = $storage->getCache(); + $this->scanner = $storage->getScanner(); + } + + /** + * @param int $policy either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function setPolicy($policy) { + $this->watchPolicy = $policy; + } + + /** + * @return int either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function getPolicy() { + return $this->watchPolicy; + } + + /** + * check $path for updates and update if needed + * + * @param string $path + * @param ICacheEntry|null $cachedEntry + * @return boolean true if path was updated + */ + public function checkUpdate($path, $cachedEntry = null) { + if (is_null($cachedEntry)) { + $cachedEntry = $this->cache->get($path); + } + if ($cachedEntry === false || $this->needsUpdate($path, $cachedEntry)) { + $this->update($path, $cachedEntry); + + if ($cachedEntry === false) { + return true; + } else { + // storage backends can sometimes return false positives, only return true if the scanner actually found a change + $newEntry = $this->cache->get($path); + return $newEntry->getStorageMTime() > $cachedEntry->getStorageMTime(); + } + } else { + return false; + } + } + + /** + * Update the cache for changes to $path + * + * @param string $path + * @param ICacheEntry $cachedData + */ + public function update($path, $cachedData) { + if ($this->storage->is_dir($path)) { + $this->scanner->scan($path, Scanner::SCAN_SHALLOW); + } else { + $this->scanner->scanFile($path); + } + if (is_array($cachedData) && $cachedData['mimetype'] === 'httpd/unix-directory') { + $this->cleanFolder($path); + } + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($path); + } + foreach ($this->onUpdate as $callback) { + $callback($path); + } + } + + /** + * Check if the cache for $path needs to be updated + * + * @param string $path + * @param ICacheEntry $cachedData + * @return bool + */ + public function needsUpdate($path, $cachedData) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { + $this->checkedPaths[] = $path; + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + } + return false; + } + + /** + * remove deleted files in $path from the cache + * + * @param string $path + */ + public function cleanFolder($path) { + $cachedContent = $this->cache->getFolderContents($path); + foreach ($cachedContent as $entry) { + if (!$this->storage->file_exists($entry['path'])) { + $this->cache->remove($entry['path']); + } + } + } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php new file mode 100644 index 00000000000..5bc4ee8529d --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -0,0 +1,329 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * Jail to a subdirectory of the wrapped cache + */ +class CacheJail extends CacheWrapper { + + protected string $unjailedRoot; + + public function __construct( + ?ICache $cache, + protected string $root, + ?CacheDependencies $dependencies = null, + ) { + parent::__construct($cache, $dependencies); + + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); + } + } + + /** + * @return string + */ + protected function getRoot() { + return $this->root; + } + + /** + * Get the root path with any nested jails resolved + * + * @return string + */ + public function getGetUnjailedRoot() { + return $this->unjailedRoot; + } + + /** + * @return string + */ + protected function getSourcePath(string $path) { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + /** + * @param string $path + * @param null|string $root + * @return null|string the jailed path or null if the path is outside the jail + */ + protected function getJailedPath(string $path, ?string $root = null) { + if ($root === null) { + $root = $this->getRoot(); + } + if ($root === '') { + return $path; + } + $rootLength = strlen($root) + 1; + if ($path === $root) { + return ''; + } elseif (substr($path, 0, $rootLength) === $root . '/') { + return substr($path, $rootLength); + } else { + return null; + } + } + + protected function formatCacheEntry($entry) { + if (isset($entry['path'])) { + $entry['path'] = $this->getJailedPath($entry['path']); + } + return $entry; + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file + * @return ICacheEntry|false + */ + public function get($file) { + if (is_string($file) or $file == '') { + $file = $this->getSourcePath($file); + } + return parent::get($file); + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->getCache()->insert($this->getSourcePath($file), $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->getCache()->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->getCache()->getId($this->getSourcePath($file)); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + return $this->getCache()->getParentId($this->getSourcePath($file)); + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getCache()->inCache($this->getSourcePath($file)); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->getCache()->remove($this->getSourcePath($file)); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->getCache()->move($this->getSourcePath($source), $this->getSourcePath($target)); + } + + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $this->getSourcePath($path)]; + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->getCache()->remove($this->getRoot()); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->getCache()->getStatus($this->getSourcePath($file)); + } + + /** + * update the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + } + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); + } else { + return 0; + } + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + // not supported + return []; + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiply incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + // not supported + return false; + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + $path = $this->getCache()->getPathById($id); + if ($path === null) { + return null; + } + + return $this->getJailedPath($path); + } + + /** + * Move a file or folder in the cache + * + * Note that this should make sure the entries are removed from the source cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(\OCP\Files\Cache\ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache === $this) { + return $this->move($sourcePath, $targetPath); + } + return $this->getCache()->moveFromCache($sourceCache, $sourcePath, $this->getSourcePath($targetPath)); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->addJailFilterQuery($this->getCache()->getQueryFilterForStorage()); + } + + protected function addJailFilterQuery(ISearchOperator $filter): ISearchOperator { + if ($this->getGetUnjailedRoot() !== '' && $this->getGetUnjailedRoot() !== '/') { + return new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, + [ + $filter, + new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, + [ + new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'path', $this->getGetUnjailedRoot()), + new SearchComparison(ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE, 'path', SearchComparison::escapeLikeParameter($this->getGetUnjailedRoot()) . '/%'), + ], + ) + ] + ); + } else { + return $filter; + } + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($this->getGetUnjailedRoot() === '' || str_starts_with($rawEntry->getPath(), $this->getGetUnjailedRoot())) { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $jailedPath = $this->getJailedPath($rawEntry->getPath()); + if ($jailedPath !== null) { + return $this->formatCacheEntry(clone $rawEntry) ?: null; + } + } + } + + return null; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php new file mode 100644 index 00000000000..ff17cb79ac7 --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php @@ -0,0 +1,32 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +class CachePermissionsMask extends CacheWrapper { + /** + * @var int + */ + protected $mask; + + /** + * @param \OCP\Files\Cache\ICache $cache + * @param int $mask + */ + public function __construct($cache, $mask) { + parent::__construct($cache); + $this->mask = $mask; + } + + protected function formatCacheEntry($entry) { + if (isset($entry['permissions'])) { + $entry['scan_permissions'] = $entry['permissions']; + $entry['permissions'] &= $this->mask; + } + return $entry; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php new file mode 100644 index 00000000000..f2f1036d6a3 --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -0,0 +1,315 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\Server; + +class CacheWrapper extends Cache { + /** + * @var ?ICache + */ + protected $cache; + + public function __construct(?ICache $cache, ?CacheDependencies $dependencies = null) { + $this->cache = $cache; + if (!$dependencies && $cache instanceof Cache) { + $this->mimetypeLoader = $cache->mimetypeLoader; + $this->connection = $cache->connection; + $this->querySearchHelper = $cache->querySearchHelper; + } else { + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + } + } + + protected function getCache() { + return $this->cache; + } + + protected function hasEncryptionWrapper(): bool { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->hasEncryptionWrapper(); + } else { + return false; + } + } + + /** + * Make it easy for wrappers to modify every returned cache entry + * + * @param ICacheEntry $entry + * @return ICacheEntry|false + */ + protected function formatCacheEntry($entry) { + return $entry; + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file + * @return ICacheEntry|false + */ + public function get($file) { + $result = $this->getCache()->get($file); + if ($result instanceof ICacheEntry) { + $result = $this->formatCacheEntry($result); + } + return $result; + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + // can't do a simple $this->getCache()->.... call here since getFolderContentsById needs to be called on this + // and not the wrapped cache + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return array + */ + public function getFolderContentsById($fileId) { + $results = $this->getCache()->getFolderContentsById($fileId); + return array_map([$this, 'formatCacheEntry'], $results); + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->getCache()->insert($file, $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->getCache()->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->getCache()->getId($file); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + return $this->getCache()->getParentId($file); + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getCache()->inCache($file); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->getCache()->remove($file); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->getCache()->move($source, $target); + } + + protected function getMoveInfo($path) { + /** @var Cache $cache */ + $cache = $this->getCache(); + return $cache->getMoveInfo($path); + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $this->getCache()->moveFromCache($sourceCache, $sourcePath, $targetPath); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->getCache()->clear(); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->getCache()->getStatus($file); + } + + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); + } + + /** + * update the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); + } + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); + } else { + return 0; + } + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + return $this->getCache()->getAll(); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + return $this->getCache()->getIncomplete(); + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + return $this->getCache()->getPathById($id); + } + + /** + * Returns the numeric storage id + * + * @return int + */ + public function getNumericStorageId() { + return $this->getCache()->getNumericStorageId(); + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @return array first element holding the storage id, second the path + */ + public static function getById($id) { + return parent::getById($id); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->getCache()->getQueryFilterForStorage(); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $entry = $this->formatCacheEntry(clone $rawEntry); + return $entry ?: null; + } + + return null; + } +} diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php new file mode 100644 index 00000000000..d6409b7875e --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -0,0 +1,28 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Propagator; +use OC\Files\Storage\Wrapper\Jail; + +class JailPropagator extends Propagator { + /** + * @var Jail + */ + protected $storage; + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + /** @var \OC\Files\Storage\Storage $storage */ + [$storage, $sourceInternalPath] = $this->storage->resolvePath($internalPath); + $storage->getPropagator()->propagateChange($sourceInternalPath, $time, $sizeDifference); + } +} diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php new file mode 100644 index 00000000000..b1ae516654a --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Watcher; + +class JailWatcher extends Watcher { + private string $root; + private Watcher $watcher; + + public function __construct(Watcher $watcher, string $root) { + $this->watcher = $watcher; + $this->root = $root; + } + + protected function getRoot(): string { + return $this->root; + } + + protected function getSourcePath($path): string { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + public function setPolicy($policy) { + $this->watcher->setPolicy($policy); + } + + public function getPolicy() { + return $this->watcher->getPolicy(); + } + + + public function checkUpdate($path, $cachedEntry = null) { + return $this->watcher->checkUpdate($this->getSourcePath($path), $cachedEntry); + } + + public function update($path, $cachedData) { + $this->watcher->update($this->getSourcePath($path), $cachedData); + } + + public function needsUpdate($path, $cachedData) { + return $this->watcher->needsUpdate($this->getSourcePath($path), $cachedData); + } + + public function cleanFolder($path) { + $this->watcher->cleanFolder($this->getSourcePath($path)); + } + + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } +} |