diff options
Diffstat (limited to 'lib/private/Files')
70 files changed, 1949 insertions, 815 deletions
diff --git a/lib/private/Files/AppData/AppData.php b/lib/private/Files/AppData/AppData.php index 237fcb42e03..1c632c3062f 100644 --- a/lib/private/Files/AppData/AppData.php +++ b/lib/private/Files/AppData/AppData.php @@ -26,9 +26,9 @@ declare(strict_types=1); */ namespace OC\Files\AppData; -use OCP\Cache\CappedMemoryCache; use OC\Files\SimpleFS\SimpleFolder; use OC\SystemConfig; +use OCP\Cache\CappedMemoryCache; use OCP\Files\Folder; use OCP\Files\IAppData; use OCP\Files\IRootFolder; @@ -53,8 +53,8 @@ class AppData implements IAppData { * @param string $appId */ public function __construct(IRootFolder $rootFolder, - SystemConfig $systemConfig, - string $appId) { + SystemConfig $systemConfig, + string $appId) { $this->rootFolder = $rootFolder; $this->config = $systemConfig; $this->appId = $appId; diff --git a/lib/private/Files/AppData/Factory.php b/lib/private/Files/AppData/Factory.php index 03f8fdedcbd..a16c3df327d 100644 --- a/lib/private/Files/AppData/Factory.php +++ b/lib/private/Files/AppData/Factory.php @@ -39,7 +39,7 @@ class Factory implements IAppDataFactory { private array $folders = []; public function __construct(IRootFolder $rootFolder, - SystemConfig $systemConfig) { + SystemConfig $systemConfig) { $this->rootFolder = $rootFolder; $this->config = $systemConfig; } diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 67d01bb6999..2c53706189a 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -44,12 +44,13 @@ use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchQuery; use OC\Files\Storage\Wrapper\Encryption; +use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Cache\CacheEntryInsertedEvent; +use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheEntryUpdatedEvent; use OCP\Files\Cache\CacheInsertEvent; -use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheUpdateEvent; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; @@ -59,6 +60,7 @@ use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use OCP\Util; use Psr\Log\LoggerInterface; @@ -81,61 +83,53 @@ class Cache implements ICache { /** * @var array partial data for the cache */ - protected $partial = []; - - /** - * @var string - */ - protected $storageId; - - private $storage; - - /** - * @var Storage $storageCache - */ - protected $storageCache; - - /** @var IMimeTypeLoader */ - protected $mimetypeLoader; - - /** - * @var IDBConnection - */ - protected $connection; - - /** - * @var IEventDispatcher - */ - protected $eventDispatcher; - - /** @var QuerySearchHelper */ - protected $querySearchHelper; - - /** - * @param IStorage $storage - */ - public function __construct(IStorage $storage) { + protected array $partial = []; + protected string $storageId; + protected Storage $storageCache; + protected IMimeTypeLoader$mimetypeLoader; + protected IDBConnection $connection; + protected SystemConfig $systemConfig; + protected LoggerInterface $logger; + protected QuerySearchHelper $querySearchHelper; + protected IEventDispatcher $eventDispatcher; + protected IFilesMetadataManager $metadataManager; + + public function __construct( + private IStorage $storage, + // this constructor is used in to many pleases to easily do proper di + // so instead we group it all together + CacheDependencies $dependencies = null, + ) { $this->storageId = $storage->getId(); - $this->storage = $storage; if (strlen($this->storageId) > 64) { $this->storageId = md5($this->storageId); } - - $this->storageCache = new Storage($storage); - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); - $this->connection = \OC::$server->getDatabaseConnection(); - $this->eventDispatcher = \OC::$server->get(IEventDispatcher::class); - $this->querySearchHelper = \OCP\Server::get(QuerySearchHelper::class); + if (!$dependencies) { + $dependencies = \OC::$server->get(CacheDependencies::class); + } + $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->systemConfig = $dependencies->getSystemConfig(); + $this->logger = $dependencies->getLogger(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + $this->eventDispatcher = $dependencies->getEventDispatcher(); + $this->metadataManager = $dependencies->getMetadataManager(); } protected function getQueryBuilder() { return new CacheQueryBuilder( $this->connection, - \OC::$server->getSystemConfig(), - \OC::$server->get(LoggerInterface::class) + $this->systemConfig, + $this->logger, + $this->metadataManager, ); } + public function getStorageCache(): Storage { + return $this->storageCache; + } + /** * Get the numeric storage id for this cache's storage * @@ -154,6 +148,7 @@ class Cache implements ICache { public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); if (is_string($file) || $file == '') { // normalize file @@ -175,6 +170,7 @@ class Cache implements ICache { } elseif (!$data) { return $data; } else { + $data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? []; return self::cacheEntryFromData($data, $this->mimetypeLoader); } } @@ -239,11 +235,14 @@ class Cache implements ICache { ->whereParent($fileId) ->orderBy('name', 'ASC'); + $metadataQuery = $query->selectMetadata(); + $result = $query->execute(); $files = $result->fetchAll(); $result->closeCursor(); - return array_map(function (array $data) { + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? []; return self::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); } @@ -447,7 +446,7 @@ class Cache implements ICache { $params = []; $extensionParams = []; foreach ($data as $name => $value) { - if (array_search($name, $fields) !== false) { + if (in_array($name, $fields)) { if ($name === 'path') { $params['path_hash'] = md5($value); } elseif ($name === 'mimetype') { @@ -467,7 +466,7 @@ class Cache implements ICache { } $params[$name] = $value; } - if (array_search($name, $extensionFields) !== false) { + if (in_array($name, $extensionFields)) { $extensionParams[$name] = $value; } } @@ -586,8 +585,13 @@ class Cache implements ICache { return $cacheEntry->getPath(); }, $children); - $deletedIds = array_merge($deletedIds, $childIds); - $deletedPaths = array_merge($deletedPaths, $childPaths); + foreach ($childIds as $childId) { + $deletedIds[] = $childId; + } + + foreach ($childPaths as $childPath) { + $deletedPaths[] = $childPath; + } $query = $this->getQueryBuilder(); $query->delete('filecache_extended') @@ -599,9 +603,12 @@ class Cache implements ICache { } /** @var ICacheEntry[] $childFolders */ - $childFolders = array_filter($children, function ($child) { - return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER; - }); + $childFolders = []; + foreach ($children as $child) { + if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $childFolders[] = $child; + } + } foreach ($childFolders as $folder) { $parentIds[] = $folder->getId(); $queue[] = $folder->getId(); diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php new file mode 100644 index 00000000000..7c51f3ff884 --- /dev/null +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -0,0 +1,57 @@ +<?php + +namespace OC\Files\Cache; + +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDependencies { + public function __construct( + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $metadataManager, + private DisplayNameCache $displayNameCache, + ) { + } + + public function getMimeTypeLoader(): IMimeTypeLoader { + return $this->mimeTypeLoader; + } + + public function getConnection(): IDBConnection { + return $this->connection; + } + + public function getEventDispatcher(): IEventDispatcher { + return $this->eventDispatcher; + } + + public function getQuerySearchHelper(): QuerySearchHelper { + return $this->querySearchHelper; + } + + public function getSystemConfig(): SystemConfig { + return $this->systemConfig; + } + + public function getLogger(): LoggerInterface { + return $this->logger; + } + + public function getDisplayNameCache(): DisplayNameCache { + return $this->displayNameCache; + } + + public function getMetadataManager(): IFilesMetadataManager { + return $this->metadataManager; + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index ce9df2823c8..d1a64552fd1 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -134,7 +134,7 @@ class CacheEntry implements ICacheEntry { } public function getUnencryptedSize(): int { - if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + if ($this->data['encrypted'] && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { return $this->data['unencrypted_size']; } else { return $this->data['size'] ?? 0; diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 34d2177b84e..365d28fc8c5 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -5,6 +5,7 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2019 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -28,6 +29,8 @@ namespace OC\Files\Cache; use OC\DB\QueryBuilder\QueryBuilder; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; use OCP\IDBConnection; use Psr\Log\LoggerInterface; @@ -35,9 +38,14 @@ use Psr\Log\LoggerInterface; * Query builder with commonly used helpers for filecache queries */ class CacheQueryBuilder extends QueryBuilder { - private $alias = null; - - public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) { + private ?string $alias = null; + + public function __construct( + IDBConnection $connection, + SystemConfig $systemConfig, + LoggerInterface $logger, + private IFilesMetadataManager $filesMetadataManager, + ) { parent::__construct($connection, $systemConfig, $logger); } @@ -63,7 +71,7 @@ class CacheQueryBuilder extends QueryBuilder { public function selectFileCache(string $alias = null, bool $joinExtendedCache = true) { $name = $alias ?: 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', - 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'unencrypted_size') + 'storage_mtime', 'encrypted', 'etag', "$name.permissions", 'checksum', 'unencrypted_size') ->from('filecache', $name); if ($joinExtendedCache) { @@ -126,4 +134,15 @@ class CacheQueryBuilder extends QueryBuilder { return $this; } + + /** + * join metadata to current query builder and returns an helper + * + * @return IMetadataQuery|null NULL if no metadata have never been generated + */ + public function selectMetadata(): ?IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery?->retrieveMetadata(); + return $metadataQuery; + } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 15c089a0f11..d8c5e66e129 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @author Tobias Kaminsky <tobias@kaminsky.me> @@ -37,52 +38,47 @@ use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUser; use Psr\Log\LoggerInterface; class QuerySearchHelper { - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - /** @var IDBConnection */ - private $connection; - /** @var SystemConfig */ - private $systemConfig; - private LoggerInterface $logger; - /** @var SearchBuilder */ - private $searchBuilder; - /** @var QueryOptimizer */ - private $queryOptimizer; - private IGroupManager $groupManager; - public function __construct( - IMimeTypeLoader $mimetypeLoader, - IDBConnection $connection, - SystemConfig $systemConfig, - LoggerInterface $logger, - SearchBuilder $searchBuilder, - QueryOptimizer $queryOptimizer, - IGroupManager $groupManager, + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; - $this->connection = $connection; - $this->systemConfig = $systemConfig; - $this->logger = $logger; - $this->searchBuilder = $searchBuilder; - $this->queryOptimizer = $queryOptimizer; - $this->groupManager = $groupManager; } protected function getQueryBuilder() { return new CacheQueryBuilder( $this->connection, $this->systemConfig, - $this->logger + $this->logger, + $this->filesMetadataManager, ); } - protected function applySearchConstraints(CacheQueryBuilder $query, ISearchQuery $searchQuery, array $caches): void { + /** + * @param CacheQueryBuilder $query + * @param ISearchQuery $searchQuery + * @param array $caches + * @param IMetadataQuery|null $metadataQuery + */ + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null + ): void { $storageFilters = array_values(array_map(function (ICache $cache) { return $cache->getQueryFilterForStorage(); }, $caches)); @@ -90,12 +86,12 @@ class QuerySearchHelper { $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); $this->queryOptimizer->processOperator($filter); - $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter); + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); if ($searchExpr) { $query->andWhere($searchExpr); } - $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder()); + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); if ($searchQuery->getLimit()) { $query->setMaxResults($searchQuery->getLimit()); @@ -144,6 +140,11 @@ class QuerySearchHelper { )); } + + protected function equipQueryForShares(CacheQueryBuilder $query): void { + $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); + } + /** * Perform a file system search in multiple caches * @@ -175,19 +176,31 @@ class QuerySearchHelper { $query = $builder->selectFileCache('file', false); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + if (in_array('systemtag', $requestedFields)) { $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); + } - $this->applySearchConstraints($query, $searchQuery, $caches); + $metadataQuery = $query->selectMetadata(); + + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); $result = $query->execute(); $files = $result->fetchAll(); - $rawEntries = array_map(function (array $data) { + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + // migrate to null safe ... + if ($metadataQuery === null) { + $data['metadata'] = []; + } else { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + } return Cache::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 52268032409..0c82e21e30d 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -37,14 +37,14 @@ namespace OC\Files\Cache; use Doctrine\DBAL\Exception; use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OC\Hooks\BasicEmitter; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; use OCP\Files\NotFoundException; use OCP\Files\Storage\IReliableEtagStorage; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; -use OC\Files\Storage\Wrapper\Jail; -use OC\Hooks\BasicEmitter; use Psr\Log\LoggerInterface; /** @@ -203,7 +203,9 @@ class Scanner extends BasicEmitter implements IScanner { $fileId = $cacheData['fileid']; $data['fileid'] = $fileId; // only reuse data if the file hasn't explicitly changed - if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { $data['mtime'] = $cacheData['mtime']; if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { $data['size'] = $cacheData['size']; @@ -220,6 +222,11 @@ class Scanner extends BasicEmitter implements IScanner { // Only update metadata that has changed $newData = array_diff_assoc($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; + } } else { // we only updated unencrypted_size if it's already set unset($data['unencrypted_size']); @@ -385,19 +392,23 @@ class Scanner extends BasicEmitter implements IScanner { * @param int $reuse a combination of self::REUSE_* * @param int $folderId id for the folder to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param int $oldSize the size of the folder before (re)scanning the children + * @param int|float $oldSize the size of the folder before (re)scanning the children * @return int|float the size of the scanned folder or -1 if the size is unknown at this stage */ - protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int $oldSize) { + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]); $size = 0; - $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size); + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); foreach ($childQueue as $child => [$childId, $childSize]) { - $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize); + // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed + $childEtagChanged = false; + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize, $childEtagChanged); + $etagChanged |= $childEtagChanged; + if ($childSize === -1) { $size = -1; } elseif ($size !== -1) { @@ -410,15 +421,27 @@ class Scanner extends BasicEmitter implements IScanner { if ($this->storage->instanceOfStorage(Encryption::class)) { $this->cache->calculateFolderSize($path); } else { - if ($this->cacheActive && $oldSize !== $size) { - $this->cache->update($folderId, ['size' => $size]); + if ($this->cacheActive) { + $updatedData = []; + if ($oldSize !== $size) { + $updatedData['size'] = $size; + } + if ($etagChanged) { + $updatedData['etag'] = uniqid(); + } + if ($updatedData) { + $this->cache->update($folderId, $updatedData); + } } } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); return $size; } - private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { + /** + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + */ + private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array { // we put this in it's own function so it cleans up the memory before we start recursing $existingChildren = $this->getExistingChildren($folderId); $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); @@ -436,7 +459,7 @@ class Scanner extends BasicEmitter implements IScanner { $childQueue = []; $newChildNames = []; foreach ($newChildren as $fileMeta) { - $permissions = isset($fileMeta['scan_permissions']) ? $fileMeta['scan_permissions'] : $fileMeta['permissions']; + $permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions']; if ($permissions === 0) { continue; } @@ -453,7 +476,7 @@ class Scanner extends BasicEmitter implements IScanner { $newChildNames[] = $file; $child = $path ? $path . '/' . $file : $file; try { - $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false; + $existingData = $existingChildren[$file] ?? false; $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta); if ($data) { if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) { @@ -466,6 +489,10 @@ class Scanner extends BasicEmitter implements IScanner { } elseif ($size !== -1) { $size += $data['size']; } + + if (isset($data['etag_changed']) && $data['etag_changed']) { + $etagChanged = true; + } } } catch (Exception $ex) { // might happen if inserting duplicate while a scanning diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index b9a70bbd39b..e8063b77aa5 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @author Tobias Kaminsky <tobias@kaminsky.me> @@ -32,11 +33,16 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IMetadataQuery; /** * Tools for transforming search queries into database queries + * + * @psalm-import-type ParamSingleValue from ISearchComparison + * @psalm-import-type ParamValue from ISearchComparison */ class SearchBuilder { + /** @var array<string, string> */ protected static $searchOperatorMap = [ ISearchComparison::COMPARE_LIKE => 'iLike', ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like', @@ -45,8 +51,11 @@ class SearchBuilder { ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', ISearchComparison::COMPARE_LESS_THAN => 'lt', ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte', + ISearchComparison::COMPARE_DEFINED => 'isNotNull', + ISearchComparison::COMPARE_IN => 'in', ]; + /** @var array<string, string> */ protected static $searchOperatorNegativeMap = [ ISearchComparison::COMPARE_LIKE => 'notLike', ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike', @@ -55,6 +64,39 @@ class SearchBuilder { ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', ISearchComparison::COMPARE_LESS_THAN => 'gte', ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt', + ISearchComparison::COMPARE_DEFINED => 'isNull', + ISearchComparison::COMPARE_IN => 'notIn', + ]; + + /** @var array<string, string> */ + protected static $fieldTypes = [ + 'mimetype' => 'string', + 'mtime' => 'integer', + 'name' => 'string', + 'path' => 'string', + 'size' => 'integer', + 'tagname' => 'string', + 'systemtag' => 'string', + 'favorite' => 'boolean', + 'fileid' => 'integer', + 'storage' => 'integer', + 'share_with' => 'string', + 'share_type' => 'integer', + 'owner' => 'string', + ]; + + /** @var array<string, int> */ + protected static $paramTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR, + 'integer' => IQueryBuilder::PARAM_INT, + 'boolean' => IQueryBuilder::PARAM_BOOL, + ]; + + /** @var array<string, int> */ + protected static $paramArrayTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR_ARRAY, + 'integer' => IQueryBuilder::PARAM_INT_ARRAY, + 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, ]; public const TAG_FAVORITE = '_$!<Favorite>!$_'; @@ -76,7 +118,7 @@ class SearchBuilder { return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); }, []); - } elseif ($operator instanceof ISearchComparison) { + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { return [$operator->getField()]; } return []; @@ -86,13 +128,21 @@ class SearchBuilder { * @param IQueryBuilder $builder * @param ISearchOperator[] $operators */ - public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) { - return array_filter(array_map(function ($operator) use ($builder) { - return $this->searchOperatorToDBExpr($builder, $operator); + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); }, $operators)); } - public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) { + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null + ) { $expr = $builder->expr(); if ($operator instanceof ISearchBinaryOperator) { @@ -104,62 +154,99 @@ class SearchBuilder { case ISearchBinaryOperator::OPERATOR_NOT: $negativeOperator = $operator->getArguments()[0]; if ($negativeOperator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap); + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); } else { throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); } // no break case ISearchBinaryOperator::OPERATOR_AND: - return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); case ISearchBinaryOperator::OPERATOR_OR: - return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); default: throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); } } elseif ($operator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap); + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); } else { throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); } } - private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) { - $this->validateComparison($comparison); + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null + ) { + if ($comparison->getExtra()) { + [$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison); + } - [$field, $value, $type] = $this->getOperatorFieldAndValue($comparison); if (isset($operatorMap[$type])) { $queryOperator = $operatorMap[$type]; - return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value)); + return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType)); } else { throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); } } - private function getOperatorFieldAndValue(ISearchComparison $operator) { + /** + * @param ISearchComparison $operator + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValue(ISearchComparison $operator): array { + $this->validateComparison($operator); $field = $operator->getField(); $value = $operator->getValue(); $type = $operator->getType(); + $pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true); + return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash); + } + + /** + * @param string $field + * @param ParamValue $value + * @param string $type + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array { + $paramType = self::$fieldTypes[$field]; + if ($type === ISearchComparison::COMPARE_IN) { + $resultField = $field; + $values = []; + foreach ($value as $arrayValue) { + /** @var ParamSingleValue $arrayValue */ + [$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash); + $resultField = $arrayField; + $values[] = $arrayValue; + } + return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType]; + } if ($field === 'mimetype') { $value = (string)$value; - if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) { - $value = (int)$this->mimetypeLoader->getId($value); - } elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) { + if ($type === ISearchComparison::COMPARE_EQUAL) { + $value = $this->mimetypeLoader->getId($value); + } elseif ($type === ISearchComparison::COMPARE_LIKE) { // transform "mimetype='foo/%'" to "mimepart='foo'" if (preg_match('|(.+)/%|', $value, $matches)) { $field = 'mimepart'; - $value = (int)$this->mimetypeLoader->getId($matches[1]); + $value = $this->mimetypeLoader->getId($matches[1]); $type = ISearchComparison::COMPARE_EQUAL; } elseif (str_contains($value, '%')) { throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); } else { $field = 'mimetype'; - $value = (int)$this->mimetypeLoader->getId($value); + $value = $this->mimetypeLoader->getId($value); $type = ISearchComparison::COMPARE_EQUAL; } } } elseif ($field === 'favorite') { $field = 'tag.category'; $value = self::TAG_FAVORITE; + $paramType = 'string'; } elseif ($field === 'name') { $field = 'file.name'; } elseif ($field === 'tagname') { @@ -168,59 +255,82 @@ class SearchBuilder { $field = 'systemtag.name'; } elseif ($field === 'fileid') { $field = 'file.fileid'; - } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true)) { + } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { $field = 'path_hash'; $value = md5((string)$value); + } elseif ($field === 'owner') { + $field = 'uid_owner'; } - return [$field, $value, $type]; + return [$field, $value, $type, $paramType]; } private function validateComparison(ISearchComparison $operator) { - $types = [ - 'mimetype' => 'string', - 'mtime' => 'integer', - 'name' => 'string', - 'path' => 'string', - 'size' => 'integer', - 'tagname' => 'string', - 'systemtag' => 'string', - 'favorite' => 'boolean', - 'fileid' => 'integer', - 'storage' => 'integer', - ]; $comparisons = [ - 'mimetype' => ['eq', 'like'], + 'mimetype' => ['eq', 'like', 'in'], 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], - 'name' => ['eq', 'like', 'clike'], - 'path' => ['eq', 'like', 'clike'], + 'name' => ['eq', 'like', 'clike', 'in'], + 'path' => ['eq', 'like', 'clike', 'in'], 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], 'tagname' => ['eq', 'like'], 'systemtag' => ['eq', 'like'], 'favorite' => ['eq'], - 'fileid' => ['eq'], - 'storage' => ['eq'], + 'fileid' => ['eq', 'in'], + 'storage' => ['eq', 'in'], + 'share_with' => ['eq'], + 'share_type' => ['eq'], + 'owner' => ['eq'], ]; - if (!isset($types[$operator->getField()])) { + if (!isset(self::$fieldTypes[$operator->getField()])) { throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); } - $type = $types[$operator->getField()]; - if (gettype($operator->getValue()) !== $type) { - throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + $type = self::$fieldTypes[$operator->getField()]; + if ($operator->getType() === ISearchComparison::COMPARE_IN) { + if (!is_array($operator->getValue())) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + foreach ($operator->getValue() as $arrayValue) { + if (gettype($arrayValue) !== $type) { + throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField()); + } + } + } else { + if (gettype($operator->getValue()) !== $type) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } } if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); } } - private function getParameterForValue(IQueryBuilder $builder, $value) { + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $paramType = self::$fieldTypes[$operator->getField()]; + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + switch($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type, $paramType]; + } + + private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) { if ($value instanceof \DateTime) { $value = $value->getTimestamp(); } - if (is_numeric($value)) { - $type = IQueryBuilder::PARAM_INT; + if (is_array($value)) { + $type = self::$paramArrayTypeMap[$paramType]; } else { - $type = IQueryBuilder::PARAM_STR; + $type = self::$paramTypeMap[$paramType]; } return $builder->createNamedParameter($value, $type); } @@ -228,24 +338,32 @@ class SearchBuilder { /** * @param IQueryBuilder $query * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery */ - public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) { + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { foreach ($orders as $order) { $field = $order->getField(); - if ($field === 'fileid') { - $field = 'file.fileid'; - } + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; - // Mysql really likes to pick an index for sorting if it can't fully satisfy the where - // filter with an index, since search queries pretty much never are fully filtered by index - // mysql often picks an index for sorting instead of the much more useful index for filtering. - // - // By changing the order by to an expression, mysql isn't smart enough to see that it could still - // use the index, so it instead picks an index for the filtering - if ($field === 'mtime') { - $field = $query->func()->add($field, $query->createNamedParameter(0)); - } + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } $query->addOrderBy($field, $order->getDirection()); } } diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 01fc638cef8..ba0f98f42f4 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -31,6 +31,7 @@ namespace OC\Files\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Storage\IStorage; +use OCP\IDBConnection; use Psr\Log\LoggerInterface; /** @@ -65,7 +66,7 @@ class Storage { * @param bool $isAvailable * @throws \RuntimeException */ - public function __construct($storage, $isAvailable = true) { + public function __construct($storage, $isAvailable, IDBConnection $connection) { if ($storage instanceof IStorage) { $this->storageId = $storage->getId(); } else { @@ -76,7 +77,6 @@ class Storage { if ($row = self::getStorageById($this->storageId)) { $this->numericId = (int)$row['numeric_id']; } else { - $connection = \OC::$server->getDatabaseConnection(); $available = $isAvailable ? 1 : 0; if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { $this->numericId = $connection->lastInsertId('*PREFIX*storages'); diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php index 457dd207e9d..a6f2f3375a4 100644 --- a/lib/private/Files/Cache/Updater.php +++ b/lib/private/Files/Cache/Updater.php @@ -119,7 +119,7 @@ class Updater implements IUpdater { * @param string $path * @param int $time */ - public function update($path, $time = null) { + public function update($path, $time = null, ?int $sizeDifference = null) { if (!$this->enabled or Scanner::isPartialFile($path)) { return; } @@ -128,20 +128,22 @@ class Updater implements IUpdater { } $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); - if ( - isset($data['oldSize']) && isset($data['size']) && - !$data['encrypted'] // encryption is a pita and touches the cache itself - ) { + + if (isset($data['oldSize']) && isset($data['size'])) { $sizeDifference = $data['size'] - $data['oldSize']; - } else { - // scanner didn't provide size info, fallback to full size calculation - $sizeDifference = 0; - if ($this->cache instanceof Cache) { - $this->cache->correctFolderSize($path, $data); - } + } + + // encryption is a pita and touches the cache itself + if (isset($data['encrypted']) && !!$data['encrypted']) { + $sizeDifference = null; + } + + // scanner didn't provide size info, fallback to full size calculation + if ($this->cache instanceof Cache && $sizeDifference === null) { + $this->cache->correctFolderSize($path, $data); } $this->correctParentStorageMtime($path); - $this->propagator->propagateChange($path, $time, $sizeDifference); + $this->propagator->propagateChange($path, $time, $sizeDifference ?? 0); } /** diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php index acc76f263dc..61ea5b2f848 100644 --- a/lib/private/Files/Cache/Watcher.php +++ b/lib/private/Files/Cache/Watcher.php @@ -129,7 +129,7 @@ class Watcher implements IWatcher { * @return bool */ public function needsUpdate($path, $cachedData) { - if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and array_search($path, $this->checkedPaths) === false)) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { $this->checkedPaths[] = $path; return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index d8cf3eb61d7..f9754e433df 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -28,8 +28,10 @@ namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; @@ -45,15 +47,13 @@ class CacheJail extends CacheWrapper { protected $root; protected $unjailedRoot; - /** - * @param ?\OCP\Files\Cache\ICache $cache - * @param string $root - */ - public function __construct($cache, $root) { - parent::__construct($cache); + public function __construct( + ?ICache $cache, + string $root, + CacheDependencies $dependencies = null, + ) { + parent::__construct($cache, $dependencies); $this->root = $root; - $this->connection = \OC::$server->getDatabaseConnection(); - $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); if ($cache instanceof CacheJail) { $this->unjailedRoot = $cache->getSourcePath($root); diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index 39a78f31343..31410eea798 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -30,33 +30,32 @@ namespace OC\Files\Cache\Wrapper; use OC\Files\Cache\Cache; -use OC\Files\Cache\QuerySearchHelper; +use OC\Files\Cache\CacheDependencies; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; -use OCP\Files\IMimeTypeLoader; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; -use OCP\IDBConnection; +use OCP\Server; class CacheWrapper extends Cache { /** - * @var \OCP\Files\Cache\ICache + * @var ?ICache */ protected $cache; - /** - * @param \OCP\Files\Cache\ICache $cache - */ - public function __construct($cache) { + public function __construct(?ICache $cache, CacheDependencies $dependencies = null) { $this->cache = $cache; - if ($cache instanceof Cache) { + if (!$dependencies && $cache instanceof Cache) { $this->mimetypeLoader = $cache->mimetypeLoader; $this->connection = $cache->connection; $this->querySearchHelper = $cache->querySearchHelper; } else { - $this->mimetypeLoader = \OC::$server->get(IMimeTypeLoader::class); - $this->connection = \OC::$server->get(IDBConnection::class); - $this->querySearchHelper = \OC::$server->get(QuerySearchHelper::class); + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); } } @@ -91,7 +90,7 @@ class CacheWrapper extends Cache { */ public function get($file) { $result = $this->getCache()->get($file); - if ($result) { + if ($result instanceof ICacheEntry) { $result = $this->formatCacheEntry($result); } return $result; diff --git a/lib/private/Files/Config/CachedMountInfo.php b/lib/private/Files/Config/CachedMountInfo.php index 43c9fae63ec..19fa87aa090 100644 --- a/lib/private/Files/Config/CachedMountInfo.php +++ b/lib/private/Files/Config/CachedMountInfo.php @@ -35,6 +35,7 @@ class CachedMountInfo implements ICachedMountInfo { protected ?int $mountId; protected string $rootInternalPath; protected string $mountProvider; + protected string $key; /** * CachedMountInfo constructor. @@ -65,6 +66,7 @@ class CachedMountInfo implements ICachedMountInfo { throw new \Exception("Mount provider $mountProvider name exceeds the limit of 128 characters"); } $this->mountProvider = $mountProvider; + $this->key = $rootId . '::' . $mountPoint; } /** @@ -95,12 +97,7 @@ class CachedMountInfo implements ICachedMountInfo { // TODO injection etc Filesystem::initMountPoints($this->getUser()->getUID()); $userNode = \OC::$server->getUserFolder($this->getUser()->getUID()); - $nodes = $userNode->getParent()->getById($this->getRootId()); - if (count($nodes) > 0) { - return $nodes[0]; - } else { - return null; - } + return $userNode->getParent()->getFirstNodeById($this->getRootId()); } /** @@ -132,4 +129,8 @@ class CachedMountInfo implements ICachedMountInfo { public function getMountProvider(): string { return $this->mountProvider; } + + public function getKey(): string { + return $this->key; + } } diff --git a/lib/private/Files/Config/LazyPathCachedMountInfo.php b/lib/private/Files/Config/LazyPathCachedMountInfo.php new file mode 100644 index 00000000000..d42052c5f83 --- /dev/null +++ b/lib/private/Files/Config/LazyPathCachedMountInfo.php @@ -0,0 +1,63 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright (c) 2022 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ +namespace OC\Files\Config; + +use OCP\IUser; + +class LazyPathCachedMountInfo extends CachedMountInfo { + // we don't allow \ in paths so it makes a great placeholder + private const PATH_PLACEHOLDER = '\\PLACEHOLDER\\'; + + /** @var callable(CachedMountInfo): string */ + protected $rootInternalPathCallback; + + /** + * @param IUser $user + * @param int $storageId + * @param int $rootId + * @param string $mountPoint + * @param string $mountProvider + * @param int|null $mountId + * @param callable(CachedMountInfo): string $rootInternalPathCallback + * @throws \Exception + */ + public function __construct( + IUser $user, + int $storageId, + int $rootId, + string $mountPoint, + string $mountProvider, + int $mountId = null, + callable $rootInternalPathCallback, + ) { + parent::__construct($user, $storageId, $rootId, $mountPoint, $mountProvider, $mountId, self::PATH_PLACEHOLDER); + $this->rootInternalPathCallback = $rootInternalPathCallback; + } + + public function getRootInternalPath(): string { + if ($this->rootInternalPath === self::PATH_PLACEHOLDER) { + $this->rootInternalPath = ($this->rootInternalPathCallback)($this); + } + return $this->rootInternalPath; + } +} diff --git a/lib/private/Files/Config/LazyStorageMountInfo.php b/lib/private/Files/Config/LazyStorageMountInfo.php index 78055a2cdb8..7e4acb2e129 100644 --- a/lib/private/Files/Config/LazyStorageMountInfo.php +++ b/lib/private/Files/Config/LazyStorageMountInfo.php @@ -39,6 +39,7 @@ class LazyStorageMountInfo extends CachedMountInfo { $this->rootId = 0; $this->storageId = 0; $this->mountPoint = ''; + $this->key = ''; } /** @@ -87,4 +88,11 @@ class LazyStorageMountInfo extends CachedMountInfo { public function getMountProvider(): string { return $this->mount->getMountProvider(); } + + public function getKey(): string { + if (!$this->key) { + $this->key = $this->getRootId() . '::' . $this->getMountPoint(); + } + return $this->key; + } } diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php index ae6481e45bb..d251199fd43 100644 --- a/lib/private/Files/Config/MountProviderCollection.php +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -238,6 +238,11 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) { return array_merge($mounts, $providerMounts); }, []); + + if (count($mounts) === 0) { + throw new \Exception("No root mounts provided by any provider"); + } + return $mounts; } diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php index 90f94b6598e..8275eee7b9f 100644 --- a/lib/private/Files/Config/UserMountCache.php +++ b/lib/private/Files/Config/UserMountCache.php @@ -28,14 +28,13 @@ */ namespace OC\Files\Config; +use OC\User\LazyUser; use OCP\Cache\CappedMemoryCache; -use OCA\Files_Sharing\SharedMount; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Diagnostics\IEventLogger; use OCP\Files\Config\ICachedMountFileInfo; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; -use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\IDBConnection; use OCP\IUser; @@ -54,6 +53,11 @@ class UserMountCache implements IUserMountCache { * @var CappedMemoryCache<ICachedMountInfo[]> **/ private CappedMemoryCache $mountsForUsers; + /** + * fileid => internal path mapping for cached mount info. + * @var CappedMemoryCache<string> + **/ + private CappedMemoryCache $internalPathCache; private LoggerInterface $logger; /** @var CappedMemoryCache<array> */ private CappedMemoryCache $cacheInfoCache; @@ -73,46 +77,33 @@ class UserMountCache implements IUserMountCache { $this->logger = $logger; $this->eventLogger = $eventLogger; $this->cacheInfoCache = new CappedMemoryCache(); + $this->internalPathCache = new CappedMemoryCache(); $this->mountsForUsers = new CappedMemoryCache(); } public function registerMounts(IUser $user, array $mounts, array $mountProviderClasses = null) { $this->eventLogger->start('fs:setup:user:register', 'Registering mounts for user'); - // filter out non-proper storages coming from unit tests - $mounts = array_filter($mounts, function (IMountPoint $mount) { - return $mount instanceof SharedMount || ($mount->getStorage() && $mount->getStorage()->getCache()); - }); - /** @var ICachedMountInfo[] $newMounts */ - $newMounts = array_map(function (IMountPoint $mount) use ($user) { + /** @var array<string, ICachedMountInfo> $newMounts */ + $newMounts = []; + foreach ($mounts as $mount) { // filter out any storages which aren't scanned yet since we aren't interested in files from those storages (yet) - if ($mount->getStorageRootId() === -1) { - return null; - } else { - return new LazyStorageMountInfo($user, $mount); + if ($mount->getStorageRootId() !== -1) { + $mountInfo = new LazyStorageMountInfo($user, $mount); + $newMounts[$mountInfo->getKey()] = $mountInfo; } - }, $mounts); - $newMounts = array_values(array_filter($newMounts)); - $newMountKeys = array_map(function (ICachedMountInfo $mount) { - return $mount->getRootId() . '::' . $mount->getMountPoint(); - }, $newMounts); - $newMounts = array_combine($newMountKeys, $newMounts); + } $cachedMounts = $this->getMountsForUser($user); if (is_array($mountProviderClasses)) { $cachedMounts = array_filter($cachedMounts, function (ICachedMountInfo $mountInfo) use ($mountProviderClasses, $newMounts) { // for existing mounts that didn't have a mount provider set // we still want the ones that map to new mounts - $mountKey = $mountInfo->getRootId() . '::' . $mountInfo->getMountPoint(); - if ($mountInfo->getMountProvider() === '' && isset($newMounts[$mountKey])) { + if ($mountInfo->getMountProvider() === '' && isset($newMounts[$mountInfo->getKey()])) { return true; } return in_array($mountInfo->getMountProvider(), $mountProviderClasses); }); } - $cachedRootKeys = array_map(function (ICachedMountInfo $mount) { - return $mount->getRootId() . '::' . $mount->getMountPoint(); - }, $cachedMounts); - $cachedMounts = array_combine($cachedRootKeys, $cachedMounts); $addedMounts = []; $removedMounts = []; @@ -131,46 +122,44 @@ class UserMountCache implements IUserMountCache { $changedMounts = $this->findChangedMounts($newMounts, $cachedMounts); - $this->connection->beginTransaction(); - try { - foreach ($addedMounts as $mount) { - $this->addToCache($mount); - /** @psalm-suppress InvalidArgument */ - $this->mountsForUsers[$user->getUID()][] = $mount; - } - foreach ($removedMounts as $mount) { - $this->removeFromCache($mount); - $index = array_search($mount, $this->mountsForUsers[$user->getUID()]); - unset($this->mountsForUsers[$user->getUID()][$index]); - } - foreach ($changedMounts as $mount) { - $this->updateCachedMount($mount); + if ($addedMounts || $removedMounts || $changedMounts) { + $this->connection->beginTransaction(); + $userUID = $user->getUID(); + try { + foreach ($addedMounts as $mount) { + $this->addToCache($mount); + /** @psalm-suppress InvalidArgument */ + $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; + } + foreach ($removedMounts as $mount) { + $this->removeFromCache($mount); + unset($this->mountsForUsers[$userUID][$mount->getKey()]); + } + foreach ($changedMounts as $mount) { + $this->updateCachedMount($mount); + /** @psalm-suppress InvalidArgument */ + $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; + } + $this->connection->commit(); + } catch (\Throwable $e) { + $this->connection->rollBack(); + throw $e; } - $this->connection->commit(); - } catch (\Throwable $e) { - $this->connection->rollBack(); - throw $e; } $this->eventLogger->end('fs:setup:user:register'); } /** - * @param ICachedMountInfo[] $newMounts - * @param ICachedMountInfo[] $cachedMounts + * @param array<string, ICachedMountInfo> $newMounts + * @param array<string, ICachedMountInfo> $cachedMounts * @return ICachedMountInfo[] */ private function findChangedMounts(array $newMounts, array $cachedMounts) { - $new = []; - foreach ($newMounts as $mount) { - $new[$mount->getRootId() . '::' . $mount->getMountPoint()] = $mount; - } $changed = []; - foreach ($cachedMounts as $cachedMount) { - $key = $cachedMount->getRootId() . '::' . $cachedMount->getMountPoint(); - if (isset($new[$key])) { - $newMount = $new[$key]; + foreach ($cachedMounts as $key => $cachedMount) { + if (isset($newMounts[$key])) { + $newMount = $newMounts[$key]; if ( - $newMount->getMountPoint() !== $cachedMount->getMountPoint() || $newMount->getStorageId() !== $cachedMount->getStorageId() || $newMount->getMountId() !== $cachedMount->getMountId() || $newMount->getMountProvider() !== $cachedMount->getMountProvider() @@ -222,24 +211,38 @@ class UserMountCache implements IUserMountCache { $query->execute(); } - private function dbRowToMountInfo(array $row) { - $user = $this->userManager->get($row['user_id']); - if (is_null($user)) { - return null; - } + /** + * @param array $row + * @param (callable(CachedMountInfo): string)|null $pathCallback + * @return CachedMountInfo + */ + private function dbRowToMountInfo(array $row, ?callable $pathCallback = null): ICachedMountInfo { + $user = new LazyUser($row['user_id'], $this->userManager); $mount_id = $row['mount_id']; if (!is_null($mount_id)) { $mount_id = (int)$mount_id; } - return new CachedMountInfo( - $user, - (int)$row['storage_id'], - (int)$row['root_id'], - $row['mount_point'], - $row['mount_provider_class'] ?? '', - $mount_id, - isset($row['path']) ? $row['path'] : '', - ); + if ($pathCallback) { + return new LazyPathCachedMountInfo( + $user, + (int)$row['storage_id'], + (int)$row['root_id'], + $row['mount_point'], + $row['mount_provider_class'] ?? '', + $mount_id, + $pathCallback, + ); + } else { + return new CachedMountInfo( + $user, + (int)$row['storage_id'], + (int)$row['root_id'], + $row['mount_point'], + $row['mount_provider_class'] ?? '', + $mount_id, + $row['path'] ?? '', + ); + } } /** @@ -247,20 +250,43 @@ class UserMountCache implements IUserMountCache { * @return ICachedMountInfo[] */ public function getMountsForUser(IUser $user) { - if (!isset($this->mountsForUsers[$user->getUID()])) { + $userUID = $user->getUID(); + if (!$this->userManager->userExists($userUID)) { + return []; + } + if (!isset($this->mountsForUsers[$userUID])) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class') ->from('mounts', 'm') - ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) - ->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($user->getUID()))); + ->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($userUID))); $result = $query->execute(); $rows = $result->fetchAll(); $result->closeCursor(); - $this->mountsForUsers[$user->getUID()] = array_filter(array_map([$this, 'dbRowToMountInfo'], $rows)); + /** @var array<string, ICachedMountInfo> $mounts */ + $mounts = []; + foreach ($rows as $row) { + $mount = $this->dbRowToMountInfo($row, [$this, 'getInternalPathForMountInfo']); + if ($mount !== null) { + $mounts[$mount->getKey()] = $mount; + } + } + $this->mountsForUsers[$userUID] = $mounts; } - return $this->mountsForUsers[$user->getUID()]; + return $this->mountsForUsers[$userUID]; + } + + public function getInternalPathForMountInfo(CachedMountInfo $info): string { + $cached = $this->internalPathCache->get($info->getRootId()); + if ($cached !== null) { + return $cached; + } + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('path') + ->from('filecache') + ->where($builder->expr()->eq('fileid', $builder->createPositionalParameter($info->getRootId()))); + return $query->executeQuery()->fetchOne() ?: ''; } /** @@ -345,30 +371,22 @@ class UserMountCache implements IUserMountCache { } catch (NotFoundException $e) { return []; } - $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') - ->from('mounts', 'm') - ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) - ->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($storageId, IQueryBuilder::PARAM_INT))); - - if ($user) { - $query->andWhere($builder->expr()->eq('user_id', $builder->createPositionalParameter($user))); - } + $mountsForStorage = $this->getMountsForStorageId($storageId, $user); - $result = $query->execute(); - $rows = $result->fetchAll(); - $result->closeCursor(); - // filter mounts that are from the same storage but a different directory - $filteredMounts = array_filter($rows, function (array $row) use ($internalPath, $fileId) { - if ($fileId === (int)$row['root_id']) { + // filter mounts that are from the same storage but not a parent of the file we care about + $filteredMounts = array_filter($mountsForStorage, function (ICachedMountInfo $mount) use ($internalPath, $fileId) { + if ($fileId === $mount->getRootId()) { return true; } - $internalMountPath = $row['path'] ?? ''; + $internalMountPath = $mount->getRootInternalPath(); + + return $internalMountPath === '' || str_starts_with($internalPath, $internalMountPath . '/'); + }); - return $internalMountPath === '' || substr($internalPath, 0, strlen($internalMountPath) + 1) === $internalMountPath . '/'; + $filteredMounts = array_filter($filteredMounts, function (ICachedMountInfo $mount) { + return $this->userManager->userExists($mount->getUser()->getUID()); }); - $filteredMounts = array_filter(array_map([$this, 'dbRowToMountInfo'], $filteredMounts)); return array_map(function (ICachedMountInfo $mount) use ($internalPath) { return new CachedMountFileInfo( $mount->getUser(), @@ -463,7 +481,7 @@ class UserMountCache implements IUserMountCache { }, $mounts); $mounts = array_combine($mountPoints, $mounts); - $current = $path; + $current = rtrim($path, '/'); // walk up the directory tree until we find a path that has a mountpoint set // the loop will return if a mountpoint is found or break if none are found while (true) { diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 3937ee16a7c..81a283e365b 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -6,6 +6,7 @@ * @author Joas Schilling <coding@schilljs.com> * @author Julius Härtl <jus@bitgrid.net> * @author Lukas Reschke <lukas@statuscode.ch> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Piotr M <mrow4a@yahoo.com> * @author Robin Appelman <robin@icewind.nl> @@ -32,12 +33,16 @@ */ namespace OC\Files; -use OCA\Files_Sharing\ISharedStorage; +use OC\Files\Mount\HomeMountPoint; +use OCA\Files_Sharing\External\Mount; +use OCA\Files_Sharing\ISharedMountPoint; use OCP\Files\Cache\ICacheEntry; -use OCP\Files\IHomeStorage; use OCP\Files\Mount\IMountPoint; use OCP\IUser; +/** + * @template-implements \ArrayAccess<string,mixed> + */ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { private array|ICacheEntry $data; /** @@ -121,21 +126,14 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { */ #[\ReturnTypeWillChange] public function offsetGet($offset) { - if ($offset === 'type') { - return $this->getType(); - } elseif ($offset === 'etag') { - return $this->getEtag(); - } elseif ($offset === 'size') { - return $this->getSize(); - } elseif ($offset === 'mtime') { - return $this->getMTime(); - } elseif ($offset === 'permissions') { - return $this->getPermissions(); - } elseif (isset($this->data[$offset])) { - return $this->data[$offset]; - } else { - return null; - } + return match ($offset) { + 'type' => $this->getType(), + 'etag' => $this->getEtag(), + 'size' => $this->getSize(), + 'mtime' => $this->getMTime(), + 'permissions' => $this->getPermissions(), + default => $this->data[$offset] ?? null, + }; } /** @@ -183,7 +181,9 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @return string */ public function getName() { - return isset($this->data['name']) ? $this->data['name'] : basename($this->getPath()); + return empty($this->data['name']) + ? basename($this->getPath()) + : $this->data['name']; } /** @@ -207,7 +207,7 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { if ($includeMounts) { $this->updateEntryfromSubMounts(); - if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + if ($this->isEncrypted() && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { return $this->data['unencrypted_size']; } else { return isset($this->data['size']) ? 0 + $this->data['size'] : 0; @@ -229,11 +229,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @return bool */ public function isEncrypted() { - return $this->data['encrypted']; + return $this->data['encrypted'] ?? false; } /** - * Return the currently version used for the HMAC in the encryption app + * Return the current version used for the HMAC in the encryption app */ public function getEncryptedVersion(): int { return isset($this->data['encryptedVersion']) ? (int) $this->data['encryptedVersion'] : 1; @@ -243,11 +243,7 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @return int */ public function getPermissions() { - $perms = (int) $this->data['permissions']; - if (\OCP\Util::isSharingDisabledForUser() || ($this->isShared() && !\OC\Share\Share::isResharingAllowed())) { - $perms = $perms & ~\OCP\Constants::PERMISSION_SHARE; - } - return $perms; + return (int) $this->data['permissions']; } /** @@ -315,13 +311,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @return bool */ public function isShared() { - $storage = $this->getStorage(); - return $storage->instanceOfStorage(ISharedStorage::class); + return $this->mount instanceof ISharedMountPoint; } public function isMounted() { - $storage = $this->getStorage(); - return !($storage->instanceOfStorage(IHomeStorage::class) || $storage->instanceOfStorage(ISharedStorage::class)); + $isHome = $this->mount instanceof HomeMountPoint; + return !$isHome && !$this->isShared(); } /** @@ -416,4 +411,16 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { public function getUploadTime(): int { return (int) $this->data['upload_time']; } + + public function getParentId(): int { + return $this->data['parent'] ?? -1; + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? []; + } } diff --git a/lib/private/Files/Filesystem.php b/lib/private/Files/Filesystem.php index 5f7c0c403db..c6a5513d5b7 100644 --- a/lib/private/Files/Filesystem.php +++ b/lib/private/Files/Filesystem.php @@ -37,9 +37,9 @@ */ namespace OC\Files; -use OCP\Cache\CappedMemoryCache; use OC\Files\Mount\MountPoint; use OC\User\NoUserException; +use OCP\Cache\CappedMemoryCache; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\Node\FilesystemTornDownEvent; use OCP\Files\Mount\IMountManager; @@ -48,6 +48,7 @@ use OCP\Files\Storage\IStorageFactory; use OCP\IUser; use OCP\IUserManager; use OCP\IUserSession; +use Psr\Log\LoggerInterface; class Filesystem { private static ?Mount\Manager $mounts = null; @@ -200,7 +201,7 @@ class Filesystem { */ public static function addStorageWrapper($wrapperName, $wrapper, $priority = 50) { if (self::$logWarningWhenAddingStorageWrapper) { - \OC::$server->getLogger()->warning("Storage wrapper '{wrapper}' was not registered via the 'OC_Filesystem - preSetup' hook which could cause potential problems.", [ + \OCP\Server::get(LoggerInterface::class)->warning("Storage wrapper '{wrapper}' was not registered via the 'OC_Filesystem - preSetup' hook which could cause potential problems.", [ 'wrapper' => $wrapperName, 'app' => 'filesystem', ]); diff --git a/lib/private/Files/Mount/HomeMountPoint.php b/lib/private/Files/Mount/HomeMountPoint.php new file mode 100644 index 00000000000..0bec12af5c2 --- /dev/null +++ b/lib/private/Files/Mount/HomeMountPoint.php @@ -0,0 +1,49 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\Mount; + +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; + +class HomeMountPoint extends MountPoint { + private IUser $user; + + public function __construct( + IUser $user, + $storage, + string $mountpoint, + array $arguments = null, + IStorageFactory $loader = null, + array $mountOptions = null, + int $mountId = null, + string $mountProvider = null + ) { + parent::__construct($storage, $mountpoint, $arguments, $loader, $mountOptions, $mountId, $mountProvider); + $this->user = $user; + } + + public function getUser(): IUser { + return $this->user; + } +} diff --git a/lib/private/Files/Mount/LocalHomeMountProvider.php b/lib/private/Files/Mount/LocalHomeMountProvider.php index 25a67fc1574..964b607d152 100644 --- a/lib/private/Files/Mount/LocalHomeMountProvider.php +++ b/lib/private/Files/Mount/LocalHomeMountProvider.php @@ -38,6 +38,6 @@ class LocalHomeMountProvider implements IHomeMountProvider { */ public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { $arguments = ['user' => $user]; - return new MountPoint('\OC\Files\Storage\Home', '/' . $user->getUID(), $arguments, $loader, null, null, self::class); + return new HomeMountPoint($user, '\OC\Files\Storage\Home', '/' . $user->getUID(), $arguments, $loader, null, null, self::class); } } diff --git a/lib/private/Files/Mount/Manager.php b/lib/private/Files/Mount/Manager.php index 805cce658a6..2b2de1fbff1 100644 --- a/lib/private/Files/Mount/Manager.php +++ b/lib/private/Files/Mount/Manager.php @@ -10,6 +10,7 @@ declare(strict_types=1); * @author Robin Appelman <robin@icewind.nl> * @author Robin McCorkell <robin@mccorkell.me.uk> * @author Roeland Jago Douma <roeland@famdouma.nl> + * @author Jonas <jonas@freesources.org> * * @license AGPL-3.0 * @@ -29,10 +30,11 @@ declare(strict_types=1); namespace OC\Files\Mount; -use OCP\Cache\CappedMemoryCache; use OC\Files\Filesystem; use OC\Files\SetupManager; use OC\Files\SetupManagerFactory; +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Mount\IMountManager; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; @@ -99,6 +101,15 @@ class Manager implements IMountManager { return $this->pathCache[$path]; } + + + if (count($this->mounts) === 0) { + $this->setupManager->setupRoot(); + if (count($this->mounts) === 0) { + throw new \Exception("No mounts even after explicitly setting up the root mounts"); + } + } + $current = $path; while (true) { $mountPoint = $current . '/'; @@ -115,7 +126,7 @@ class Manager implements IMountManager { } } - throw new NotFoundException("No mount for path " . $path . " existing mounts: " . implode(",", array_keys($this->mounts))); + throw new NotFoundException("No mount for path " . $path . " existing mounts (" . count($this->mounts) ."): " . implode(",", array_keys($this->mounts))); } /** @@ -226,4 +237,21 @@ class Manager implements IMountManager { }); } } + + /** + * Return the mount matching a cached mount info (or mount file info) + * + * @param ICachedMountInfo $info + * + * @return IMountPoint|null + */ + public function getMountFromMountInfo(ICachedMountInfo $info): ?IMountPoint { + $this->setupManager->setupForPath($info->getMountPoint()); + foreach ($this->mounts as $mount) { + if ($mount->getMountPoint() === $info->getMountPoint()) { + return $mount; + } + } + return null; + } } diff --git a/lib/private/Files/Mount/MountPoint.php b/lib/private/Files/Mount/MountPoint.php index f526928cc15..fe6358b32f1 100644 --- a/lib/private/Files/Mount/MountPoint.php +++ b/lib/private/Files/Mount/MountPoint.php @@ -272,7 +272,7 @@ class MountPoint implements IMountPoint { * @return mixed */ public function getOption($name, $default) { - return isset($this->mountOptions[$name]) ? $this->mountOptions[$name] : $default; + return $this->mountOptions[$name] ?? $default; } /** diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 77912adfd34..3593a95c311 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -65,7 +65,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { return null; } - return new MountPoint('\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader, null, null, self::class); + return new HomeMountPoint($user, '\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader, null, null, self::class); } /** @@ -122,7 +122,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { $config['arguments']['bucket'] = ''; } $mapper = new \OC\Files\ObjectStore\Mapper($user, $this->config); - $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $numBuckets = $config['arguments']['num_buckets'] ?? 64; $config['arguments']['bucket'] .= $mapper->getBucket($numBuckets); $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $config['arguments']['bucket']); diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php index ccd10da9d0c..014b66fdf63 100644 --- a/lib/private/Files/Node/Folder.php +++ b/lib/private/Files/Node/Folder.php @@ -177,7 +177,7 @@ class Folder extends Node implements \OCP\Files\Folder { * @throws \OCP\Files\NotPermittedException */ public function newFile($path, $content = null) { - if (empty($path)) { + if ($path === '') { throw new NotPermittedException('Could not create as provided path is empty'); } if ($this->checkPermissions(\OCP\Constants::PERMISSION_CREATE)) { @@ -307,12 +307,16 @@ class Folder extends Node implements \OCP\Files\Folder { /** * @param int $id - * @return \OC\Files\Node\Node[] + * @return \OCP\Files\Node[] */ public function getById($id) { return $this->root->getByIdInPath((int)$id, $this->getPath()); } + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + return current($this->getById($id)) ?: null; + } + protected function getAppDataDirectoryName(): string { $instanceId = \OC::$server->getConfig()->getSystemValueString('instanceid'); return 'appdata_' . $instanceId; diff --git a/lib/private/Files/Node/HookConnector.php b/lib/private/Files/Node/HookConnector.php index a8e76d95c22..d66c7a60664 100644 --- a/lib/private/Files/Node/HookConnector.php +++ b/lib/private/Files/Node/HookConnector.php @@ -6,6 +6,7 @@ declare(strict_types=1); * @copyright Copyright (c) 2016, ownCloud, Inc. * * @author Arthur Schiwon <blizzz@arthur-schiwon.de> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @@ -30,6 +31,7 @@ use OC\Files\Filesystem; use OC\Files\View; use OCP\EventDispatcher\GenericEvent; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Exceptions\AbortedEventException; use OCP\Files\Events\Node\BeforeNodeCopiedEvent; use OCP\Files\Events\Node\BeforeNodeCreatedEvent; use OCP\Files\Events\Node\BeforeNodeDeletedEvent; @@ -46,27 +48,18 @@ use OCP\Files\Events\Node\NodeWrittenEvent; use OCP\Files\FileInfo; use OCP\Files\IRootFolder; use OCP\Util; +use Psr\Log\LoggerInterface; class HookConnector { - /** @var IRootFolder */ - private $root; - - /** @var View */ - private $view; - /** @var FileInfo[] */ - private $deleteMetaCache = []; - - /** @var IEventDispatcher */ - private $dispatcher; + private array $deleteMetaCache = []; public function __construct( - IRootFolder $root, - View $view, - IEventDispatcher $dispatcher) { - $this->root = $root; - $this->view = $view; - $this->dispatcher = $dispatcher; + private IRootFolder $root, + private View $view, + private IEventDispatcher $dispatcher, + private LoggerInterface $logger + ) { } public function viewToNode() { @@ -134,7 +127,12 @@ class HookConnector { $this->dispatcher->dispatch('\OCP\Files::preDelete', new GenericEvent($node)); $event = new BeforeNodeDeletedEvent($node); - $this->dispatcher->dispatchTyped($event); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('delete process aborted', ['exception' => $e]); + } } public function postDelete($arguments) { @@ -172,7 +170,12 @@ class HookConnector { $this->dispatcher->dispatch('\OCP\Files::preRename', new GenericEvent([$source, $target])); $event = new BeforeNodeRenamedEvent($source, $target); - $this->dispatcher->dispatchTyped($event); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('rename process aborted', ['exception' => $e]); + } } public function postRename($arguments) { @@ -192,7 +195,12 @@ class HookConnector { $this->dispatcher->dispatch('\OCP\Files::preCopy', new GenericEvent([$source, $target])); $event = new BeforeNodeCopiedEvent($source, $target); - $this->dispatcher->dispatchTyped($event); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('copy process aborted', ['exception' => $e]); + } } public function postCopy($arguments) { diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php index d495d6f4c57..4aae5cf9804 100644 --- a/lib/private/Files/Node/LazyFolder.php +++ b/lib/private/Files/Node/LazyFolder.php @@ -5,6 +5,7 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -26,10 +27,13 @@ declare(strict_types=1); namespace OC\Files\Node; +use OC\Files\Filesystem; use OC\Files\Utils\PathHelper; -use OCP\Files\Folder; use OCP\Constants; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotPermittedException; /** * Class LazyFolder @@ -41,23 +45,33 @@ use OCP\Files\Mount\IMountPoint; */ class LazyFolder implements Folder { /** @var \Closure(): Folder */ - private $folderClosure; - - /** @var LazyFolder | null */ - protected $folder = null; - + private \Closure $folderClosure; + protected ?Folder $folder = null; + protected IRootFolder $rootFolder; protected array $data; /** - * LazyFolder constructor. - * + * @param IRootFolder $rootFolder * @param \Closure(): Folder $folderClosure + * @param array $data */ - public function __construct(\Closure $folderClosure, array $data = []) { + public function __construct(IRootFolder $rootFolder, \Closure $folderClosure, array $data = []) { + $this->rootFolder = $rootFolder; $this->folderClosure = $folderClosure; $this->data = $data; } + protected function getRootFolder(): IRootFolder { + return $this->rootFolder; + } + + protected function getRealFolder(): Folder { + if ($this->folder === null) { + $this->folder = call_user_func($this->folderClosure); + } + return $this->folder; + } + /** * Magic method to first get the real rootFolder and then * call $method with $args on it @@ -67,11 +81,7 @@ class LazyFolder implements Folder { * @return mixed */ public function __call($method, $args) { - if ($this->folder === null) { - $this->folder = call_user_func($this->folderClosure); - } - - return call_user_func_array([$this->folder, $method], $args); + return call_user_func_array([$this->getRealFolder(), $method], $args); } /** @@ -148,7 +158,7 @@ class LazyFolder implements Folder { * @inheritDoc */ public function get($path) { - return $this->__call(__FUNCTION__, func_get_args()); + return $this->getRootFolder()->get($this->getFullPath($path)); } /** @@ -207,6 +217,9 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getId() { + if (isset($this->data['fileid'])) { + return $this->data['fileid']; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -221,6 +234,9 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getMTime() { + if (isset($this->data['mtime'])) { + return $this->data['mtime']; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -228,6 +244,9 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getSize($includeMounts = true): int|float { + if (isset($this->data['size'])) { + return $this->data['size']; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -235,6 +254,9 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getEtag() { + if (isset($this->data['etag'])) { + return $this->data['etag']; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -299,6 +321,12 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getName() { + if (isset($this->data['path'])) { + return basename($this->data['path']); + } + if (isset($this->data['name'])) { + return $this->data['name']; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -390,6 +418,13 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getFullPath($path) { + if (isset($this->data['path'])) { + $path = PathHelper::normalizePath($path); + if (!Filesystem::isValidPath($path)) { + throw new NotPermittedException('Invalid path "' . $path . '"'); + } + return $this->data['path'] . $path; + } return $this->__call(__FUNCTION__, func_get_args()); } @@ -457,7 +492,11 @@ class LazyFolder implements Folder { * @inheritDoc */ public function getById($id) { - return $this->__call(__FUNCTION__, func_get_args()); + return $this->getRootFolder()->getByIdInPath((int)$id, $this->getPath()); + } + + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + return $this->getRootFolder()->getFirstNodeByIdInPath($id, $this->getPath()); } /** @@ -533,4 +572,19 @@ class LazyFolder implements Folder { public function getRelativePath($path) { return PathHelper::getRelativePath($this->getPath(), $path); } + + public function getParentId(): int { + if (isset($this->data['parent'])) { + return $this->data['parent']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); + } } diff --git a/lib/private/Files/Node/LazyRoot.php b/lib/private/Files/Node/LazyRoot.php index c01b9fdbb83..dd1596319fa 100644 --- a/lib/private/Files/Node/LazyRoot.php +++ b/lib/private/Files/Node/LazyRoot.php @@ -22,7 +22,11 @@ */ namespace OC\Files\Node; +use OCP\Files\Cache\ICacheEntry; use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Node; +use OCP\Files\Node as INode; /** * Class LazyRoot @@ -33,9 +37,18 @@ use OCP\Files\IRootFolder; * @package OC\Files\Node */ class LazyRoot extends LazyFolder implements IRootFolder { - /** - * @inheritDoc - */ + public function __construct(\Closure $folderClosure, array $data = []) { + parent::__construct($this, $folderClosure, $data); + } + + protected function getRootFolder(): IRootFolder { + $folder = $this->getRealFolder(); + if (!$folder instanceof IRootFolder) { + throw new \Exception('Lazy root folder closure didn\'t return a root folder'); + } + return $folder; + } + public function getUserFolder($userId) { return $this->__call(__FUNCTION__, func_get_args()); } @@ -43,4 +56,12 @@ class LazyRoot extends LazyFolder implements IRootFolder { public function getByIdInPath(int $id, string $path) { return $this->__call(__FUNCTION__, func_get_args()); } + + public function getFirstNodeByIdInPath(int $id, string $path): ?Node { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getNodeFromCacheEntryAndMount(ICacheEntry $cacheEntry, IMountPoint $mountPoint): INode { + return $this->getRootFolder()->getNodeFromCacheEntryAndMount($cacheEntry, $mountPoint); + } } diff --git a/lib/private/Files/Node/LazyUserFolder.php b/lib/private/Files/Node/LazyUserFolder.php index 8fbdec4b49d..3cb840ccb6d 100644 --- a/lib/private/Files/Node/LazyUserFolder.php +++ b/lib/private/Files/Node/LazyUserFolder.php @@ -23,30 +23,28 @@ declare(strict_types=1); namespace OC\Files\Node; -use OCP\Files\FileInfo; use OCP\Constants; +use OCP\Files\File; +use OCP\Files\FileInfo; +use OCP\Files\Folder; use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountManager; use OCP\Files\NotFoundException; -use OCP\Files\Folder; -use OCP\Files\File; use OCP\IUser; use Psr\Log\LoggerInterface; class LazyUserFolder extends LazyFolder { - private IRootFolder $root; private IUser $user; private string $path; private IMountManager $mountManager; public function __construct(IRootFolder $rootFolder, IUser $user, IMountManager $mountManager) { - $this->root = $rootFolder; $this->user = $user; $this->mountManager = $mountManager; $this->path = '/' . $user->getUID() . '/files'; - parent::__construct(function () use ($user): Folder { + parent::__construct($rootFolder, function () use ($user): Folder { try { - $node = $this->root->get($this->path); + $node = $this->getRootFolder()->get($this->path); if ($node instanceof File) { $e = new \RuntimeException(); \OCP\Server::get(LoggerInterface::class)->error('User root storage is not a folder: ' . $this->path, [ @@ -56,31 +54,20 @@ class LazyUserFolder extends LazyFolder { } return $node; } catch (NotFoundException $e) { - if (!$this->root->nodeExists('/' . $user->getUID())) { - $this->root->newFolder('/' . $user->getUID()); + if (!$this->getRootFolder()->nodeExists('/' . $user->getUID())) { + $this->getRootFolder()->newFolder('/' . $user->getUID()); } - return $this->root->newFolder($this->path); + return $this->getRootFolder()->newFolder($this->path); } }, [ 'path' => $this->path, - 'permissions' => Constants::PERMISSION_ALL, + // Sharing user root folder is not allowed + 'permissions' => Constants::PERMISSION_ALL ^ Constants::PERMISSION_SHARE, 'type' => FileInfo::TYPE_FOLDER, 'mimetype' => FileInfo::MIMETYPE_FOLDER, ]); } - public function get($path) { - return $this->root->get('/' . $this->user->getUID() . '/files/' . ltrim($path, '/')); - } - - /** - * @param int $id - * @return \OCP\Files\Node[] - */ - public function getById($id) { - return $this->root->getByIdInPath((int)$id, $this->getPath()); - } - public function getMountPoint() { if ($this->folder !== null) { return $this->folder->getMountPoint(); diff --git a/lib/private/Files/Node/Node.php b/lib/private/Files/Node/Node.php index 61ae762638f..acd91c56d3f 100644 --- a/lib/private/Files/Node/Node.php +++ b/lib/private/Files/Node/Node.php @@ -7,6 +7,7 @@ * @author Christoph Wurst <christoph@winzerhof-wurst.at> * @author Joas Schilling <coding@schilljs.com> * @author Julius Härtl <jus@bitgrid.net> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> @@ -43,7 +44,7 @@ use OCP\Files\NotPermittedException; use OCP\Lock\LockedException; use OCP\PreConditionNotMetException; -// FIXME: this class really should be abstract +// FIXME: this class really should be abstract (+1) class Node implements INode { /** * @var \OC\Files\View $view @@ -59,10 +60,7 @@ class Node implements INode { protected ?FileInfo $fileInfo; - /** - * @var Node|null - */ - protected $parent; + protected ?INode $parent; private bool $infoHasSubMountsIncluded; @@ -72,7 +70,7 @@ class Node implements INode { * @param string $path * @param FileInfo $fileInfo */ - public function __construct(IRootFolder $root, $view, $path, $fileInfo = null, ?Node $parent = null, bool $infoHasSubMountsIncluded = true) { + public function __construct(IRootFolder $root, $view, $path, $fileInfo = null, ?INode $parent = null, bool $infoHasSubMountsIncluded = true) { if (Filesystem::normalizePath($view->getRoot()) !== '/') { throw new PreConditionNotMetException('The view passed to the node should not have any fake root set'); } @@ -134,7 +132,14 @@ class Node implements INode { if (method_exists($this->root, 'emit')) { $this->root->emit('\OC\Files', $hook, $args); } - $dispatcher->dispatch('\OCP\Files::' . $hook, new GenericEvent($args)); + + if (in_array($hook, ['preWrite', 'postWrite', 'preCreate', 'postCreate', 'preTouch', 'postTouch', 'preDelete', 'postDelete'], true)) { + $event = new GenericEvent($args[0]); + } else { + $event = new GenericEvent($args); + } + + $dispatcher->dispatch('\OCP\Files::' . $hook, $event); } } @@ -300,7 +305,25 @@ class Node implements INode { return $this->root; } - $this->parent = $this->root->get($newPath); + // Manually fetch the parent if the current node doesn't have a file info yet + try { + $fileInfo = $this->getFileInfo(); + } catch (NotFoundException) { + $this->parent = $this->root->get($newPath); + /** @var \OCP\Files\Folder $this->parent */ + return $this->parent; + } + + // gather the metadata we already know about our parent + $parentData = [ + 'path' => $newPath, + 'fileid' => $fileInfo->getParentId(), + ]; + + // and create lazy folder with it instead of always querying + $this->parent = new LazyFolder($this->root, function () use ($newPath) { + return $this->root->get($newPath); + }, $parentData); } return $this->parent; @@ -328,13 +351,7 @@ class Node implements INode { * @return bool */ public function isValidPath($path) { - if (!$path || $path[0] !== '/') { - $path = '/' . $path; - } - if (strstr($path, '/../') || strrchr($path, '/') === '/..') { - return false; - } - return true; + return Filesystem::isValidPath($path); } public function isMounted() { @@ -477,4 +494,16 @@ class Node implements INode { public function getUploadTime(): int { return $this->getFileInfo()->getUploadTime(); } + + public function getParentId(): int { + return $this->fileInfo->getParentId(); + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } } diff --git a/lib/private/Files/Node/NonExistingFolder.php b/lib/private/Files/Node/NonExistingFolder.php index 34621b18f19..af0ad002f24 100644 --- a/lib/private/Files/Node/NonExistingFolder.php +++ b/lib/private/Files/Node/NonExistingFolder.php @@ -162,6 +162,10 @@ class NonExistingFolder extends Folder { throw new NotFoundException(); } + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + throw new NotFoundException(); + } + public function getFreeSpace() { throw new NotFoundException(); } diff --git a/lib/private/Files/Node/Root.php b/lib/private/Files/Node/Root.php index 7bd88981ff2..71bc7a6da6b 100644 --- a/lib/private/Files/Node/Root.php +++ b/lib/private/Files/Node/Root.php @@ -32,7 +32,6 @@ namespace OC\Files\Node; -use OCP\Cache\CappedMemoryCache; use OC\Files\FileInfo; use OC\Files\Mount\Manager; use OC\Files\Mount\MountPoint; @@ -40,7 +39,9 @@ use OC\Files\Utils\PathHelper; use OC\Files\View; use OC\Hooks\PublicEmitter; use OC\User\NoUserException; +use OCP\Cache\CappedMemoryCache; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\ICacheEntry; use OCP\Files\Config\IUserMountCache; use OCP\Files\Events\Node\FilesystemTornDownEvent; use OCP\Files\IRootFolder; @@ -48,6 +49,8 @@ use OCP\Files\Mount\IMountPoint; use OCP\Files\Node as INode; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; +use OCP\ICache; +use OCP\ICacheFactory; use OCP\IUser; use OCP\IUserManager; use Psr\Log\LoggerInterface; @@ -80,6 +83,7 @@ class Root extends Folder implements IRootFolder { private LoggerInterface $logger; private IUserManager $userManager; private IEventDispatcher $eventDispatcher; + private ICache $pathByIdCache; /** * @param Manager $manager @@ -93,7 +97,8 @@ class Root extends Folder implements IRootFolder { IUserMountCache $userMountCache, LoggerInterface $logger, IUserManager $userManager, - IEventDispatcher $eventDispatcher + IEventDispatcher $eventDispatcher, + ICacheFactory $cacheFactory, ) { parent::__construct($this, $view, ''); $this->mountManager = $manager; @@ -106,6 +111,7 @@ class Root extends Folder implements IRootFolder { $eventDispatcher->addListener(FilesystemTornDownEvent::class, function () { $this->userFolderCache = new CappedMemoryCache(); }); + $this->pathByIdCache = $cacheFactory->createLocal('path-by-id'); } /** @@ -382,7 +388,7 @@ class Root extends Folder implements IRootFolder { try { $folder = $this->get('/' . $userId . '/files'); if (!$folder instanceof \OCP\Files\Folder) { - throw new \Exception("User folder for $userId exists as a file"); + throw new \Exception("Account folder for \"$userId\" exists as a file"); } } catch (NotFoundException $e) { if (!$this->nodeExists('/' . $userId)) { @@ -404,6 +410,31 @@ class Root extends Folder implements IRootFolder { return $this->userMountCache; } + public function getFirstNodeByIdInPath(int $id, string $path): ?INode { + // scope the cache by user, so we don't return nodes for different users + if ($this->user) { + $cachedPath = $this->pathByIdCache->get($this->user->getUID() . '::' . $id); + if ($cachedPath && str_starts_with($path, $cachedPath)) { + // getting the node by path is significantly cheaper than finding it by id + $node = $this->get($cachedPath); + // by validating that the cached path still has the requested fileid we can work around the need to invalidate the cached path + // if the cached path is invalid or a different file now we fall back to the uncached logic + if ($node && $node->getId() === $id) { + return $node; + } + } + } + $node = current($this->getByIdInPath($id, $path)); + if (!$node) { + return null; + } + + if ($this->user) { + $this->pathByIdCache->set($this->user->getUID() . '::' . $id, $node->getPath()); + } + return $node; + } + /** * @param int $id * @return Node[] @@ -487,4 +518,29 @@ class Root extends Folder implements IRootFolder { }); return $folders; } + + public function getNodeFromCacheEntryAndMount(ICacheEntry $cacheEntry, IMountPoint $mountPoint): INode { + $path = $cacheEntry->getPath(); + $fullPath = $mountPoint->getMountPoint() . $path; + // todo: LazyNode? + $info = new FileInfo($fullPath, $mountPoint->getStorage(), $path, $cacheEntry, $mountPoint); + $parentPath = dirname($fullPath); + $parent = new LazyFolder($this, function () use ($parentPath) { + $parent = $this->get($parentPath); + if ($parent instanceof \OCP\Files\Folder) { + return $parent; + } else { + throw new \Exception("parent $parentPath is not a folder"); + } + }, [ + 'path' => $parentPath, + ]); + $isDir = $info->getType() === FileInfo::TYPE_FOLDER; + $view = new View(''); + if ($isDir) { + return new Folder($this, $view, $path, $info, $parent); + } else { + return new File($this, $view, $path, $info, $parent); + } + } } diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php index 2f6db935236..2c473cb6da9 100644 --- a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -26,9 +26,12 @@ declare(strict_types=1); namespace OC\Files\ObjectStore; class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { - /** @var string */ - private $internalId; + private string $internalId; + /** + * @param array $params + * @throws \Exception + */ public function __construct($params) { if (!isset($params['internal-id'])) { throw new \Exception('missing id in parameters'); @@ -37,7 +40,7 @@ class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { parent::__construct($params); } - public function getId() { + public function getId(): string { return 'object::appdata::preview:' . $this->internalId; } } diff --git a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php index 824adcc1d0e..443c5147742 100644 --- a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php @@ -7,6 +7,7 @@ * @author Jörn Friedrich Dreyer <jfd@butonic.de> * @author Morris Jobke <hey@morrisjobke.de> * @author Roeland Jago Douma <roeland@famdouma.nl> + * @author Thomas Citharel <nextcloud@tcit.fr> * * @license AGPL-3.0 * @@ -25,22 +26,28 @@ */ namespace OC\Files\ObjectStore; -use OC\User\User; +use Exception; +use OCP\Files\IHomeStorage; +use OCP\IUser; + +class HomeObjectStoreStorage extends ObjectStoreStorage implements IHomeStorage { + protected IUser $user; -class HomeObjectStoreStorage extends ObjectStoreStorage implements \OCP\Files\IHomeStorage { /** * The home user storage requires a user object to create a unique storage id + * * @param array $params + * @throws Exception */ public function __construct($params) { - if (! isset($params['user']) || ! $params['user'] instanceof User) { - throw new \Exception('missing user object in parameters'); + if (! isset($params['user']) || ! $params['user'] instanceof IUser) { + throw new Exception('missing user object in parameters'); } $this->user = $params['user']; parent::__construct($params); } - public function getId() { + public function getId(): string { return 'object::user:' . $this->user->getUID(); } @@ -48,20 +55,13 @@ class HomeObjectStoreStorage extends ObjectStoreStorage implements \OCP\Files\IH * get the owner of a path * * @param string $path The path to get the owner - * @return false|string uid + * @return string uid */ - public function getOwner($path) { - if (is_object($this->user)) { - return $this->user->getUID(); - } - return false; + public function getOwner($path): string { + return $this->user->getUID(); } - /** - * @param string $path, optional - * @return \OC\User\User - */ - public function getUser($path = null) { + public function getUser(): IUser { return $this->user; } } diff --git a/lib/private/Files/ObjectStore/ObjectStoreScanner.php b/lib/private/Files/ObjectStore/ObjectStoreScanner.php index f001f90fdaa..8a9b844c47f 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreScanner.php +++ b/lib/private/Files/ObjectStore/ObjectStoreScanner.php @@ -39,7 +39,7 @@ class ObjectStoreScanner extends Scanner { return []; } - protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int $oldSize) { + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { return 0; } diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index d918bd98729..7eb284fc774 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -47,30 +47,24 @@ use OCP\Files\ObjectStore\IObjectStore; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; - /** - * @var \OCP\Files\ObjectStore\IObjectStore $objectStore - */ - protected $objectStore; - /** - * @var string $id - */ - protected $id; - /** - * @var \OC\User\User $user - */ - protected $user; + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; - private $objectPrefix = 'urn:oid:'; + private LoggerInterface $logger; - private $logger; - - /** @var bool */ - protected $validateWrites = true; + private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + /** + * @param array $params + * @throws \Exception + */ public function __construct($params) { if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { $this->objectStore = $params['objectstore']; @@ -88,8 +82,9 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if (isset($params['validateWrites'])) { $this->validateWrites = (bool)$params['validateWrites']; } + $this->handleCopiesAsOwned = (bool)($params['handleCopiesAsOwned'] ?? false); - $this->logger = \OC::$server->getLogger(); + $this->logger = \OCP\Server::get(LoggerInterface::class); } public function mkdir($path, bool $force = false) { @@ -225,10 +220,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $this->objectStore->deleteObject($this->getURN($entry->getId())); } catch (\Exception $ex) { if ($ex->getCode() !== 404) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), - ]); + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); return false; } //removing from cache is ok as it does not exist in the objectstore anyway @@ -291,7 +289,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return IteratorDirectory::wrap($files); } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); return false; } } @@ -341,16 +339,22 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } return $handle; } catch (NotFoundException $e) { - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw $e; - } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); return false; } } else { @@ -447,10 +451,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil ]; $this->getCache()->put($path, $stat); } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object for ' . $path, - ]); + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); throw $ex; } } @@ -545,20 +552,28 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil * Else people lose access to existing files */ $this->getCache()->remove($uploadPath); - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } else { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not update object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } throw $ex; // make this bubble up } if ($exists) { + // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways + $stat['unencrypted_size'] = $stat['size']; $this->getCache()->update($fileId, $stat); } else { if (!$this->validateWrites || $this->objectStore->objectExists($urn)) { @@ -649,6 +664,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil try { $this->objectStore->copyObject($sourceUrn, $targetUrn); + if ($this->handleCopiesAsOwned) { + // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage ! + $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]); + } } catch (\Exception $e) { $cache->remove($to); @@ -712,10 +731,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } catch (S3MultipartUploadException|S3Exception $e) { $this->objectStore->abortMultipartUpload($urn, $writeToken); - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, - ]); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw new GenericFileException('Could not write chunked file'); } return $size; diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index deb03571c76..a1edfa1eb99 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -71,6 +71,11 @@ trait S3ConnectionTrait { /** @var int */ private $putSizeLimit; + /** @var int */ + private $copySizeLimit; + + private bool $useMultipartCopy = true; + protected $test; protected function parseParams($params) { @@ -87,12 +92,14 @@ trait S3ConnectionTrait { $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600; + $this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000; + $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true); $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } - $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; + $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true; $this->params = $params; } @@ -128,7 +135,7 @@ trait S3ConnectionTrait { ); $options = [ - 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest', + 'version' => $this->params['version'] ?? 'latest', 'credentials' => $provider, 'endpoint' => $base_url, 'region' => $this->params['region'], diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index e0d0f2ce9c7..623c4d08c74 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -27,6 +27,7 @@ namespace OC\Files\ObjectStore; use Aws\S3\Exception\S3MultipartUploadException; +use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; use Aws\S3\S3Client; use GuzzleHttp\Psr7; @@ -189,9 +190,31 @@ trait S3ObjectTrait { return $this->getConnection()->doesObjectExist($this->bucket, $urn, $this->getSSECParameters()); } - public function copyObject($from, $to) { - $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ - 'params' => $this->getSSECParameters() + $this->getSSECParameters(true) - ]); + public function copyObject($from, $to, array $options = []) { + $sourceMetadata = $this->getConnection()->headObject([ + 'Bucket' => $this->getBucket(), + 'Key' => $from, + ] + $this->getSSECParameters()); + + $size = (int)($sourceMetadata->get('Size') ?? $sourceMetadata->get('ContentLength')); + + if ($this->useMultipartCopy && $size > $this->copySizeLimit) { + $copy = new MultipartCopy($this->getConnection(), [ + "source_bucket" => $this->getBucket(), + "source_key" => $from + ], array_merge([ + "bucket" => $this->getBucket(), + "key" => $to, + "acl" => "private", + "params" => $this->getSSECParameters() + $this->getSSECParameters(true), + "source_metadata" => $sourceMetadata + ], $options)); + $copy->copy(); + } else { + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([ + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'mup_threshold' => PHP_INT_MAX, + ], $options)); + } } } diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php new file mode 100644 index 00000000000..c573e3af3c0 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php @@ -0,0 +1,29 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +class FlattenNestedBool extends QueryOptimizerStep { + public function processOperator(ISearchOperator &$operator) { + if ( + $operator instanceof SearchBinaryOperator && ( + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR || + $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + ) + ) { + $newArguments = []; + foreach ($operator->getArguments() as $oldArgument) { + if ($oldArgument instanceof SearchBinaryOperator && $oldArgument->getType() === $operator->getType()) { + $newArguments = array_merge($newArguments, $oldArgument->getArguments()); + } else { + $newArguments[] = $oldArgument; + } + } + $operator->setArguments($newArguments); + } + parent::processOperator($operator); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php new file mode 100644 index 00000000000..2c32f2e0174 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php @@ -0,0 +1,27 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * replace single argument AND and OR operations with their single argument + */ +class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + parent::processOperator($operator); + if ( + $operator instanceof ISearchBinaryOperator && + count($operator->getArguments()) === 1 && + ( + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR || + $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + ) + ) { + $operator = $operator->getArguments()[0]; + return true; + } + return false; + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php b/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php new file mode 100644 index 00000000000..7986df82adc --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php @@ -0,0 +1,95 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * Attempt to transform + * + * (A AND B) OR (A AND C) OR (A AND D AND E) into A AND (B OR C OR (D AND E)) + * + * This is always valid because logical 'AND' and 'OR' are distributive[1]. + * + * [1]: https://en.wikipedia.org/wiki/Distributive_property + */ +class MergeDistributiveOperations extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof SearchBinaryOperator) { + // either 'AND' or 'OR' + $topLevelType = $operator->getType(); + + // split the arguments into groups that share a first argument + $groups = $this->groupBinaryOperatorsByChild($operator->getArguments(), 0); + $outerOperations = array_map(function (array $operators) use ($topLevelType) { + // no common operations, no need to change anything + if (count($operators) === 1) { + return $operators[0]; + } + + // for groups with size >1 we know they are binary operators with at least 1 child + /** @var ISearchBinaryOperator $firstArgument */ + $firstArgument = $operators[0]; + + // we already checked that all arguments have the same type, so this type applies for all, either 'AND' or 'OR' + $innerType = $firstArgument->getType(); + + // the common operation we move out ('A' from the example) + $extractedLeftHand = $firstArgument->getArguments()[0]; + + // for each argument we remove the extracted operation to get the leftovers ('B', 'C' and '(D AND E)' in the example) + // note that we leave them inside the "inner" binary operation for when the "inner" operation contained more than two parts + // in the (common) case where the "inner" operation only has 1 item left it will be cleaned up in a follow up step + $rightHandArguments = array_map(function (ISearchOperator $inner) { + /** @var ISearchBinaryOperator $inner */ + $arguments = $inner->getArguments(); + array_shift($arguments); + if (count($arguments) === 1) { + return $arguments[0]; + } + return new SearchBinaryOperator($inner->getType(), $arguments); + }, $operators); + + // combine the extracted operation ('A') with the remaining bit ('(B OR C OR (D AND E))') + // note that because of how distribution work, we use the "outer" type "inside" and the "inside" type "outside". + $extractedRightHand = new SearchBinaryOperator($topLevelType, $rightHandArguments); + return new SearchBinaryOperator( + $innerType, + [$extractedLeftHand, $extractedRightHand] + ); + }, $groups); + + // combine all groups again + $operator = new SearchBinaryOperator($topLevelType, $outerOperations); + parent::processOperator($operator); + return true; + } + return parent::processOperator($operator); + } + + /** + * Group a list of binary search operators that have a common argument + * + * Non-binary operators, or empty binary operators will each get their own 1-sized group + * + * @param ISearchOperator[] $operators + * @return ISearchOperator[][] + */ + private function groupBinaryOperatorsByChild(array $operators, int $index = 0): array { + $result = []; + foreach ($operators as $operator) { + if ($operator instanceof ISearchBinaryOperator && count($operator->getArguments()) > 0) { + /** @var SearchBinaryOperator|SearchComparison $child */ + $child = $operator->getArguments()[$index]; + $childKey = (string)$child; + $result[$childKey][] = $operator; + } else { + $result[] = [$operator]; + } + } + return array_values($result); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php new file mode 100644 index 00000000000..d39eb2e29a9 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php @@ -0,0 +1,70 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * transform (field == A OR field == B ...) into field IN (A, B, ...) + */ +class OrEqualsToIn extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ( + $operator instanceof ISearchBinaryOperator && + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + ) { + $groups = $this->groupEqualsComparisonsByField($operator->getArguments()); + $newParts = array_map(function (array $group) { + if (count($group) > 1) { + // because of the logic from `groupEqualsComparisonsByField` we now that group is all comparisons on the same field + /** @var ISearchComparison[] $group */ + $field = $group[0]->getField(); + $values = array_map(function (ISearchComparison $comparison) { + /** @var string|integer|bool|\DateTime $value */ + $value = $comparison->getValue(); + return $value; + }, $group); + $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values); + $pathEqHash = array_reduce($group, function ($pathEqHash, ISearchComparison $comparison) { + return $comparison->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true) && $pathEqHash; + }, true); + $in->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, $pathEqHash); + return $in; + } else { + return $group[0]; + } + }, $groups); + if (count($newParts) === 1) { + $operator = $newParts[0]; + } else { + $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $newParts); + } + parent::processOperator($operator); + return true; + } + parent::processOperator($operator); + return false; + } + + /** + * Non-equals operators are put in a separate group for each + * + * @param ISearchOperator[] $operators + * @return ISearchOperator[][] + */ + private function groupEqualsComparisonsByField(array $operators): array { + $result = []; + foreach ($operators as $operator) { + if ($operator instanceof ISearchComparison && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + $result[$operator->getField()][] = $operator; + } else { + $result[] = [$operator]; + } + } + return array_values($result); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php index 0caa9b12a02..664402f1238 100644 --- a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php +++ b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php @@ -4,6 +4,9 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> + * @author Robin Appelman <robin@icewind.nl> + * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -48,7 +51,7 @@ class PathPrefixOptimizer extends QueryOptimizerStep { } public function processOperator(ISearchOperator &$operator) { - if (!$this->useHashEq && $operator instanceof ISearchComparison && $operator->getField() === 'path' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + if (!$this->useHashEq && $operator instanceof ISearchComparison && !$operator->getExtra() && $operator->getField() === 'path' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { $operator->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, false); } @@ -69,7 +72,7 @@ class PathPrefixOptimizer extends QueryOptimizerStep { private function operatorPairIsPathPrefix(ISearchOperator $like, ISearchOperator $equal): bool { return ( $like instanceof ISearchComparison && $equal instanceof ISearchComparison && - $like->getField() === 'path' && $equal->getField() === 'path' && + !$like->getExtra() && !$equal->getExtra() && $like->getField() === 'path' && $equal->getField() === 'path' && $like->getType() === ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE && $equal->getType() === ISearchComparison::COMPARE_EQUAL && $like->getValue() === SearchComparison::escapeLikeParameter($equal->getValue()) . '/%' ); diff --git a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php index 160b27b7f11..46d27e38081 100644 --- a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php +++ b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php @@ -29,15 +29,20 @@ class QueryOptimizer { /** @var QueryOptimizerStep[] */ private $steps = []; - public function __construct( - PathPrefixOptimizer $pathPrefixOptimizer - ) { + public function __construct() { + // note that the order here is relevant $this->steps = [ - $pathPrefixOptimizer + new PathPrefixOptimizer(), + new MergeDistributiveOperations(), + new FlattenSingleArgumentBinaryOperation(), + new FlattenNestedBool(), + new OrEqualsToIn(), + new FlattenNestedBool(), + new SplitLargeIn(), ]; } - public function processOperator(ISearchOperator $operator) { + public function processOperator(ISearchOperator &$operator) { foreach ($this->steps as $step) { $step->inspectOperator($operator); } diff --git a/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php b/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php new file mode 100644 index 00000000000..473f8a87151 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php @@ -0,0 +1,33 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * Optimizer step that can replace the $operator altogether instead of just modifying it + * These steps need some extra logic to properly replace the arguments of binary operators + */ +class ReplacingOptimizerStep extends QueryOptimizerStep { + /** + * Allow optimizer steps to modify query operators + * + * Returns true if the reference $operator points to a new value + */ + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof SearchBinaryOperator) { + $modified = false; + $arguments = $operator->getArguments(); + foreach ($arguments as &$argument) { + if ($this->processOperator($argument)) { + $modified = true; + } + } + if ($modified) { + $operator->setArguments($arguments); + } + } + return false; + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php new file mode 100644 index 00000000000..6f85037b3e9 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php @@ -0,0 +1,32 @@ +<?php + +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * transform IN (1000+ element) into (IN (1000 elements) OR IN(...)) + */ +class SplitLargeIn extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ( + $operator instanceof ISearchComparison && + $operator->getType() === ISearchComparison::COMPARE_IN && + count($operator->getValue()) > 1000 + ) { + $chunks = array_chunk($operator->getValue(), 1000); + $chunkComparisons = array_map(function (array $values) use ($operator) { + return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values); + }, $chunks); + + $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons); + return true; + } + parent::processOperator($operator); + return false; + } +} diff --git a/lib/private/Files/Search/SearchBinaryOperator.php b/lib/private/Files/Search/SearchBinaryOperator.php index d7bba8f1b4e..2b01ad58e5f 100644 --- a/lib/private/Files/Search/SearchBinaryOperator.php +++ b/lib/private/Files/Search/SearchBinaryOperator.php @@ -28,7 +28,7 @@ use OCP\Files\Search\ISearchOperator; class SearchBinaryOperator implements ISearchBinaryOperator { /** @var string */ private $type; - /** @var ISearchOperator[] */ + /** @var (SearchBinaryOperator|SearchComparison)[] */ private $arguments; private $hints = []; @@ -36,7 +36,7 @@ class SearchBinaryOperator implements ISearchBinaryOperator { * SearchBinaryOperator constructor. * * @param string $type - * @param ISearchOperator[] $arguments + * @param (SearchBinaryOperator|SearchComparison)[] $arguments */ public function __construct($type, array $arguments) { $this->type = $type; @@ -57,6 +57,14 @@ class SearchBinaryOperator implements ISearchBinaryOperator { return $this->arguments; } + /** + * @param ISearchOperator[] $arguments + * @return void + */ + public function setArguments(array $arguments): void { + $this->arguments = $arguments; + } + public function getQueryHint(string $name, $default) { return $this->hints[$name] ?? $default; } @@ -64,4 +72,11 @@ class SearchBinaryOperator implements ISearchBinaryOperator { public function setQueryHint(string $name, $value): void { $this->hints[$name] = $value; } + + public function __toString(): string { + if ($this->type === ISearchBinaryOperator::OPERATOR_NOT) { + return '(not ' . $this->arguments[0] . ')'; + } + return '(' . implode(' ' . $this->type . ' ', $this->arguments) . ')'; + } } diff --git a/lib/private/Files/Search/SearchComparison.php b/lib/private/Files/Search/SearchComparison.php index 122a1f730b4..b3c4832d776 100644 --- a/lib/private/Files/Search/SearchComparison.php +++ b/lib/private/Files/Search/SearchComparison.php @@ -1,7 +1,10 @@ <?php + +declare(strict_types=1); /** * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -24,47 +27,45 @@ namespace OC\Files\Search; use OCP\Files\Search\ISearchComparison; +/** + * @psalm-import-type ParamValue from ISearchComparison + */ class SearchComparison implements ISearchComparison { - /** @var string */ - private $type; - /** @var string */ - private $field; - /** @var string|integer|\DateTime */ - private $value; - private $hints = []; + private array $hints = []; - /** - * SearchComparison constructor. - * - * @param string $type - * @param string $field - * @param \DateTime|int|string $value - */ - public function __construct($type, $field, $value) { - $this->type = $type; - $this->field = $field; - $this->value = $value; + public function __construct( + private string $type, + private string $field, + /** @var ParamValue $value */ + private \DateTime|int|string|bool|array $value, + private string $extra = '' + ) { } /** * @return string */ - public function getType() { + public function getType(): string { return $this->type; } /** * @return string */ - public function getField() { + public function getField(): string { return $this->field; } + public function getValue(): string|int|bool|\DateTime|array { + return $this->value; + } + /** - * @return \DateTime|int|string + * @return string + * @since 28.0.0 */ - public function getValue() { - return $this->value; + public function getExtra(): string { + return $this->extra; } public function getQueryHint(string $name, $default) { @@ -78,4 +79,8 @@ class SearchComparison implements ISearchComparison { public static function escapeLikeParameter(string $param): string { return addcslashes($param, '\\_%'); } + + public function __toString(): string { + return $this->field . ' ' . $this->type . ' ' . json_encode($this->value); + } } diff --git a/lib/private/Files/Search/SearchOrder.php b/lib/private/Files/Search/SearchOrder.php index 1395a87ac72..de514262bf5 100644 --- a/lib/private/Files/Search/SearchOrder.php +++ b/lib/private/Files/Search/SearchOrder.php @@ -2,6 +2,7 @@ /** * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -26,34 +27,33 @@ use OCP\Files\FileInfo; use OCP\Files\Search\ISearchOrder; class SearchOrder implements ISearchOrder { - /** @var string */ - private $direction; - /** @var string */ - private $field; + public function __construct( + private string $direction, + private string $field, + private string $extra = '' + ) { + } /** - * SearchOrder constructor. - * - * @param string $direction - * @param string $field + * @return string */ - public function __construct($direction, $field) { - $this->direction = $direction; - $this->field = $field; + public function getDirection(): string { + return $this->direction; } /** * @return string */ - public function getDirection() { - return $this->direction; + public function getField(): string { + return $this->field; } /** * @return string + * @since 28.0.0 */ - public function getField() { - return $this->field; + public function getExtra(): string { + return $this->extra; } public function sortFileInfo(FileInfo $a, FileInfo $b): int { diff --git a/lib/private/Files/SetupManager.php b/lib/private/Files/SetupManager.php index b44ead003a8..93b7dc37b6b 100644 --- a/lib/private/Files/SetupManager.php +++ b/lib/private/Files/SetupManager.php @@ -24,8 +24,8 @@ declare(strict_types=1); namespace OC\Files; use OC\Files\Config\MountProviderCollection; +use OC\Files\Mount\HomeMountPoint; use OC\Files\Mount\MountPoint; -use OC\Files\ObjectStore\HomeObjectStoreStorage; use OC\Files\Storage\Common; use OC\Files\Storage\Home; use OC\Files\Storage\Storage; @@ -34,9 +34,15 @@ use OC\Files\Storage\Wrapper\Encoding; use OC\Files\Storage\Wrapper\PermissionsMask; use OC\Files\Storage\Wrapper\Quota; use OC\Lockdown\Filesystem\NullStorage; +use OC\Share\Share; +use OC\Share20\ShareDisableChecker; use OC_App; use OC_Hook; use OC_Util; +use OCA\Files_External\Config\ConfigAdapter; +use OCA\Files_Sharing\External\Mount; +use OCA\Files_Sharing\ISharedMountPoint; +use OCA\Files_Sharing\SharedMount; use OCP\Constants; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; @@ -64,52 +70,35 @@ use Psr\Log\LoggerInterface; class SetupManager { private bool $rootSetup = false; - private IEventLogger $eventLogger; - private MountProviderCollection $mountProviderCollection; - private IMountManager $mountManager; - private IUserManager $userManager; // List of users for which at least one mount is setup private array $setupUsers = []; // List of users for which all mounts are setup private array $setupUsersComplete = []; /** @var array<string, string[]> */ private array $setupUserMountProviders = []; - private IEventDispatcher $eventDispatcher; - private IUserMountCache $userMountCache; - private ILockdownManager $lockdownManager; - private IUserSession $userSession; private ICache $cache; - private LoggerInterface $logger; - private IConfig $config; private bool $listeningForProviders; private array $fullSetupRequired = []; private bool $setupBuiltinWrappersDone = false; + private bool $forceFullSetup = false; public function __construct( - IEventLogger $eventLogger, - MountProviderCollection $mountProviderCollection, - IMountManager $mountManager, - IUserManager $userManager, - IEventDispatcher $eventDispatcher, - IUserMountCache $userMountCache, - ILockdownManager $lockdownManager, - IUserSession $userSession, + private IEventLogger $eventLogger, + private MountProviderCollection $mountProviderCollection, + private IMountManager $mountManager, + private IUserManager $userManager, + private IEventDispatcher $eventDispatcher, + private IUserMountCache $userMountCache, + private ILockdownManager $lockdownManager, + private IUserSession $userSession, ICacheFactory $cacheFactory, - LoggerInterface $logger, - IConfig $config + private LoggerInterface $logger, + private IConfig $config, + private ShareDisableChecker $shareDisableChecker, ) { - $this->eventLogger = $eventLogger; - $this->mountProviderCollection = $mountProviderCollection; - $this->mountManager = $mountManager; - $this->userManager = $userManager; - $this->eventDispatcher = $eventDispatcher; - $this->userMountCache = $userMountCache; - $this->lockdownManager = $lockdownManager; - $this->logger = $logger; - $this->userSession = $userSession; $this->cache = $cacheFactory->createDistributed('setupmanager::'); $this->listeningForProviders = false; - $this->config = $config; + $this->forceFullSetup = $this->config->getSystemValueBool('debug.force-full-fs-setup'); $this->setupListeners(); } @@ -133,52 +122,55 @@ class SetupManager { $prevLogging = Filesystem::logWarningWhenAddingStorageWrapper(false); Filesystem::addStorageWrapper('mount_options', function ($mountPoint, IStorage $storage, IMountPoint $mount) { - if ($storage->instanceOfStorage(Common::class)) { + if ($mount->getOptions() && $storage->instanceOfStorage(Common::class)) { $storage->setMountOptions($mount->getOptions()); } return $storage; }); - Filesystem::addStorageWrapper('enable_sharing', function ($mountPoint, IStorage $storage, IMountPoint $mount) { - if (!$mount->getOption('enable_sharing', true)) { - return new PermissionsMask([ - 'storage' => $storage, - 'mask' => Constants::PERMISSION_ALL - Constants::PERMISSION_SHARE, - ]); + $reSharingEnabled = Share::isResharingAllowed(); + $user = $this->userSession->getUser(); + $sharingEnabledForUser = $user ? !$this->shareDisableChecker->sharingDisabledForUser($user->getUID()) : true; + Filesystem::addStorageWrapper( + 'sharing_mask', + function ($mountPoint, IStorage $storage, IMountPoint $mount) use ($reSharingEnabled, $sharingEnabledForUser) { + $sharingEnabledForMount = $mount->getOption('enable_sharing', true); + $isShared = $mount instanceof ISharedMountPoint; + if (!$sharingEnabledForMount || !$sharingEnabledForUser || (!$reSharingEnabled && $isShared)) { + return new PermissionsMask([ + 'storage' => $storage, + 'mask' => Constants::PERMISSION_ALL - Constants::PERMISSION_SHARE, + ]); + } + return $storage; } - return $storage; - }); + ); // install storage availability wrapper, before most other wrappers - Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, IStorage $storage) { - if (!$storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage') && !$storage->isLocal()) { + Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, IStorage $storage, IMountPoint $mount) { + $externalMount = $mount instanceof ConfigAdapter || $mount instanceof Mount; + if ($externalMount && !$storage->isLocal()) { return new Availability(['storage' => $storage]); } return $storage; }); Filesystem::addStorageWrapper('oc_encoding', function ($mountPoint, IStorage $storage, IMountPoint $mount) { - if ($mount->getOption('encoding_compatibility', false) && !$storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage')) { + if ($mount->getOption('encoding_compatibility', false) && !$mount instanceof SharedMount) { return new Encoding(['storage' => $storage]); } return $storage; }); $quotaIncludeExternal = $this->config->getSystemValue('quota_include_external_storage', false); - Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage) use ($quotaIncludeExternal) { + Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage, IMountPoint $mount) use ($quotaIncludeExternal) { // set up quota for home storages, even for other users // which can happen when using sharing - - /** - * @var Storage $storage - */ - if ($storage->instanceOfStorage(HomeObjectStoreStorage::class) || $storage->instanceOfStorage(Home::class)) { - if (is_object($storage->getUser())) { - $user = $storage->getUser(); - return new Quota(['storage' => $storage, 'quotaCallback' => function () use ($user) { - return OC_Util::getUserQuota($user); - }, 'root' => 'files', 'include_external_storage' => $quotaIncludeExternal]); - } + if ($mount instanceof HomeMountPoint) { + $user = $mount->getUser(); + return new Quota(['storage' => $storage, 'quotaCallback' => function () use ($user) { + return OC_Util::getUserQuota($user); + }, 'root' => 'files', 'include_external_storage' => $quotaIncludeExternal]); } return $storage; @@ -345,12 +337,13 @@ class SetupManager { if ($this->rootSetup) { return; } + + $this->setupBuiltinWrappers(); + $this->rootSetup = true; $this->eventLogger->start('fs:setup:root', 'Setup root filesystem'); - $this->setupBuiltinWrappers(); - $rootMounts = $this->mountProviderCollection->getRootMounts(); foreach ($rootMounts as $rootMountProvider) { $this->mountManager->addMount($rootMountProvider); @@ -479,6 +472,10 @@ class SetupManager { } private function fullSetupRequired(IUser $user): bool { + if ($this->forceFullSetup) { + return true; + } + // we perform a "cached" setup only after having done the full setup recently // this is also used to trigger a full setup after handling events that are likely // to change the available mounts diff --git a/lib/private/Files/SetupManagerFactory.php b/lib/private/Files/SetupManagerFactory.php index 1d9efbd411f..8589cbdea42 100644 --- a/lib/private/Files/SetupManagerFactory.php +++ b/lib/private/Files/SetupManagerFactory.php @@ -23,6 +23,7 @@ declare(strict_types=1); namespace OC\Files; +use OC\Share20\ShareDisableChecker; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Config\IMountProviderCollection; @@ -36,40 +37,21 @@ use OCP\Lockdown\ILockdownManager; use Psr\Log\LoggerInterface; class SetupManagerFactory { - private IEventLogger $eventLogger; - private IMountProviderCollection $mountProviderCollection; - private IUserManager $userManager; - private IEventDispatcher $eventDispatcher; - private IUserMountCache $userMountCache; - private ILockdownManager $lockdownManager; - private IUserSession $userSession; private ?SetupManager $setupManager; - private ICacheFactory $cacheFactory; - private LoggerInterface $logger; - private IConfig $config; public function __construct( - IEventLogger $eventLogger, - IMountProviderCollection $mountProviderCollection, - IUserManager $userManager, - IEventDispatcher $eventDispatcher, - IUserMountCache $userMountCache, - ILockdownManager $lockdownManager, - IUserSession $userSession, - ICacheFactory $cacheFactory, - LoggerInterface $logger, - IConfig $config + private IEventLogger $eventLogger, + private IMountProviderCollection $mountProviderCollection, + private IUserManager $userManager, + private IEventDispatcher $eventDispatcher, + private IUserMountCache $userMountCache, + private ILockdownManager $lockdownManager, + private IUserSession $userSession, + private ICacheFactory $cacheFactory, + private LoggerInterface $logger, + private IConfig $config, + private ShareDisableChecker $shareDisableChecker, ) { - $this->eventLogger = $eventLogger; - $this->mountProviderCollection = $mountProviderCollection; - $this->userManager = $userManager; - $this->eventDispatcher = $eventDispatcher; - $this->userMountCache = $userMountCache; - $this->lockdownManager = $lockdownManager; - $this->userSession = $userSession; - $this->cacheFactory = $cacheFactory; - $this->logger = $logger; - $this->config = $config; $this->setupManager = null; } @@ -86,7 +68,8 @@ class SetupManagerFactory { $this->userSession, $this->cacheFactory, $this->logger, - $this->config + $this->config, + $this->shareDisableChecker, ); } return $this->setupManager; diff --git a/lib/private/Files/SimpleFS/SimpleFolder.php b/lib/private/Files/SimpleFS/SimpleFolder.php index 4d24aa138c1..2c1f23f8e44 100644 --- a/lib/private/Files/SimpleFS/SimpleFolder.php +++ b/lib/private/Files/SimpleFS/SimpleFolder.php @@ -28,8 +28,8 @@ use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\Node; use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\Files\SimpleFS\ISimpleFile; +use OCP\Files\SimpleFS\ISimpleFolder; class SimpleFolder implements ISimpleFolder { /** @var Folder */ diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php index 5ab411434d0..0d4e8d29295 100644 --- a/lib/private/Files/Storage/Common.php +++ b/lib/private/Files/Storage/Common.php @@ -43,6 +43,7 @@ namespace OC\Files\Storage; use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; use OC\Files\Cache\Propagator; use OC\Files\Cache\Scanner; use OC\Files\Cache\Updater; @@ -61,8 +62,10 @@ use OCP\Files\ReservedWordException; use OCP\Files\Storage\ILockingStorage; use OCP\Files\Storage\IStorage; use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Files\StorageNotAvailableException; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; +use OCP\Server; use Psr\Log\LoggerInterface; /** @@ -338,12 +341,20 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { return $this->filemtime($path) > $time; } + protected function getCacheDependencies(): CacheDependencies { + static $dependencies = null; + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + return $dependencies; + } + public function getCache($path = '', $storage = null) { if (!$storage) { $storage = $this; } if (!isset($storage->cache)) { - $storage->cache = new Cache($storage); + $storage->cache = new Cache($storage, $this->getCacheDependencies()); } return $storage->cache; } @@ -398,13 +409,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { } public function getStorageCache($storage = null) { - if (!$storage) { - $storage = $this; - } - if (!isset($this->storageCache)) { - $this->storageCache = new \OC\Files\Cache\Storage($storage); - } - return $this->storageCache; + return $this->getCache($storage)->getStorageCache(); } /** @@ -562,7 +567,9 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { * @throws InvalidPathException */ protected function verifyPosixPath($fileName) { - $this->scanForInvalidCharacters($fileName, "\\/"); + $invalidChars = \OCP\Util::getForbiddenFileNameChars(); + $this->scanForInvalidCharacters($fileName, $invalidChars); + $fileName = trim($fileName); $reservedNames = ['*']; if (in_array($fileName, $reservedNames)) { @@ -572,11 +579,11 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { /** * @param string $fileName - * @param string $invalidChars + * @param string[] $invalidChars * @throws InvalidPathException */ - private function scanForInvalidCharacters($fileName, $invalidChars) { - foreach (str_split($invalidChars) as $char) { + private function scanForInvalidCharacters(string $fileName, array $invalidChars) { + foreach ($invalidChars as $char) { if (str_contains($fileName, $char)) { throw new InvalidCharacterInPathException(); } @@ -601,7 +608,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { * @return mixed */ public function getMountOption($name, $default = null) { - return isset($this->mountOptions[$name]) ? $this->mountOptions[$name] : $default; + return $this->mountOptions[$name] ?? $default; } /** @@ -663,7 +670,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { private function isSameStorage(IStorage $storage): bool { while ($storage->instanceOfStorage(Wrapper::class)) { /** - * @var Wrapper $sourceStorage + * @var Wrapper $storage */ $storage = $storage->getWrapperStorage(); } @@ -894,6 +901,11 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage { public function getDirectoryContent($directory): \Traversable { $dh = $this->opendir($directory); + + if ($dh === false) { + throw new StorageNotAvailableException('Directory listing failed'); + } + if (is_resource($dh)) { $basePath = rtrim($directory, '/'); while (($file = readdir($dh)) !== false) { diff --git a/lib/private/Files/Storage/DAV.php b/lib/private/Files/Storage/DAV.php index 70f22a17034..e5bbbb560da 100644 --- a/lib/private/Files/Storage/DAV.php +++ b/lib/private/Files/Storage/DAV.php @@ -55,11 +55,11 @@ use OCP\ICertificateManager; use OCP\IConfig; use OCP\Util; use Psr\Http\Message\ResponseInterface; +use Psr\Log\LoggerInterface; use Sabre\DAV\Client; use Sabre\DAV\Xml\Property\ResourceType; use Sabre\HTTP\ClientException; use Sabre\HTTP\ClientHttpException; -use Psr\Log\LoggerInterface; use Sabre\HTTP\RequestInterface; /** @@ -120,9 +120,9 @@ class DAV extends Common { if (isset($params['host']) && isset($params['user']) && isset($params['password'])) { $host = $params['host']; //remove leading http[s], will be generated in createBaseUri() - if (substr($host, 0, 8) == "https://") { + if (str_starts_with($host, "https://")) { $host = substr($host, 8); - } elseif (substr($host, 0, 7) == "http://") { + } elseif (str_starts_with($host, "http://")) { $host = substr($host, 7); } $this->host = $host; @@ -921,7 +921,7 @@ class DAV extends Common { } foreach ($responses as $file => $response) { - $file = urldecode($file); + $file = rawurldecode($file); $file = substr($file, strlen($this->root)); $file = $this->cleanPath($file); $this->statCache->set($file, $response); diff --git a/lib/private/Files/Storage/Home.php b/lib/private/Files/Storage/Home.php index 5427bc425c2..051652b7562 100644 --- a/lib/private/Files/Storage/Home.php +++ b/lib/private/Files/Storage/Home.php @@ -26,6 +26,7 @@ namespace OC\Files\Storage; use OC\Files\Cache\HomePropagator; +use OCP\IUser; /** * Specialized version of Local storage for home directory usage @@ -67,7 +68,7 @@ class Home extends Local implements \OCP\Files\IHomeStorage { $storage = $this; } if (!isset($this->cache)) { - $this->cache = new \OC\Files\Cache\HomeCache($storage); + $this->cache = new \OC\Files\Cache\HomeCache($storage, $this->getCacheDependencies()); } return $this->cache; } @@ -94,7 +95,7 @@ class Home extends Local implements \OCP\Files\IHomeStorage { * * @return \OC\User\User owner of this home storage */ - public function getUser() { + public function getUser(): IUser { return $this->user; } diff --git a/lib/private/Files/Storage/Local.php b/lib/private/Files/Storage/Local.php index 02708ed4f7d..c49cf91dc91 100644 --- a/lib/private/Files/Storage/Local.php +++ b/lib/private/Files/Storage/Local.php @@ -51,6 +51,7 @@ use OCP\Files\ForbiddenException; use OCP\Files\GenericFileException; use OCP\Files\IMimeTypeDetector; use OCP\Files\Storage\IStorage; +use OCP\Files\StorageNotAvailableException; use OCP\IConfig; use OCP\Util; use Psr\Log\LoggerInterface; @@ -73,6 +74,8 @@ class Local extends \OC\Files\Storage\Common { protected bool $unlinkOnTruncate; + protected bool $caseInsensitive = false; + public function __construct($arguments) { if (!isset($arguments['datadir']) || !is_string($arguments['datadir'])) { throw new \InvalidArgumentException('No data directory set for local storage'); @@ -85,16 +88,23 @@ class Local extends \OC\Files\Storage\Common { $realPath = realpath($this->datadir) ?: $this->datadir; $this->realDataDir = rtrim($realPath, '/') . '/'; } - if (substr($this->datadir, -1) !== '/') { + if (!str_ends_with($this->datadir, '/')) { $this->datadir .= '/'; } $this->dataDirLength = strlen($this->realDataDir); $this->config = \OC::$server->get(IConfig::class); $this->mimeTypeDetector = \OC::$server->get(IMimeTypeDetector::class); $this->defUMask = $this->config->getSystemValue('localstorage.umask', 0022); + $this->caseInsensitive = $this->config->getSystemValueBool('localstorage.case_insensitive', false); // support Write-Once-Read-Many file systems $this->unlinkOnTruncate = $this->config->getSystemValueBool('localstorage.unlink_on_truncate', false); + + if (isset($arguments['isExternal']) && $arguments['isExternal'] && !$this->stat('')) { + // data dir not accessible or available, can happen when using an external storage of type Local + // on an unmounted system mount point + throw new StorageNotAvailableException('Local storage path does not exist "' . $this->getSourcePath('') . '"'); + } } public function __destruct() { @@ -155,13 +165,19 @@ class Local extends \OC\Files\Storage\Common { } public function is_dir($path) { - if (substr($path, -1) == '/') { + if ($this->caseInsensitive && !$this->file_exists($path)) { + return false; + } + if (str_ends_with($path, '/')) { $path = substr($path, 0, -1); } return is_dir($this->getSourcePath($path)); } public function is_file($path) { + if ($this->caseInsensitive && !$this->file_exists($path)) { + return false; + } return is_file($this->getSourcePath($path)); } @@ -264,7 +280,17 @@ class Local extends \OC\Files\Storage\Common { } public function file_exists($path) { - return file_exists($this->getSourcePath($path)); + if ($this->caseInsensitive) { + $fullPath = $this->getSourcePath($path); + $parentPath = dirname($fullPath); + if (!is_dir($parentPath)) { + return false; + } + $content = scandir($parentPath, SCANDIR_SORT_NONE); + return is_array($content) && array_search(basename($fullPath), $content) !== false; + } else { + return file_exists($this->getSourcePath($path)); + } } public function filemtime($path) { @@ -365,6 +391,11 @@ class Local extends \OC\Files\Storage\Common { } if (@rename($this->getSourcePath($source), $this->getSourcePath($target))) { + if ($this->caseInsensitive) { + if (mb_strtolower($target) === mb_strtolower($source) && !$this->file_exists($target)) { + return false; + } + } return true; } @@ -381,6 +412,11 @@ class Local extends \OC\Files\Storage\Common { } $result = copy($this->getSourcePath($source), $this->getSourcePath($target)); umask($oldMask); + if ($this->caseInsensitive) { + if (mb_strtolower($target) === mb_strtolower($source) && !$this->file_exists($target)) { + return false; + } + } return $result; } } diff --git a/lib/private/Files/Storage/Wrapper/Encoding.php b/lib/private/Files/Storage/Wrapper/Encoding.php index 6633cbf41e3..1bdb0e39f14 100644 --- a/lib/private/Files/Storage/Wrapper/Encoding.php +++ b/lib/private/Files/Storage/Wrapper/Encoding.php @@ -28,8 +28,8 @@ */ namespace OC\Files\Storage\Wrapper; -use OCP\Cache\CappedMemoryCache; use OC\Files\Filesystem; +use OCP\Cache\CappedMemoryCache; use OCP\Files\Storage\IStorage; use OCP\ICache; diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php index a27f499a210..7ce4338256f 100644 --- a/lib/private/Files/Storage/Wrapper/Encryption.php +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -100,6 +100,8 @@ class Encryption extends Wrapper { /** @var CappedMemoryCache<bool> */ private CappedMemoryCache $encryptedPaths; + private $enabled = true; + /** * @param array $parameters */ @@ -392,6 +394,10 @@ class Encryption extends Wrapper { return $this->storage->fopen($path, $mode); } + if (!$this->enabled) { + return $this->storage->fopen($path, $mode); + } + $encryptionEnabled = $this->encryptionManager->isEnabled(); $shouldEncrypt = false; $encryptionModule = null; @@ -938,34 +944,6 @@ class Encryption extends Wrapper { } /** - * parse raw header to array - * - * @param string $rawHeader - * @return array - */ - protected function parseRawHeader($rawHeader) { - $result = []; - if (str_starts_with($rawHeader, Util::HEADER_START)) { - $header = $rawHeader; - $endAt = strpos($header, Util::HEADER_END); - if ($endAt !== false) { - $header = substr($header, 0, $endAt + strlen(Util::HEADER_END)); - - // +1 to not start with an ':' which would result in empty element at the beginning - $exploded = explode(':', substr($header, strlen(Util::HEADER_START) + 1)); - - $element = array_shift($exploded); - while ($element !== Util::HEADER_END) { - $result[$element] = array_shift($exploded); - $element = array_shift($exploded); - } - } - } - - return $result; - } - - /** * read header from file * * @param string $path @@ -988,7 +966,7 @@ class Encryption extends Wrapper { if ($isEncrypted) { $firstBlock = $this->readFirstBlock($path); - $result = $this->parseRawHeader($firstBlock); + $result = $this->util->parseRawHeader($firstBlock); // if the header doesn't contain a encryption module we check if it is a // legacy file. If true, we add the default encryption module @@ -1093,7 +1071,7 @@ class Encryption extends Wrapper { // object store, stores the size after write and doesn't update this during scan // manually store the unencrypted size - if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) { + if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class) && $this->shouldEncrypt($path)) { $this->getCache()->put($path, ['unencrypted_size' => $count]); } @@ -1103,4 +1081,14 @@ class Encryption extends Wrapper { public function clearIsEncryptedCache(): void { $this->encryptedPaths->clear(); } + + /** + * Allow temporarily disabling the wrapper + * + * @param bool $enabled + * @return void + */ + public function setEnabled(bool $enabled): void { + $this->enabled = $enabled; + } } diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php index 1921ac27848..592acd418ec 100644 --- a/lib/private/Files/Storage/Wrapper/Jail.php +++ b/lib/private/Files/Storage/Wrapper/Jail.php @@ -396,10 +396,7 @@ class Jail extends Wrapper { * @return \OC\Files\Cache\Cache */ public function getCache($path = '', $storage = null) { - if (!$storage) { - $storage = $this->getWrapperStorage(); - } - $sourceCache = $this->getWrapperStorage()->getCache($this->getUnjailedPath($path), $storage); + $sourceCache = $this->getWrapperStorage()->getCache($this->getUnjailedPath($path)); return new CacheJail($sourceCache, $this->rootPath); } diff --git a/lib/private/Files/Storage/Wrapper/KnownMtime.php b/lib/private/Files/Storage/Wrapper/KnownMtime.php new file mode 100644 index 00000000000..dde209c44ab --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/KnownMtime.php @@ -0,0 +1,142 @@ +<?php + +namespace OC\Files\Storage\Wrapper; + +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Storage\IStorage; +use Psr\Clock\ClockInterface; + +/** + * Wrapper that overwrites the mtime return by stat/getMetaData if the returned value + * is lower than when we last modified the file. + * + * This is useful because some storage servers can return an outdated mtime right after writes + */ +class KnownMtime extends Wrapper { + private CappedMemoryCache $knowMtimes; + private ClockInterface $clock; + + public function __construct($arguments) { + parent::__construct($arguments); + $this->knowMtimes = new CappedMemoryCache(); + $this->clock = $arguments['clock']; + } + + public function file_put_contents($path, $data) { + $result = parent::file_put_contents($path, $data); + if ($result) { + $now = $this->clock->now()->getTimestamp(); + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function stat($path) { + $stat = parent::stat($path); + if ($stat) { + $this->applyKnownMtime($path, $stat); + } + return $stat; + } + + public function getMetaData($path) { + $stat = parent::getMetaData($path); + if ($stat) { + $this->applyKnownMtime($path, $stat); + } + return $stat; + } + + private function applyKnownMtime(string $path, array &$stat) { + if (isset($stat['mtime'])) { + $knownMtime = $this->knowMtimes->get($path) ?? 0; + $stat['mtime'] = max($stat['mtime'], $knownMtime); + } + } + + public function filemtime($path) { + $knownMtime = $this->knowMtimes->get($path) ?? 0; + return max(parent::filemtime($path), $knownMtime); + } + + public function mkdir($path) { + $result = parent::mkdir($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function rmdir($path) { + $result = parent::rmdir($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function unlink($path) { + $result = parent::unlink($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function rename($source, $target) { + $result = parent::rename($source, $target); + if ($result) { + $this->knowMtimes->set($target, $this->clock->now()->getTimestamp()); + $this->knowMtimes->set($source, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function copy($source, $target) { + $result = parent::copy($source, $target); + if ($result) { + $this->knowMtimes->set($target, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function fopen($path, $mode) { + $result = parent::fopen($path, $mode); + if ($result && $mode === 'w') { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function touch($path, $mtime = null) { + $result = parent::touch($path, $mtime); + if ($result) { + $this->knowMtimes->set($path, $mtime ?? $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { + $result = parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + if ($result) { + $this->knowMtimes->set($targetInternalPath, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { + $result = parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + if ($result) { + $this->knowMtimes->set($targetInternalPath, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function writeStream(string $path, $stream, int $size = null): int { + $result = parent::writeStream($path, $stream, $size); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } +} diff --git a/lib/private/Files/Storage/Wrapper/PermissionsMask.php b/lib/private/Files/Storage/Wrapper/PermissionsMask.php index 0d140e0a39d..a79eaad0569 100644 --- a/lib/private/Files/Storage/Wrapper/PermissionsMask.php +++ b/lib/private/Files/Storage/Wrapper/PermissionsMask.php @@ -140,7 +140,7 @@ class PermissionsMask extends Wrapper { $data = parent::getMetaData($path); if ($data && isset($data['permissions'])) { - $data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions']; + $data['scan_permissions'] = $data['scan_permissions'] ?? $data['permissions']; $data['permissions'] &= $this->mask; } return $data; @@ -155,7 +155,7 @@ class PermissionsMask extends Wrapper { public function getDirectoryContent($directory): \Traversable { foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) { - $data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions']; + $data['scan_permissions'] = $data['scan_permissions'] ?? $data['permissions']; $data['permissions'] &= $this->mask; yield $data; diff --git a/lib/private/Files/Storage/Wrapper/Wrapper.php b/lib/private/Files/Storage/Wrapper/Wrapper.php index 9f5564b4490..665914df2a7 100644 --- a/lib/private/Files/Storage/Wrapper/Wrapper.php +++ b/lib/private/Files/Storage/Wrapper/Wrapper.php @@ -654,4 +654,15 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea public function getDirectoryContent($directory): \Traversable { return $this->getWrapperStorage()->getDirectoryContent($directory); } + + public function isWrapperOf(IStorage $storage) { + $wrapped = $this->getWrapperStorage(); + if ($wrapped === $storage) { + return true; + } + if ($wrapped instanceof Wrapper) { + return $wrapped->isWrapperOf($storage); + } + return false; + } } diff --git a/lib/private/Files/Stream/Encryption.php b/lib/private/Files/Stream/Encryption.php index bcf0a10740b..c57991f35a9 100644 --- a/lib/private/Files/Stream/Encryption.php +++ b/lib/private/Files/Stream/Encryption.php @@ -153,18 +153,18 @@ class Encryption extends Wrapper { * @throws \BadMethodCallException */ public static function wrap($source, $internalPath, $fullPath, array $header, - $uid, - \OCP\Encryption\IEncryptionModule $encryptionModule, - \OC\Files\Storage\Storage $storage, - \OC\Files\Storage\Wrapper\Encryption $encStorage, - \OC\Encryption\Util $util, - \OC\Encryption\File $file, - $mode, - $size, - $unencryptedSize, - $headerSize, - $signed, - $wrapper = Encryption::class) { + $uid, + \OCP\Encryption\IEncryptionModule $encryptionModule, + \OC\Files\Storage\Storage $storage, + \OC\Files\Storage\Wrapper\Encryption $encStorage, + \OC\Encryption\Util $util, + \OC\Encryption\File $file, + $mode, + $size, + $unencryptedSize, + $headerSize, + $signed, + $wrapper = Encryption::class) { $context = stream_context_create([ 'ocencryption' => [ 'source' => $source, diff --git a/lib/private/Files/Template/TemplateManager.php b/lib/private/Files/Template/TemplateManager.php index bf72e9e23e8..9d9f6416208 100644 --- a/lib/private/Files/Template/TemplateManager.php +++ b/lib/private/Files/Template/TemplateManager.php @@ -31,8 +31,8 @@ use OC\AppFramework\Bootstrap\Coordinator; use OC\Files\Cache\Scanner; use OC\Files\Filesystem; use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Folder; use OCP\Files\File; +use OCP\Files\Folder; use OCP\Files\GenericFileException; use OCP\Files\IRootFolder; use OCP\Files\Node; @@ -240,7 +240,8 @@ class TemplateManager implements ITemplateManager { 'mime' => $file->getMimetype(), 'size' => $file->getSize(), 'type' => $file->getType(), - 'hasPreview' => $this->previewManager->isAvailable($file) + 'hasPreview' => $this->previewManager->isAvailable($file), + 'permissions' => $file->getPermissions(), ]; } @@ -274,6 +275,11 @@ class TemplateManager implements ITemplateManager { $isDefaultTemplates = $skeletonTemplatePath === $defaultTemplateDirectory; $userLang = $this->l10nFactory->getUserLanguage($this->userManager->get($this->userId)); + if ($skeletonTemplatePath === '') { + $this->setTemplatePath(''); + return ''; + } + try { $l10n = $this->l10nFactory->get('lib', $userLang); $userFolder = $this->rootFolder->getUserFolder($this->userId); diff --git a/lib/private/Files/Type/Detection.php b/lib/private/Files/Type/Detection.php index 9a61aa93b95..71b8cb986d7 100644 --- a/lib/private/Files/Type/Detection.php +++ b/lib/private/Files/Type/Detection.php @@ -75,9 +75,9 @@ class Detection implements IMimeTypeDetector { private $defaultConfigDir; public function __construct(IURLGenerator $urlGenerator, - LoggerInterface $logger, - string $customConfigDir, - string $defaultConfigDir) { + LoggerInterface $logger, + string $customConfigDir, + string $defaultConfigDir) { $this->urlGenerator = $urlGenerator; $this->logger = $logger; $this->customConfigDir = $customConfigDir; @@ -96,8 +96,8 @@ class Detection implements IMimeTypeDetector { * @param string|null $secureMimeType */ public function registerType(string $extension, - string $mimetype, - ?string $secureMimeType = null): void { + string $mimetype, + ?string $secureMimeType = null): void { $this->mimetypes[$extension] = [$mimetype, $secureMimeType]; $this->secureMimeTypes[$mimetype] = $secureMimeType ?: $mimetype; } diff --git a/lib/private/Files/Type/Loader.php b/lib/private/Files/Type/Loader.php index 20c298f21b3..f0dbf0c665a 100644 --- a/lib/private/Files/Type/Loader.php +++ b/lib/private/Files/Type/Loader.php @@ -38,14 +38,13 @@ use OCP\IDBConnection; class Loader implements IMimeTypeLoader { use TTransactional; - /** @var IDBConnection */ - private $dbConnection; + private IDBConnection $dbConnection; - /** @var array [id => mimetype] */ - protected $mimetypes; + /** @psalm-var array<int, string> */ + protected array $mimetypes; - /** @var array [mimetype => id] */ - protected $mimetypeIds; + /** @psalm-var array<string, int> */ + protected array $mimetypeIds; /** * @param IDBConnection $dbConnection @@ -58,11 +57,8 @@ class Loader implements IMimeTypeLoader { /** * Get a mimetype from its ID - * - * @param int $id - * @return string|null */ - public function getMimetypeById($id) { + public function getMimetypeById(int $id): ?string { if (!$this->mimetypes) { $this->loadMimetypes(); } @@ -74,11 +70,8 @@ class Loader implements IMimeTypeLoader { /** * Get a mimetype ID, adding the mimetype to the DB if it does not exist - * - * @param string $mimetype - * @return int */ - public function getId($mimetype) { + public function getId(string $mimetype): int { if (!$this->mimetypeIds) { $this->loadMimetypes(); } @@ -90,11 +83,8 @@ class Loader implements IMimeTypeLoader { /** * Test if a mimetype exists in the database - * - * @param string $mimetype - * @return bool */ - public function exists($mimetype) { + public function exists(string $mimetype): bool { if (!$this->mimetypeIds) { $this->loadMimetypes(); } @@ -104,7 +94,7 @@ class Loader implements IMimeTypeLoader { /** * Clear all loaded mimetypes, allow for re-loading */ - public function reset() { + public function reset(): void { $this->mimetypes = []; $this->mimetypeIds = []; } @@ -115,9 +105,9 @@ class Loader implements IMimeTypeLoader { * @param string $mimetype * @return int inserted ID */ - protected function store($mimetype) { - $mimetypeId = $this->atomic(function () use ($mimetype) { - try { + protected function store(string $mimetype): int { + try { + $mimetypeId = $this->atomic(function () use ($mimetype) { $insert = $this->dbConnection->getQueryBuilder(); $insert->insert('mimetypes') ->values([ @@ -125,26 +115,24 @@ class Loader implements IMimeTypeLoader { ]) ->executeStatement(); return $insert->getLastInsertId(); - } catch (DbalException $e) { - if ($e->getReason() !== DBException::REASON_UNIQUE_CONSTRAINT_VIOLATION) { - throw $e; - } - $qb = $this->dbConnection->getQueryBuilder(); - $qb->select('id') - ->from('mimetypes') - ->where($qb->expr()->eq('mimetype', $qb->createNamedParameter($mimetype))); - $result = $qb->executeQuery(); - $id = $result->fetchOne(); - $result->closeCursor(); - if ($id !== false) { - return (int) $id; - } + }, $this->dbConnection); + } catch (DbalException $e) { + if ($e->getReason() !== DBException::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('id') + ->from('mimetypes') + ->where($qb->expr()->eq('mimetype', $qb->createNamedParameter($mimetype))); + $result = $qb->executeQuery(); + $id = $result->fetchOne(); + $result->closeCursor(); + if ($id === false) { throw new \Exception("Database threw an unique constraint on inserting a new mimetype, but couldn't return the ID for this very mimetype"); } - }, $this->dbConnection); - if (!$mimetypeId) { - throw new \Exception("Failed to get mimetype id for $mimetype after trying to store it"); + $mimetypeId = (int) $id; } $this->mimetypes[$mimetypeId] = $mimetype; @@ -155,29 +143,27 @@ class Loader implements IMimeTypeLoader { /** * Load all mimetypes from DB */ - private function loadMimetypes() { + private function loadMimetypes(): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->select('id', 'mimetype') ->from('mimetypes'); - $result = $qb->execute(); + $result = $qb->executeQuery(); $results = $result->fetchAll(); $result->closeCursor(); foreach ($results as $row) { - $this->mimetypes[$row['id']] = $row['mimetype']; - $this->mimetypeIds[$row['mimetype']] = $row['id']; + $this->mimetypes[(int) $row['id']] = $row['mimetype']; + $this->mimetypeIds[$row['mimetype']] = (int) $row['id']; } } /** * Update filecache mimetype based on file extension * - * @param string $ext file extension - * @param int $mimeTypeId * @return int number of changed rows */ - public function updateFilecache($ext, $mimeTypeId) { + public function updateFilecache(string $ext, int $mimeTypeId): int { $folderMimeTypeId = $this->getId('httpd/unix-directory'); $update = $this->dbConnection->getQueryBuilder(); $update->update('filecache') diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index b7f6972ee10..6348427cb3b 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -41,11 +41,11 @@ use OCA\Files_Sharing\SharedStorage; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\BeforeFileScannedEvent; use OCP\Files\Events\BeforeFolderScannedEvent; -use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\FileCacheUpdated; -use OCP\Files\Events\NodeRemovedFromCache; use OCP\Files\Events\FileScannedEvent; use OCP\Files\Events\FolderScannedEvent; +use OCP\Files\Events\NodeAddedToCache; +use OCP\Files\Events\NodeRemovedFromCache; use OCP\Files\NotFoundException; use OCP\Files\Storage\IStorage; use OCP\Files\StorageNotAvailableException; @@ -156,33 +156,37 @@ class Scanner extends PublicEmitter { public function backgroundScan($dir) { $mounts = $this->getMounts($dir); foreach ($mounts as $mount) { - $storage = $mount->getStorage(); - if (is_null($storage)) { - continue; - } + try { + $storage = $mount->getStorage(); + if (is_null($storage)) { + continue; + } - // don't bother scanning failed storages (shortcut for same result) - if ($storage->instanceOfStorage(FailedStorage::class)) { - continue; - } + // don't bother scanning failed storages (shortcut for same result) + if ($storage->instanceOfStorage(FailedStorage::class)) { + continue; + } - $scanner = $storage->getScanner(); - $this->attachListener($mount); + $scanner = $storage->getScanner(); + $this->attachListener($mount); - $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { - $this->triggerPropagator($storage, $path); - }); - $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { - $this->triggerPropagator($storage, $path); - }); - $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) { - $this->triggerPropagator($storage, $path); - }); + $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); - $propagator = $storage->getPropagator(); - $propagator->beginBatch(); - $scanner->backgroundScan(); - $propagator->commitBatch(); + $propagator = $storage->getPropagator(); + $propagator->beginBatch(); + $scanner->backgroundScan(); + $propagator->commitBatch(); + } catch (\Exception $e) { + $this->logger->error("Error while trying to scan mount as {$mount->getMountPoint()}:" . $e->getMessage(), ['exception' => $e, 'app' => 'files']); + } } } diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 71815939310..c80b42134c4 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -49,13 +49,14 @@ namespace OC\Files; use Icewind\Streams\CallbackWrapper; use OC\Files\Mount\MoveableMount; use OC\Files\Storage\Storage; -use OC\User\LazyUser; use OC\Share\Share; -use OC\User\User; +use OC\User\LazyUser; use OC\User\Manager as UserManager; +use OC\User\User; use OCA\Files_Sharing\SharedMount; use OCP\Constants; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\ConnectionLostException; use OCP\Files\EmptyFileNameException; use OCP\Files\FileNameTooLongException; use OCP\Files\InvalidCharacterInPathException; @@ -64,10 +65,12 @@ use OCP\Files\InvalidPathException; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\ReservedWordException; -use OCP\Files\Storage\IStorage; use OCP\IUser; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; +use OCP\Server; +use OCP\Share\IManager; +use OCP\Share\IShare; use Psr\Log\LoggerInterface; /** @@ -286,12 +289,12 @@ class View { $this->updaterEnabled = true; } - protected function writeUpdate(Storage $storage, string $internalPath, ?int $time = null): void { + protected function writeUpdate(Storage $storage, string $internalPath, ?int $time = null, ?int $sizeDifference = null): void { if ($this->updaterEnabled) { if (is_null($time)) { $time = time(); } - $storage->getUpdater()->update($internalPath, $time); + $storage->getUpdater()->update($internalPath, $time, $sizeDifference); } } @@ -397,10 +400,11 @@ class View { } $handle = $this->fopen($path, 'rb'); if ($handle) { - $chunkSize = 524288; // 512 kB chunks + $chunkSize = 524288; // 512 kiB chunks while (!feof($handle)) { echo fread($handle, $chunkSize); flush(); + $this->checkConnectionStatus(); } fclose($handle); return $this->filesize($path); @@ -423,7 +427,7 @@ class View { } $handle = $this->fopen($path, 'rb'); if ($handle) { - $chunkSize = 524288; // 512 kB chunks + $chunkSize = 524288; // 512 kiB chunks $startReading = true; if ($from !== 0 && $from !== '0' && fseek($handle, $from) !== 0) { @@ -453,6 +457,7 @@ class View { } echo fread($handle, $len); flush(); + $this->checkConnectionStatus(); } return ftell($handle) - $from; } @@ -462,6 +467,13 @@ class View { return false; } + private function checkConnectionStatus(): void { + $connectionStatus = \connection_status(); + if ($connectionStatus !== CONNECTION_NORMAL) { + throw new ConnectionLostException("Connection lost. Status: $connectionStatus"); + } + } + /** * @param string $path * @return mixed @@ -721,6 +733,8 @@ class View { public function rename($source, $target) { $absolutePath1 = Filesystem::normalizePath($this->getAbsolutePath($source)); $absolutePath2 = Filesystem::normalizePath($this->getAbsolutePath($target)); + $targetParts = explode('/', $absolutePath2); + $targetUser = $targetParts[1] ?? null; $result = false; if ( Filesystem::isValidPath($target) @@ -775,7 +789,7 @@ class View { if ($internalPath1 === '') { if ($mount1 instanceof MoveableMount) { $sourceParentMount = $this->getMount(dirname($source)); - if ($sourceParentMount === $mount2 && $this->targetIsNotShared($storage2, $internalPath2)) { + if ($sourceParentMount === $mount2 && $this->targetIsNotShared($targetUser, $absolutePath2)) { /** * @var \OC\Files\Mount\MountPoint | \OC\Files\Mount\MoveableMount $mount1 */ @@ -1163,7 +1177,9 @@ class View { $this->removeUpdate($storage, $internalPath); } if ($result !== false && in_array('write', $hooks, true) && $operation !== 'fopen' && $operation !== 'touch') { - $this->writeUpdate($storage, $internalPath); + $isCreateOperation = $operation === 'mkdir' || ($operation === 'file_put_contents' && in_array('create', $hooks, true)); + $sizeDifference = $operation === 'mkdir' ? 0 : $result; + $this->writeUpdate($storage, $internalPath, null, $isCreateOperation ? $sizeDifference : null); } if ($result !== false && in_array('touch', $hooks)) { $this->writeUpdate($storage, $internalPath, $extraParam); @@ -1515,7 +1531,7 @@ class View { $rootEntry['path'] = substr(Filesystem::normalizePath($path . '/' . $rootEntry['name']), strlen($user) + 2); // full path without /$user/ // if sharing was disabled for the user we remove the share permissions - if (\OCP\Util::isSharingDisabledForUser()) { + if ($sharingDisabled) { $rootEntry['permissions'] = $rootEntry['permissions'] & ~\OCP\Constants::PERMISSION_SHARE; } @@ -1769,28 +1785,29 @@ class View { * It is not allowed to move a mount point into a different mount point or * into an already shared folder */ - private function targetIsNotShared(IStorage $targetStorage, string $targetInternalPath): bool { - // note: cannot use the view because the target is already locked - $fileId = $targetStorage->getCache()->getId($targetInternalPath); - if ($fileId === -1) { - // target might not exist, need to check parent instead - $fileId = $targetStorage->getCache()->getId(dirname($targetInternalPath)); - } - - // check if any of the parents were shared by the current owner (include collections) - $shares = Share::getItemShared( - 'folder', - (string)$fileId, - \OC\Share\Constants::FORMAT_NONE, - null, - true - ); - - if (count($shares) > 0) { - $this->logger->debug( - 'It is not allowed to move one mount point into a shared folder', - ['app' => 'files']); - return false; + private function targetIsNotShared(string $user, string $targetPath): bool { + $providers = [ + IShare::TYPE_USER, + IShare::TYPE_GROUP, + IShare::TYPE_EMAIL, + IShare::TYPE_CIRCLE, + IShare::TYPE_ROOM, + IShare::TYPE_DECK, + IShare::TYPE_SCIENCEMESH + ]; + $shareManager = Server::get(IManager::class); + /** @var IShare[] $shares */ + $shares = array_merge(...array_map(function (int $type) use ($shareManager, $user) { + return $shareManager->getSharesBy($user, $type); + }, $providers)); + + foreach ($shares as $share) { + if (str_starts_with($targetPath, $share->getNode()->getPath())) { + $this->logger->debug( + 'It is not allowed to move one mount point into a shared folder', + ['app' => 'files']); + return false; + } } return true; @@ -1834,19 +1851,19 @@ class View { [$storage, $internalPath] = $this->resolvePath($path); $storage->verifyPath($internalPath, $fileName); } catch (ReservedWordException $ex) { - $l = \OC::$server->getL10N('lib'); + $l = \OCP\Util::getL10N('lib'); throw new InvalidPathException($l->t('File name is a reserved word')); } catch (InvalidCharacterInPathException $ex) { - $l = \OC::$server->getL10N('lib'); + $l = \OCP\Util::getL10N('lib'); throw new InvalidPathException($l->t('File name contains at least one invalid character')); } catch (FileNameTooLongException $ex) { - $l = \OC::$server->getL10N('lib'); + $l = \OCP\Util::getL10N('lib'); throw new InvalidPathException($l->t('File name is too long')); } catch (InvalidDirectoryException $ex) { - $l = \OC::$server->getL10N('lib'); + $l = \OCP\Util::getL10N('lib'); throw new InvalidPathException($l->t('Dot files are not allowed')); } catch (EmptyFileNameException $ex) { - $l = \OC::$server->getL10N('lib'); + $l = \OCP\Util::getL10N('lib'); throw new InvalidPathException($l->t('Empty filename is not allowed')); } } |