diff options
Diffstat (limited to 'lib/private/Files')
64 files changed, 1567 insertions, 702 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index cb841755efd..329466e682d 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -109,7 +109,7 @@ class Cache implements ICache { /** * get the stored metadata of a file or folder * - * @param string | int $file either the path of a file or folder or the file id for a file or folder + * @param string|int $file either the path of a file or folder or the file id for a file or folder * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache */ public function get($file) { @@ -131,15 +131,17 @@ class Cache implements ICache { $data = $result->fetch(); $result->closeCursor(); - //merge partial data - if (!$data && is_string($file) && isset($this->partial[$file])) { - return $this->partial[$file]; - } elseif (!$data) { - return $data; - } else { + if ($data !== false) { $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } } + + return false; } /** @@ -661,13 +663,13 @@ class Cache implements ICache { $sourceData = $sourceCache->get($sourcePath); if (!$sourceData) { - throw new \Exception('Invalid source storage path: ' . $sourcePath); + throw new \Exception('Source path not found in cache: ' . $sourcePath); } $shardDefinition = $this->connection->getShardDefinition('filecache'); if ( - $shardDefinition && - $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) ) { $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); return; @@ -886,19 +888,23 @@ class Cache implements ICache { /** * Re-calculate the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { $this->calculateFolderSize($path, $data); + if ($path !== '') { $parent = dirname($path); if ($parent === '.' || $parent === '/') { $parent = ''; } + if ($isBackgroundScan) { $parentData = $this->get($parent); - if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) { + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { $this->correctFolderSize($parent, $parentData, $isBackgroundScan); } } else { @@ -920,7 +926,7 @@ class Cache implements ICache { ->from('filecache') ->whereParent($fileId) ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); $result = $query->executeQuery(); $size = (int)$result->fetchOne(); @@ -1009,8 +1015,8 @@ class Cache implements ICache { } // only set unencrypted size for a folder if any child entries have it set, or the folder is empty - $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || $entry['unencrypted_size'] > 0; - if ($entry['size'] !== $totalSize || ($entry['unencrypted_size'] !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { if ($shouldWriteUnEncryptedSize) { // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes if ($unencryptedMax === 0) { @@ -1061,28 +1067,19 @@ class Cache implements ICache { * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { - // we select the fileid here first instead of directly selecting the path since this helps mariadb/mysql - // to use the correct index. - // The overhead of this should be minimal since the cost of selecting the path by id should be much lower - // than the cost of finding an item with size < 0 $query = $this->getQueryBuilder(); - $query->select('fileid') + $query->select('path') ->from('filecache') ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('fileid', 'DESC') ->setMaxResults(1); $result = $query->executeQuery(); - $id = $result->fetchOne(); + $path = $result->fetchOne(); $result->closeCursor(); - if ($id === false) { - return false; - } - - $path = $this->getPathById($id); - return $path ?? false; + return $path === false ? false : (string)$path; } /** diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index e9417c8012a..ab5bae316f4 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -110,6 +110,10 @@ class CacheEntry implements ICacheEntry { return $this->data['upload_time'] ?? null; } + public function getParentId(): int { + return $this->data['parent']; + } + public function getData() { return $this->data; } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 5ae60ee80b6..5492452273b 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -28,7 +28,7 @@ class CacheQueryBuilder extends ExtendedQueryBuilder { public function selectTagUsage(): self { $this - ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag') + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') ->from('filecache', 'filecache') diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 11a95b5d897..c3f3614f3ca 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -10,6 +10,7 @@ namespace OC\Files\Cache; use OC\FilesMetadata\FilesMetadataManager; use OC\SystemConfig; +use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IFileAccess; use OCP\Files\IMimeTypeLoader; @@ -94,4 +95,128 @@ class FileAccess implements IFileAccess { $rows = $query->executeQuery()->fetchAll(); return $this->rowsToEntries($rows); } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index ff2d6766893..3ddcf1ca4e6 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 3bd674f79e2..b067f70b8cb 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -83,7 +83,7 @@ class Scanner extends BasicEmitter implements IScanner { * * @param bool $useTransactions */ - public function setUseTransactions($useTransactions) { + public function setUseTransactions($useTransactions): void { $this->useTransactions = $useTransactions; } @@ -108,9 +108,9 @@ class Scanner extends BasicEmitter implements IScanner { * @param string $file * @param int $reuseExisting * @param int $parentId - * @param array|null|false $cacheData existing data in the cache for the file to be scanned + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning - * @param null $data the metadata for the file, as returned by the storage + * @param array|null $data the metadata for the file, as returned by the storage * @return array|null an array of metadata of the scanned file * @throws \OCP\Lock\LockedException */ @@ -122,139 +122,130 @@ class Scanner extends BasicEmitter implements IScanner { return null; } } + // only proceed if $file is not a partial file, blacklist is handled by the storage - if (!self::isPartialFile($file)) { - // acquire a lock + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { if ($lock) { if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } } - try { - $data = $data ?? $this->getData($file); - } catch (ForbiddenException $e) { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } - } + return null; + } - return null; - } + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + } - try { - if ($data) { - // pre-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); - } + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } - $parent = dirname($file); - if ($parent === '.' || $parent === '/') { - $parent = ''; - } - if ($parentId === -1) { - $parentId = $this->cache->getParentId($file); + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; } - // scan the parent if it's not in the cache (id -1) and the current file is not the root folder - if ($file && $parentId === -1) { - $parentData = $this->scanFile($parent); - if (!$parentData) { - return null; - } - $parentId = $parentData['fileid']; - } - if ($parent) { - $data['parent'] = $parentId; - } - if (is_null($cacheData)) { - /** @var CacheEntry $cacheData */ - $cacheData = $this->cache->get($file); - } - if ($cacheData && $reuseExisting && isset($cacheData['fileid'])) { - // prevent empty etag - $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; - $fileId = $cacheData['fileid']; - $data['fileid'] = $fileId; - // only reuse data if the file hasn't explicitly changed - $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; - // if the folder is marked as unscanned, never reuse etags - if ($mtimeUnchanged && $cacheData['size'] !== -1) { - $data['mtime'] = $cacheData['mtime']; - if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { - $data['size'] = $cacheData['size']; - } - if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { - $data['etag'] = $etag; - } - } + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } - // we only updated unencrypted_size if it's already set - if ($cacheData['unencrypted_size'] === 0) { - unset($data['unencrypted_size']); + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; } - - // Only update metadata that has changed - // i.e. get all the values in $data that are not present in the cache already - $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); - - // make it known to the caller that etag has been changed and needs propagation - if (isset($newData['etag'])) { - $data['etag_changed'] = true; + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; } - } else { - // we only updated unencrypted_size if it's already set - unset($data['unencrypted_size']); - $newData = $data; - $fileId = -1; } - if (!empty($newData)) { - // Reset the checksum if the data has changed - $newData['checksum'] = ''; - $newData['parent'] = $parentId; - $data['fileid'] = $this->addToCache($file, $newData, $fileId); - } - - $data['oldSize'] = ($cacheData && isset($cacheData['size'])) ? $cacheData['size'] : 0; - if ($cacheData && isset($cacheData['encrypted'])) { - $data['encrypted'] = $cacheData['encrypted']; + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); } - // post-emit only if it was a file. By that we avoid counting/treating folders as files - if ($data['mimetype'] !== 'httpd/unix-directory') { - $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); - \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; } } else { - $this->removeFromCache($file); + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; } - } catch (\Exception $e) { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); } - throw $e; - } - // release the acquired lock - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; } - } - if ($data && !isset($data['encrypted'])) { - $data['encrypted'] = false; + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } - return $data; } - return null; + return $data; } protected function removeFromCache($path) { @@ -319,29 +310,26 @@ class Scanner extends BasicEmitter implements IScanner { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); } + try { - try { - $data = $this->scanFile($path, $reuse, -1, null, $lock); - if ($data && $data['mimetype'] === 'httpd/unix-directory') { - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); - $data['size'] = $size; - } - } catch (NotFoundException $e) { - $this->removeFromCache($path); - return null; + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); + $data['size'] = $size; } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; } finally { - if ($lock) { - if ($this->storage->instanceOfStorage(ILockingStorage::class)) { - $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); - $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); - } + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); } } return $data; @@ -363,23 +351,23 @@ class Scanner extends BasicEmitter implements IScanner { * */ protected function array_diff_assoc_multi(array $array1, array $array2) { - + $result = []; foreach ($array1 as $key => $value) { - + // if $array2 doesn't have the same key, that's a result if (!array_key_exists($key, $array2)) { $result[$key] = $value; continue; } - + // if $array2's value for the same key is different, that's a result if ($array2[$key] !== $value && !is_array($value)) { $result[$key] = $value; continue; } - + if (is_array($value)) { $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); if (!empty($nestedDiff)) { @@ -395,9 +383,9 @@ class Scanner extends BasicEmitter implements IScanner { * Get the children currently in the cache * * @param int $folderId - * @return array[] + * @return array<string, \OCP\Files\Cache\ICacheEntry> */ - protected function getExistingChildren($folderId) { + protected function getExistingChildren($folderId): array { $existingChildren = []; $children = $this->cache->getFolderContentsById($folderId); foreach ($children as $child) { diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index e89b95b756f..e1d3c42a8a2 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -12,6 +13,7 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FilesMetadata\IMetadataQuery; /** @@ -80,13 +82,10 @@ class SearchBuilder { public const TAG_FAVORITE = '_$!<Favorite>!$_'; - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - public function __construct( - IMimeTypeLoader $mimetypeLoader, + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; } /** @@ -285,11 +284,18 @@ class SearchBuilder { private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { - $paramType = self::$fieldTypes[$operator->getField()]; $field = $operator->getField(); $value = $operator->getValue(); $type = $operator->getType(); + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + switch ($operator->getExtra()) { case IMetadataQuery::EXTRA: $metadataQuery->joinIndex($field); // join index table if not joined yet diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 2b49e65f0b4..1a3bda58e6a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -213,6 +213,7 @@ class Storage { $query = $db->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); $query->executeStatement(); $query = $db->getQueryBuilder(); diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php index 2e42b716695..f1de5d3cfb8 100644 --- a/lib/private/Files/Cache/Watcher.php +++ b/lib/private/Files/Cache/Watcher.php @@ -33,6 +33,9 @@ class Watcher implements IWatcher { */ protected $scanner; + /** @var callable[] */ + protected $onUpdate = []; + /** * @param \OC\Files\Storage\Storage $storage */ @@ -100,6 +103,9 @@ class Watcher implements IWatcher { if ($this->cache instanceof Cache) { $this->cache->correctFolderSize($path); } + foreach ($this->onUpdate as $callback) { + $callback($path); + } } /** @@ -112,7 +118,7 @@ class Watcher implements IWatcher { public function needsUpdate($path, $cachedData) { if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { $this->checkedPaths[] = $path; - return $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); } return false; } @@ -130,4 +136,11 @@ class Watcher implements IWatcher { } } } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } } diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index ea0f992114a..5bc4ee8529d 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -21,27 +21,29 @@ use OCP\Files\Search\ISearchOperator; * Jail to a subdirectory of the wrapped cache */ class CacheJail extends CacheWrapper { - /** - * @var string - */ - protected $root; - protected $unjailedRoot; + + protected string $unjailedRoot; public function __construct( ?ICache $cache, - string $root, + protected string $root, ?CacheDependencies $dependencies = null, ) { parent::__construct($cache, $dependencies); - $this->root = $root; - if ($cache instanceof CacheJail) { - $this->unjailedRoot = $cache->getSourcePath($root); - } else { - $this->unjailedRoot = $root; + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); } } + /** + * @return string + */ protected function getRoot() { return $this->root; } @@ -51,11 +53,14 @@ class CacheJail extends CacheWrapper { * * @return string */ - protected function getGetUnjailedRoot() { + public function getGetUnjailedRoot() { return $this->unjailedRoot; } - protected function getSourcePath($path) { + /** + * @return string + */ + protected function getSourcePath(string $path) { if ($path === '') { return $this->getRoot(); } else { @@ -95,7 +100,7 @@ class CacheJail extends CacheWrapper { /** * get the stored metadata of a file or folder * - * @param string /int $file + * @param string|int $file * @return ICacheEntry|false */ public function get($file) { @@ -206,12 +211,12 @@ class CacheJail extends CacheWrapper { /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); } } @@ -223,8 +228,9 @@ class CacheJail extends CacheWrapper { * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($this->getSourcePath($path), $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); } else { return 0; } diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php index fdaa2cf4b7a..f2f1036d6a3 100644 --- a/lib/private/Files/Cache/Wrapper/CacheWrapper.php +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -221,12 +221,12 @@ class CacheWrapper extends Cache { /** * update the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { - if ($this->getCache() instanceof Cache) { - $this->getCache()->correctFolderSize($path, $data, $isBackgroundScan); + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); } } @@ -238,8 +238,9 @@ class CacheWrapper extends Cache { * @return int|float */ public function calculateFolderSize($path, $entry = null) { - if ($this->getCache() instanceof Cache) { - return $this->getCache()->calculateFolderSize($path, $entry); + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); } else { return 0; } diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php index 19ca4a13ece..d6409b7875e 100644 --- a/lib/private/Files/Cache/Wrapper/JailPropagator.php +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php index 9bd7da57233..b1ae516654a 100644 --- a/lib/private/Files/Cache/Wrapper/JailWatcher.php +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -55,4 +55,7 @@ class JailWatcher extends Watcher { $this->watcher->cleanFolder($this->getSourcePath($path)); } + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } } diff --git a/lib/private/Files/Config/CachedMountFileInfo.php b/lib/private/Files/Config/CachedMountFileInfo.php index 90a6b47f9d8..69bd4e9301e 100644 --- a/lib/private/Files/Config/CachedMountFileInfo.php +++ b/lib/private/Files/Config/CachedMountFileInfo.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php index 6a5407934c8..9d63184e05f 100644 --- a/lib/private/Files/Config/MountProviderCollection.php +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -24,60 +24,43 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { use EmitterTrait; /** - * @var \OCP\Files\Config\IHomeMountProvider[] + * @var list<IHomeMountProvider> */ - private $homeProviders = []; + private array $homeProviders = []; /** - * @var \OCP\Files\Config\IMountProvider[] + * @var list<IMountProvider> */ - private $providers = []; + private array $providers = []; - /** @var \OCP\Files\Config\IRootMountProvider[] */ - private $rootProviders = []; + /** @var list<IRootMountProvider> */ + private array $rootProviders = []; - /** - * @var \OCP\Files\Storage\IStorageFactory - */ - private $loader; - - /** - * @var \OCP\Files\Config\IUserMountCache - */ - private $mountCache; - - /** @var callable[] */ - private $mountFilters = []; + /** @var list<callable> */ + private array $mountFilters = []; - private IEventLogger $eventLogger; - - /** - * @param \OCP\Files\Storage\IStorageFactory $loader - * @param IUserMountCache $mountCache - */ public function __construct( - IStorageFactory $loader, - IUserMountCache $mountCache, - IEventLogger $eventLogger, + private IStorageFactory $loader, + private IUserMountCache $mountCache, + private IEventLogger $eventLogger, ) { - $this->loader = $loader; - $this->mountCache = $mountCache; - $this->eventLogger = $eventLogger; } + /** + * @return list<IMountPoint> + */ private function getMountsFromProvider(IMountProvider $provider, IUser $user, IStorageFactory $loader): array { $class = str_replace('\\', '_', get_class($provider)); $uid = $user->getUID(); $this->eventLogger->start('fs:setup:provider:' . $class, "Getting mounts from $class for $uid"); $mounts = $provider->getMountsForUser($user, $loader) ?? []; $this->eventLogger->end('fs:setup:provider:' . $class); - return $mounts; + return array_values($mounts); } /** - * @param IUser $user - * @param IMountProvider[] $providers - * @return IMountPoint[] + * @param list<IMountProvider> $providers + * @return list<IMountPoint> */ private function getUserMountsForProviders(IUser $user, array $providers): array { $loader = $this->loader; @@ -90,10 +73,16 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { return $this->filterMounts($user, $mounts); } + /** + * @return list<IMountPoint> + */ public function getMountsForUser(IUser $user): array { return $this->getUserMountsForProviders($user, $this->providers); } + /** + * @return list<IMountPoint> + */ public function getUserMountsForProviderClasses(IUser $user, array $mountProviderClasses): array { $providers = array_filter( $this->providers, @@ -102,7 +91,10 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { return $this->getUserMountsForProviders($user, $providers); } - public function addMountForUser(IUser $user, IMountManager $mountManager, ?callable $providerFilter = null) { + /** + * @return list<IMountPoint> + */ + public function addMountForUser(IUser $user, IMountManager $mountManager, ?callable $providerFilter = null): array { // shared mount provider gets to go last since it needs to know existing files // to check for name collisions $firstMounts = []; @@ -135,18 +127,15 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { array_walk($lateMounts, [$mountManager, 'addMount']); $this->eventLogger->end('fs:setup:add-mounts'); - return array_merge($lateMounts, $firstMounts); + return array_values(array_merge($lateMounts, $firstMounts)); } /** * Get the configured home mount for this user * - * @param \OCP\IUser $user - * @return \OCP\Files\Mount\IMountPoint * @since 9.1.0 */ - public function getHomeMountForUser(IUser $user) { - /** @var \OCP\Files\Config\IHomeMountProvider[] $providers */ + public function getHomeMountForUser(IUser $user): IMountPoint { $providers = array_reverse($this->homeProviders); // call the latest registered provider first to give apps an opportunity to overwrite builtin foreach ($providers as $homeProvider) { if ($mount = $homeProvider->getHomeMountForUser($user, $this->loader)) { @@ -159,34 +148,36 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { /** * Add a provider for mount points - * - * @param \OCP\Files\Config\IMountProvider $provider */ - public function registerProvider(IMountProvider $provider) { + public function registerProvider(IMountProvider $provider): void { $this->providers[] = $provider; $this->emit('\OC\Files\Config', 'registerMountProvider', [$provider]); } - public function registerMountFilter(callable $filter) { + public function registerMountFilter(callable $filter): void { $this->mountFilters[] = $filter; } - private function filterMounts(IUser $user, array $mountPoints) { - return array_filter($mountPoints, function (IMountPoint $mountPoint) use ($user) { + /** + * @param list<IMountPoint> $mountPoints + * @return list<IMountPoint> + */ + private function filterMounts(IUser $user, array $mountPoints): array { + return array_values(array_filter($mountPoints, function (IMountPoint $mountPoint) use ($user) { foreach ($this->mountFilters as $filter) { if ($filter($mountPoint, $user) === false) { return false; } } return true; - }); + })); } /** * Add a provider for home mount points * - * @param \OCP\Files\Config\IHomeMountProvider $provider + * @param IHomeMountProvider $provider * @since 9.1.0 */ public function registerHomeProvider(IHomeMountProvider $provider) { @@ -196,21 +187,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { /** * Get the mount cache which can be used to search for mounts without setting up the filesystem - * - * @return IUserMountCache */ - public function getMountCache() { + public function getMountCache(): IUserMountCache { return $this->mountCache; } - public function registerRootProvider(IRootMountProvider $provider) { + public function registerRootProvider(IRootMountProvider $provider): void { $this->rootProviders[] = $provider; } /** * Get all root mountpoints * - * @return \OCP\Files\Mount\IMountPoint[] + * @return list<IMountPoint> * @since 20.0.0 */ public function getRootMounts(): array { @@ -226,16 +215,33 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { throw new \Exception('No root mounts provided by any provider'); } - return $mounts; + return array_values($mounts); } - public function clearProviders() { + public function clearProviders(): void { $this->providers = []; $this->homeProviders = []; $this->rootProviders = []; } + /** + * @return list<IMountProvider> + */ public function getProviders(): array { return $this->providers; } + + /** + * @return list<IHomeMountProvider> + */ + public function getHomeProviders(): array { + return $this->homeProviders; + } + + /** + * @return list<IRootMountProvider> + */ + public function getRootProviders(): array { + return $this->rootProviders; + } } diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php index 7d9ecc16f76..3e53a67a044 100644 --- a/lib/private/Files/Config/UserMountCache.php +++ b/lib/private/Files/Config/UserMountCache.php @@ -11,6 +11,10 @@ use OC\User\LazyUser; use OCP\Cache\CappedMemoryCache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Diagnostics\IEventLogger; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Config\Event\UserMountAddedEvent; +use OCP\Files\Config\Event\UserMountRemovedEvent; +use OCP\Files\Config\Event\UserMountUpdatedEvent; use OCP\Files\Config\ICachedMountFileInfo; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; @@ -46,6 +50,7 @@ class UserMountCache implements IUserMountCache { private IUserManager $userManager, private LoggerInterface $logger, private IEventLogger $eventLogger, + private IEventDispatcher $eventDispatcher, ) { $this->cacheInfoCache = new CappedMemoryCache(); $this->internalPathCache = new CappedMemoryCache(); @@ -98,24 +103,39 @@ class UserMountCache implements IUserMountCache { $userUID = $user->getUID(); try { foreach ($addedMounts as $mount) { + $this->logger->debug("Adding mount '{$mount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $mount->getMountProvider()]); $this->addToCache($mount); /** @psalm-suppress InvalidArgument */ $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; } foreach ($removedMounts as $mount) { + $this->logger->debug("Removing mount '{$mount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $mount->getMountProvider()]); $this->removeFromCache($mount); unset($this->mountsForUsers[$userUID][$mount->getKey()]); } - foreach ($changedMounts as $mount) { - $this->updateCachedMount($mount); + foreach ($changedMounts as $mountPair) { + $newMount = $mountPair[1]; + $this->logger->debug("Updating mount '{$newMount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $newMount->getMountProvider()]); + $this->updateCachedMount($newMount); /** @psalm-suppress InvalidArgument */ - $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; + $this->mountsForUsers[$userUID][$newMount->getKey()] = $newMount; } $this->connection->commit(); } catch (\Throwable $e) { $this->connection->rollBack(); throw $e; } + + // Only fire events after all mounts have already been adjusted in the database. + foreach ($addedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountAddedEvent($mount)); + } + foreach ($removedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountRemovedEvent($mount)); + } + foreach ($changedMounts as $mountPair) { + $this->eventDispatcher->dispatchTyped(new UserMountUpdatedEvent($mountPair[0], $mountPair[1])); + } } $this->eventLogger->end('fs:setup:user:register'); } @@ -123,19 +143,19 @@ class UserMountCache implements IUserMountCache { /** * @param array<string, ICachedMountInfo> $newMounts * @param array<string, ICachedMountInfo> $cachedMounts - * @return ICachedMountInfo[] + * @return list<list{0: ICachedMountInfo, 1: ICachedMountInfo}> Pairs of old and new mounts */ - private function findChangedMounts(array $newMounts, array $cachedMounts) { + private function findChangedMounts(array $newMounts, array $cachedMounts): array { $changed = []; foreach ($cachedMounts as $key => $cachedMount) { if (isset($newMounts[$key])) { $newMount = $newMounts[$key]; if ( - $newMount->getStorageId() !== $cachedMount->getStorageId() || - $newMount->getMountId() !== $cachedMount->getMountId() || - $newMount->getMountProvider() !== $cachedMount->getMountProvider() + $newMount->getStorageId() !== $cachedMount->getStorageId() + || $newMount->getMountId() !== $cachedMount->getMountId() + || $newMount->getMountProvider() !== $cachedMount->getMountProvider() ) { - $changed[] = $newMount; + $changed[] = [$cachedMount, $newMount]; } } } diff --git a/lib/private/Files/Conversion/ConversionManager.php b/lib/private/Files/Conversion/ConversionManager.php new file mode 100644 index 00000000000..2c98a4c6404 --- /dev/null +++ b/lib/private/Files/Conversion/ConversionManager.php @@ -0,0 +1,181 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Conversion; + +use OC\AppFramework\Bootstrap\Coordinator; +use OC\ForbiddenException; +use OC\SystemConfig; +use OCP\Files\Conversion\IConversionManager; +use OCP\Files\Conversion\IConversionProvider; +use OCP\Files\File; +use OCP\Files\GenericFileException; +use OCP\Files\IRootFolder; +use OCP\IL10N; +use OCP\ITempManager; +use OCP\L10N\IFactory; +use OCP\PreConditionNotMetException; +use Psr\Container\ContainerExceptionInterface; +use Psr\Container\ContainerInterface; +use Psr\Container\NotFoundExceptionInterface; +use Psr\Log\LoggerInterface; +use RuntimeException; +use Throwable; + +class ConversionManager implements IConversionManager { + /** @var string[] */ + private array $preferredApps = [ + 'richdocuments', + ]; + + /** @var list<IConversionProvider> */ + private array $preferredProviders = []; + + /** @var list<IConversionProvider> */ + private array $providers = []; + + private IL10N $l10n; + public function __construct( + private Coordinator $coordinator, + private ContainerInterface $serverContainer, + private IRootFolder $rootFolder, + private ITempManager $tempManager, + private LoggerInterface $logger, + private SystemConfig $config, + IFactory $l10nFactory, + ) { + $this->l10n = $l10nFactory->get('files'); + } + + public function hasProviders(): bool { + $context = $this->coordinator->getRegistrationContext(); + return !empty($context->getFileConversionProviders()); + } + + public function getProviders(): array { + $providers = []; + foreach ($this->getRegisteredProviders() as $provider) { + $providers = array_merge($providers, $provider->getSupportedMimeTypes()); + } + return $providers; + } + + public function convert(File $file, string $targetMimeType, ?string $destination = null): string { + if (!$this->hasProviders()) { + throw new PreConditionNotMetException($this->l10n->t('No file conversion providers available')); + } + + // Operate in mebibytes + $fileSize = $file->getSize() / (1024 * 1024); + $threshold = $this->config->getValue('max_file_conversion_filesize', 100); + if ($fileSize > $threshold) { + throw new GenericFileException($this->l10n->t('File is too large to convert')); + } + + $fileMimeType = $file->getMimetype(); + $validProvider = $this->getValidProvider($fileMimeType, $targetMimeType); + + if ($validProvider !== null) { + // Get the target extension given by the provider + $targetExtension = ''; + foreach ($validProvider->getSupportedMimeTypes() as $mimeProvider) { + if ($mimeProvider->getTo() === $targetMimeType) { + $targetExtension = $mimeProvider->getExtension(); + break; + } + } + // If destination not provided, we use the same path + // as the original file, but with the new extension + if ($destination === null) { + $basename = pathinfo($file->getPath(), PATHINFO_FILENAME); + $parent = $file->getParent(); + $destination = $parent->getFullPath($basename . '.' . $targetExtension); + } + + // If destination doesn't match the target extension, we throw an error + if (pathinfo($destination, PATHINFO_EXTENSION) !== $targetExtension) { + throw new GenericFileException($this->l10n->t('Destination does not match conversion extension')); + } + + // Check destination before converting + $this->checkDestination($destination); + + // Convert the file and write it to the destination + $convertedFile = $validProvider->convertFile($file, $targetMimeType); + $convertedFile = $this->writeToDestination($destination, $convertedFile); + return $convertedFile->getPath(); + } + + throw new RuntimeException($this->l10n->t('Could not convert file')); + } + + /** + * @return list<IConversionProvider> + */ + private function getRegisteredProviders(): array { + $context = $this->coordinator->getRegistrationContext(); + foreach ($context->getFileConversionProviders() as $providerRegistration) { + $class = $providerRegistration->getService(); + $appId = $providerRegistration->getAppId(); + + try { + if (in_array($appId, $this->preferredApps)) { + $this->preferredProviders[$class] = $this->serverContainer->get($class); + continue; + } + + $this->providers[$class] = $this->serverContainer->get($class); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface|Throwable $e) { + $this->logger->error('Failed to load file conversion provider ' . $class, [ + 'exception' => $e, + ]); + } + } + + return array_values(array_merge([], $this->preferredProviders, $this->providers)); + } + + private function checkDestination(string $destination): void { + if (!$this->rootFolder->nodeExists(dirname($destination))) { + throw new ForbiddenException($this->l10n->t('Destination does not exist')); + } + + $folder = $this->rootFolder->get(dirname($destination)); + if (!$folder->isCreatable()) { + throw new ForbiddenException($this->l10n->t('Destination is not creatable')); + } + } + + private function writeToDestination(string $destination, mixed $content): File { + $this->checkDestination($destination); + + if ($this->rootFolder->nodeExists($destination)) { + $file = $this->rootFolder->get($destination); + $parent = $file->getParent(); + + // Folder permissions is already checked in checkDestination method + $newName = $parent->getNonExistingName(basename($destination)); + $destination = $parent->getFullPath($newName); + } + + return $this->rootFolder->newFile($destination, $content); + } + + private function getValidProvider(string $fileMimeType, string $targetMimeType): ?IConversionProvider { + foreach ($this->getRegisteredProviders() as $provider) { + foreach ($provider->getSupportedMimeTypes() as $mimeProvider) { + if ($mimeProvider->getFrom() === $fileMimeType && $mimeProvider->getTo() === $targetMimeType) { + return $provider; + } + } + } + + return null; + } +} diff --git a/lib/private/Files/FilenameValidator.php b/lib/private/Files/FilenameValidator.php index fde45068df7..a78c6d3cc3c 100644 --- a/lib/private/Files/FilenameValidator.php +++ b/lib/private/Files/FilenameValidator.php @@ -127,9 +127,6 @@ class FilenameValidator implements IFilenameValidator { if (empty($this->forbiddenCharacters)) { // Get always forbidden characters $forbiddenCharacters = str_split(\OCP\Constants::FILENAME_INVALID_CHARS); - if ($forbiddenCharacters === false) { - $forbiddenCharacters = []; - } // Get admin defined invalid characters $additionalChars = $this->config->getSystemValue('forbidden_filename_characters', []); @@ -231,7 +228,45 @@ class FilenameValidator implements IFilenameValidator { return false; } - protected function checkForbiddenName($filename): void { + public function sanitizeFilename(string $name, ?string $charReplacement = null): string { + $forbiddenCharacters = $this->getForbiddenCharacters(); + + if ($charReplacement === null) { + $charReplacement = array_diff(['_', '-', ' '], $forbiddenCharacters); + $charReplacement = reset($charReplacement) ?: ''; + } + if (mb_strlen($charReplacement) !== 1) { + throw new \InvalidArgumentException('No or invalid character replacement given'); + } + + $nameLowercase = mb_strtolower($name); + foreach ($this->getForbiddenExtensions() as $extension) { + if (str_ends_with($nameLowercase, $extension)) { + $name = substr($name, 0, strlen($name) - strlen($extension)); + } + } + + $basename = strlen($name) > 1 + ? substr($name, 0, strpos($name, '.', 1) ?: null) + : $name; + if (in_array(mb_strtolower($basename), $this->getForbiddenBasenames())) { + $name = str_replace($basename, $this->l10n->t('%1$s (renamed)', [$basename]), $name); + } + + if ($name === '') { + $name = $this->l10n->t('renamed file'); + } + + if (in_array(mb_strtolower($name), $this->getForbiddenFilenames())) { + $name = $this->l10n->t('%1$s (renamed)', [$name]); + } + + $name = str_replace($forbiddenCharacters, $charReplacement, $name); + return $name; + } + + protected function checkForbiddenName(string $filename): void { + $filename = mb_strtolower($filename); if ($this->isForbidden($filename)) { throw new ReservedWordException($this->l10n->t('"%1$s" is a forbidden file or folder name.', [$filename])); } @@ -295,6 +330,6 @@ class FilenameValidator implements IFilenameValidator { $values = $fallback; } - return array_map('mb_strtolower', $values); + return array_map(mb_strtolower(...), $values); } }; diff --git a/lib/private/Files/Filesystem.php b/lib/private/Files/Filesystem.php index 48c069de0b9..8fe56cf060c 100644 --- a/lib/private/Files/Filesystem.php +++ b/lib/private/Files/Filesystem.php @@ -8,6 +8,7 @@ namespace OC\Files; use OC\Files\Mount\MountPoint; +use OC\Files\Storage\StorageFactory; use OC\User\NoUserException; use OCP\Cache\CappedMemoryCache; use OCP\EventDispatcher\IEventDispatcher; @@ -178,7 +179,9 @@ class Filesystem { } $mounts = self::getMountManager()->getAll(); - if (!self::getLoader()->addStorageWrapper($wrapperName, $wrapper, $priority, $mounts)) { + /** @var StorageFactory $loader */ + $loader = self::getLoader(); + if (!$loader->addStorageWrapper($wrapperName, $wrapper, $priority, $mounts)) { // do not re-wrap if storage with this name already existed return; } diff --git a/lib/private/Files/Mount/MountPoint.php b/lib/private/Files/Mount/MountPoint.php index 7a5a87be35b..bab2dc8e4bd 100644 --- a/lib/private/Files/Mount/MountPoint.php +++ b/lib/private/Files/Mount/MountPoint.php @@ -198,7 +198,7 @@ class MountPoint implements IMountPoint { if (is_null($storage)) { return -1; } - $this->numericStorageId = $storage->getStorageCache()->getNumericId(); + $this->numericStorageId = $storage->getCache()->getNumericStorageId(); } return $this->numericStorageId; } diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 99c52108fa8..4b088f2c808 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -7,117 +7,39 @@ */ namespace OC\Files\Mount; +use OC\Files\ObjectStore\HomeObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage\IStorageFactory; -use OCP\IConfig; use OCP\IUser; -use Psr\Log\LoggerInterface; /** * Mount provider for object store home storages */ class ObjectHomeMountProvider implements IHomeMountProvider { - /** - * @var IConfig - */ - private $config; - - /** - * ObjectStoreHomeMountProvider constructor. - * - * @param IConfig $config - */ - public function __construct(IConfig $config) { - $this->config = $config; + public function __construct( + private PrimaryObjectStoreConfig $objectStoreConfig, + ) { } /** - * Get the cache mount for a user + * Get the home mount for a user * * @param IUser $user * @param IStorageFactory $loader - * @return \OCP\Files\Mount\IMountPoint + * @return ?IMountPoint */ - public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { - $config = $this->getMultiBucketObjectStoreConfig($user); - if ($config === null) { - $config = $this->getSingleBucketObjectStoreConfig($user); - } - - if ($config === null) { + public function getHomeMountForUser(IUser $user, IStorageFactory $loader): ?IMountPoint { + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForUser($user); + if ($objectStoreConfig === null) { return null; } + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + 'user' => $user, + ]); - return new HomeMountPoint($user, '\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader, null, null, self::class); - } - - /** - * @param IUser $user - * @return array|null - */ - private function getSingleBucketObjectStoreConfig(IUser $user) { - $config = $this->config->getSystemValue('objectstore'); - if (!is_array($config)) { - return null; - } - - // sanity checks - if (empty($config['class'])) { - \OC::$server->get(LoggerInterface::class)->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['user'] = $user; - - return $config; - } - - /** - * @param IUser $user - * @return array|null - */ - private function getMultiBucketObjectStoreConfig(IUser $user) { - $config = $this->config->getSystemValue('objectstore_multibucket'); - if (!is_array($config)) { - return null; - } - - // sanity checks - if (empty($config['class'])) { - \OC::$server->get(LoggerInterface::class)->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); - - if ($bucket === null) { - /* - * Use any provided bucket argument as prefix - * and add the mapping from username => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - $mapper = new \OC\Files\ObjectStore\Mapper($user, $this->config); - $numBuckets = $config['arguments']['num_buckets'] ?? 64; - $config['arguments']['bucket'] .= $mapper->getBucket($numBuckets); - - $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $config['arguments']['bucket']); - } else { - $config['arguments']['bucket'] = $bucket; - } - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['user'] = $user; - - return $config; + return new HomeMountPoint($user, HomeObjectStoreStorage::class, '/' . $user->getUID(), $arguments, $loader, null, null, self::class); } } diff --git a/lib/private/Files/Mount/RootMountProvider.php b/lib/private/Files/Mount/RootMountProvider.php index 86f8188978f..5e0c924ad38 100644 --- a/lib/private/Files/Mount/RootMountProvider.php +++ b/lib/private/Files/Mount/RootMountProvider.php @@ -10,79 +10,41 @@ namespace OC\Files\Mount; use OC; use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\Storage\LocalRootStorage; -use OC_App; use OCP\Files\Config\IRootMountProvider; use OCP\Files\Storage\IStorageFactory; use OCP\IConfig; -use Psr\Log\LoggerInterface; class RootMountProvider implements IRootMountProvider { + private PrimaryObjectStoreConfig $objectStoreConfig; private IConfig $config; - private LoggerInterface $logger; - public function __construct(IConfig $config, LoggerInterface $logger) { + public function __construct(PrimaryObjectStoreConfig $objectStoreConfig, IConfig $config) { + $this->objectStoreConfig = $objectStoreConfig; $this->config = $config; - $this->logger = $logger; } public function getRootMounts(IStorageFactory $loader): array { - $objectStore = $this->config->getSystemValue('objectstore', null); - $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - if ($objectStoreMultiBucket) { - return [$this->getMultiBucketStoreRootMount($loader, $objectStoreMultiBucket)]; - } elseif ($objectStore) { - return [$this->getObjectStoreRootMount($loader, $objectStore)]; + if ($objectStoreConfig) { + return [$this->getObjectStoreRootMount($loader, $objectStoreConfig)]; } else { return [$this->getLocalRootMount($loader)]; } } - private function validateObjectStoreConfig(array &$config) { - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - // instantiate object store implementation - $name = $config['class']; - if (str_starts_with($name, 'OCA\\') && substr_count($name, '\\') >= 2) { - $segments = explode('\\', $name); - OC_App::loadApp(strtolower($segments[1])); - } - } - private function getLocalRootMount(IStorageFactory $loader): MountPoint { $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); return new MountPoint(LocalRootStorage::class, '/', ['datadir' => $configDataDirectory], $loader, null, null, self::class); } - private function getObjectStoreRootMount(IStorageFactory $loader, array $config): MountPoint { - $this->validateObjectStoreConfig($config); - - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - // mount with plain / root object store implementation - $config['class'] = ObjectStoreStorage::class; - - return new MountPoint($config['class'], '/', $config['arguments'], $loader, null, null, self::class); - } - - private function getMultiBucketStoreRootMount(IStorageFactory $loader, array $config): MountPoint { - $this->validateObjectStoreConfig($config); - - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - // put the root FS always in first bucket for multibucket configuration - $config['arguments']['bucket'] .= '0'; - - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - // mount with plain / root object store implementation - $config['class'] = ObjectStoreStorage::class; + private function getObjectStoreRootMount(IStorageFactory $loader, array $objectStoreConfig): MountPoint { + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + ]); - return new MountPoint($config['class'], '/', $config['arguments'], $loader, null, null, self::class); + return new MountPoint(ObjectStoreStorage::class, '/', $arguments, $loader, null, null, self::class); } } diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php index a894c69649a..7453b553119 100644 --- a/lib/private/Files/Node/Folder.php +++ b/lib/private/Files/Node/Folder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors * SPDX-FileCopyrightText: 2016 ownCloud, Inc. @@ -126,8 +127,21 @@ class Folder extends Node implements \OCP\Files\Folder { $fullPath = $this->getFullPath($path); $nonExisting = new NonExistingFolder($this->root, $this->view, $fullPath); $this->sendHooks(['preWrite', 'preCreate'], [$nonExisting]); - if (!$this->view->mkdir($fullPath) && !$this->view->is_dir($fullPath)) { - throw new NotPermittedException('Could not create folder "' . $fullPath . '"'); + if (!$this->view->mkdir($fullPath)) { + // maybe another concurrent process created the folder already + if (!$this->view->is_dir($fullPath)) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '"'); + } else { + // we need to ensure we don't return before the concurrent request has finished updating the cache + $tries = 5; + while (!$this->view->getFileInfo($fullPath)) { + if ($tries < 1) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '", folder exists but unable to get cache entry'); + } + usleep(5 * 1000); + $tries--; + } + } } $parent = dirname($fullPath) === $this->getPath() ? $this : null; $node = new Folder($this->root, $this->view, $fullPath, null, $parent); @@ -447,4 +461,12 @@ class Folder extends Node implements \OCP\Files\Folder { return $this->search($query); } + + public function verifyPath($fileName, $readonly = false): void { + $this->view->verifyPath( + $this->getPath(), + $fileName, + $readonly, + ); + } } diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php index 5879748d951..37b1efa0fad 100644 --- a/lib/private/Files/Node/LazyFolder.php +++ b/lib/private/Files/Node/LazyFolder.php @@ -561,4 +561,8 @@ class LazyFolder implements Folder { public function getMetadata(): array { return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); } + + public function verifyPath($fileName, $readonly = false): void { + $this->__call(__FUNCTION__, func_get_args()); + } } diff --git a/lib/private/Files/Node/Root.php b/lib/private/Files/Node/Root.php index 1686051131d..d82ec08f362 100644 --- a/lib/private/Files/Node/Root.php +++ b/lib/private/Files/Node/Root.php @@ -384,13 +384,17 @@ class Root extends Folder implements IRootFolder { // scope the cache by user, so we don't return nodes for different users if ($this->user) { $cachedPath = $this->pathByIdCache->get($this->user->getUID() . '::' . $id); - if ($cachedPath && str_starts_with($path, $cachedPath)) { + if ($cachedPath && str_starts_with($cachedPath, $path)) { // getting the node by path is significantly cheaper than finding it by id - $node = $this->get($cachedPath); - // by validating that the cached path still has the requested fileid we can work around the need to invalidate the cached path - // if the cached path is invalid or a different file now we fall back to the uncached logic - if ($node && $node->getId() === $id) { - return $node; + try { + $node = $this->get($cachedPath); + // by validating that the cached path still has the requested fileid we can work around the need to invalidate the cached path + // if the cached path is invalid or a different file now we fall back to the uncached logic + if ($node && $node->getId() === $id) { + return $node; + } + } catch (NotFoundException|NotPermittedException) { + // The file may be moved but the old path still in cache } } } @@ -522,9 +526,9 @@ class Root extends Folder implements IRootFolder { $isDir = $info->getType() === FileInfo::TYPE_FOLDER; $view = new View(''); if ($isDir) { - return new Folder($this, $view, $path, $info, $parent); + return new Folder($this, $view, $fullPath, $info, $parent); } else { - return new File($this, $view, $path, $info, $parent); + return new File($this, $view, $fullPath, $info, $parent); } } } diff --git a/lib/private/Files/Notify/Change.php b/lib/private/Files/Notify/Change.php index d2448e5deec..c8eccd11ae2 100644 --- a/lib/private/Files/Notify/Change.php +++ b/lib/private/Files/Notify/Change.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Notify/RenameChange.php b/lib/private/Files/Notify/RenameChange.php index 98fd7099a58..28554ceaa26 100644 --- a/lib/private/Files/Notify/RenameChange.php +++ b/lib/private/Files/Notify/RenameChange.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php index 575cc336ba8..2729bb3c037 100644 --- a/lib/private/Files/ObjectStore/Azure.php +++ b/lib/private/Files/ObjectStore/Azure.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/ObjectStore/ObjectStoreScanner.php b/lib/private/Files/ObjectStore/ObjectStoreScanner.php index d8a77d36dee..5c3992b8458 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreScanner.php +++ b/lib/private/Files/ObjectStore/ObjectStoreScanner.php @@ -13,11 +13,11 @@ use OCP\Files\FileInfo; class ObjectStoreScanner extends Scanner { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { - return []; + return null; } public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { - return []; + return null; } protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { @@ -61,7 +61,7 @@ class ObjectStoreScanner extends Scanner { $query->select('path') ->from('filecache') ->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('path', 'DESC') ->setMaxResults(1); diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index e05de7cd91d..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -22,6 +22,7 @@ use OCP\Files\FileInfo; use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; @@ -66,7 +67,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir(string $path, bool $force = false): bool { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); if (!$force && $this->file_exists($path)) { $this->logger->warning("Tried to create an object store folder that already exists: $path"); @@ -76,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -412,16 +413,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { $this->logger->error( 'Could not create object for ' . $path, @@ -479,6 +470,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -500,24 +499,27 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream, $mimetype); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream, $mimetype); - if (is_resource($stream)) { - fclose($stream); - } + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); + } + if (is_resource($countStream)) { + fclose($countStream); } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -541,7 +543,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil ] ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { @@ -557,7 +559,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { @@ -592,7 +594,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { $sourceCache = $sourceStorage->getCache(); - if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); // Do not import any data when source and target bucket are identical. return true; @@ -615,6 +624,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil /** @var ObjectStoreStorage $sourceStorage */ $sourceStorage->setPreserveCacheOnDelete(false); } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); return true; @@ -684,7 +697,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php new file mode 100644 index 00000000000..fdfe989addc --- /dev/null +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -0,0 +1,140 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\ObjectStore; + +use OCP\App\IAppManager; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +use OCP\IUser; + +/** + * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}} + */ +class PrimaryObjectStoreConfig { + public function __construct( + private readonly IConfig $config, + private readonly IAppManager $appManager, + ) { + } + + /** + * @param ObjectStoreConfig $config + */ + public function buildObjectStore(array $config): IObjectStore { + return new $config['class']($config['arguments']); + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForRoot(): ?array { + $config = $this->getObjectStoreConfig(); + + if ($config && $config['arguments']['multibucket']) { + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + // put the root FS always in first bucket for multibucket configuration + $config['arguments']['bucket'] .= '0'; + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForUser(IUser $user): ?array { + $config = $this->getObjectStoreConfig(); + + if ($config && $config['arguments']['multibucket']) { + $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + private function getObjectStoreConfig(): ?array { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + + // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config + if ($objectStoreMultiBucket) { + $objectStoreMultiBucket['arguments']['multibucket'] = true; + return $this->validateObjectStoreConfig($objectStoreMultiBucket); + } elseif ($objectStore) { + return $this->validateObjectStoreConfig($objectStore); + } else { + return null; + } + } + + /** + * @return ObjectStoreConfig + */ + private function validateObjectStoreConfig(array $config) { + if (!isset($config['class'])) { + throw new \Exception('No class configured for object store'); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $class = $config['class']; + $arguments = $config['arguments']; + if (!is_array($arguments)) { + throw new \Exception('Configured object store arguments are not an array'); + } + if (!isset($arguments['multibucket'])) { + $arguments['multibucket'] = false; + } + if (!is_bool($arguments['multibucket'])) { + throw new \Exception('arguments.multibucket must be a boolean in object store configuration'); + } + + if (!is_string($class)) { + throw new \Exception('Configured class for object store is not a string'); + } + + if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { + [$appId] = explode('\\', $class); + $this->appManager->loadApp(strtolower($appId)); + } + + if (!is_a($class, IObjectStore::class, true)) { + throw new \Exception('Configured class for object store is not an object store'); + } + return [ + 'class' => $class, + 'arguments' => $arguments, + ]; + } + + private function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new Mapper($user, $this->config); + $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); + } + + return $bucket; + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 41ab75caf45..72e1751e23d 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -1,16 +1,19 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later */ + namespace OC\Files\ObjectStore; use Aws\Result; use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore, IObjectStoreMultiPartUpload { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; @@ -61,7 +64,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { 'Key' => $urn, 'UploadId' => $uploadId, 'MaxParts' => 1000, - 'PartNumberMarker' => $partNumberMarker + 'PartNumberMarker' => $partNumberMarker, ] + $this->getSSECParameters()); $parts = array_merge($parts, $result->get('Parts') ?? []); $isTruncated = $result->get('IsTruncated'); @@ -89,7 +92,51 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { $this->getConnection()->abortMultipartUpload([ 'Bucket' => $this->bucket, 'Key' => $urn, - 'UploadId' => $uploadId + 'UploadId' => $uploadId, ]); } + + private function parseS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + if (str_starts_with($key, 'x-amz-meta-')) { + $result[substr($key, strlen('x-amz-meta-'))] = $value; + } + } + return $result; + } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ] + $this->parseS3Metadata($object['Metadata'] ?? []); + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } } diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php index 63f14ac2d00..5b086db8f77 100644 --- a/lib/private/Files/ObjectStore/S3ConfigTrait.php +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -18,6 +18,10 @@ trait S3ConfigTrait { /** Maximum number of concurrent multipart uploads */ protected int $concurrency; + /** Timeout, in seconds, for the connection to S3 server, not for the + * request. */ + protected float $connectTimeout; + protected int $timeout; protected string|false $proxy; diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index cce8752c360..67b82a44ab7 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -39,6 +40,7 @@ trait S3ConnectionTrait { // Default to 5 like the S3 SDK does $this->concurrency = $params['concurrency'] ?? 5; $this->proxy = $params['proxy'] ?? false; + $this->connectTimeout = $params['connect_timeout'] ?? 5; $this->timeout = $params['timeout'] ?? 15; $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; @@ -102,10 +104,13 @@ trait S3ConnectionTrait { 'use_arn_region' => false, 'http' => [ 'verify' => $this->getCertificateBundlePath(), - // Timeout for the connection to S3 server, not for the request. - 'connect_timeout' => 5 + 'connect_timeout' => $this->connectTimeout, ], 'use_aws_shared_config_files' => false, + 'retries' => [ + 'mode' => 'standard', + 'max_attempts' => 5, + ], ]; if ($this->params['s3-accelerate']) { @@ -128,7 +133,7 @@ trait S3ConnectionTrait { $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', ['app' => 'objectstore']); } - + if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { try { $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); @@ -185,10 +190,12 @@ trait S3ConnectionTrait { return function () { $key = empty($this->params['key']) ? null : $this->params['key']; $secret = empty($this->params['secret']) ? null : $this->params['secret']; + $sessionToken = empty($this->params['session_token']) ? null : $this->params['session_token']; if ($key && $secret) { return Create::promiseFor( - new Credentials($key, $secret) + // a null sessionToken match the default signature of the constructor + new Credentials($key, $secret, $sessionToken) ); } @@ -213,7 +220,7 @@ trait S3ConnectionTrait { } protected function getSSECKey(): ?string { - if (isset($this->params['sse_c_key'])) { + if (isset($this->params['sse_c_key']) && !empty($this->params['sse_c_key'])) { return $this->params['sse_c_key']; } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 9d7cfa644e6..89405de2e8e 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -6,6 +6,8 @@ */ namespace OC\Files\ObjectStore; +use Aws\Command; +use Aws\Exception\MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; @@ -77,24 +79,42 @@ trait S3ObjectTrait { return $fh; } + private function buildS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + $result['x-amz-meta-' . $key] = $value; + } + return $result; + } /** * Single object put helper * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeSingle(string $urn, StreamInterface $stream, ?string $mimetype = null): void { - $this->getConnection()->putObject([ + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $args = [ 'Bucket' => $this->bucket, 'Key' => $urn, 'Body' => $stream, 'ACL' => 'private', 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters()); + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); } @@ -103,45 +123,91 @@ trait S3ObjectTrait { * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeMultiPart(string $urn, StreamInterface $stream, ?string $mimetype = null): void { - $uploader = new MultipartUploader($this->getConnection(), $stream, [ - 'bucket' => $this->bucket, - 'concurrency' => $this->concurrency, - 'key' => $urn, - 'part_size' => $this->uploadPartSize, - 'params' => [ - 'ContentType' => $mimetype, - 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters(), - ]); + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $attempts = 0; + $uploaded = false; + $concurrency = $this->concurrency; + $exception = null; + $state = null; + $size = $stream->getSize(); + $totalWritten = 0; + + // retry multipart upload once with concurrency at half on failure + while (!$uploaded && $attempts <= 1) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'concurrency' => $concurrency, + 'key' => $urn, + 'part_size' => $this->uploadPartSize, + 'state' => $state, + 'params' => [ + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, + ]); + + try { + $uploader->upload(); + $uploaded = true; + } catch (S3MultipartUploadException $e) { + $exception = $e; + $attempts++; + + if ($concurrency > 1) { + $concurrency = round($concurrency / 2); + } + + if ($stream->isSeekable()) { + $stream->rewind(); + } + } catch (MultipartUploadException $e) { + $exception = $e; + break; + } + } - try { - $uploader->upload(); - } catch (S3MultipartUploadException $e) { + if (!$uploaded) { // if anything goes wrong with multipart, make sure that you don´t poison and // slow down s3 bucket with orphaned fragments - $uploadInfo = $e->getState()->getId(); - if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { + $uploadInfo = $exception->getState()->getId(); + if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { $this->getConnection()->abortMultipartUpload($uploadInfo); } - throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $e); + + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception); } } - - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } + + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { $canSeek = fseek($stream, 0, SEEK_CUR) === 0; - $psrStream = Utils::streamFor($stream); + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); $size = $psrStream->getSize(); @@ -154,16 +220,16 @@ trait S3ObjectTrait { $buffer->seek(0); if ($buffer->getSize() < $this->putSizeLimit) { // buffer is fully seekable, so use it directly for the small upload - $this->writeSingle($urn, $buffer, $mimetype); + $this->writeSingle($urn, $buffer, $metaData); } else { $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); - $this->writeMultiPart($urn, $loadStream, $mimetype); + $this->writeMultiPart($urn, $loadStream, $metaData); } } else { if ($size < $this->putSizeLimit) { - $this->writeSingle($urn, $psrStream, $mimetype); + $this->writeSingle($urn, $psrStream, $metaData); } else { - $this->writeMultiPart($urn, $psrStream, $mimetype); + $this->writeMultiPart($urn, $psrStream, $metaData); } } $psrStream->close(); diff --git a/lib/private/Files/ObjectStore/S3Signature.php b/lib/private/Files/ObjectStore/S3Signature.php index 994d65cc219..b80382ff67d 100644 --- a/lib/private/Files/ObjectStore/S3Signature.php +++ b/lib/private/Files/ObjectStore/S3Signature.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php index 4361795ec45..888602a62e4 100644 --- a/lib/private/Files/ObjectStore/StorageObjectStore.php +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php index 2bde6f0bbdb..bb7bef2ed63 100644 --- a/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php +++ b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php @@ -14,8 +14,8 @@ class FlattenNestedBool extends QueryOptimizerStep { public function processOperator(ISearchOperator &$operator) { if ( $operator instanceof SearchBinaryOperator && ( - $operator->getType() === ISearchBinaryOperator::OPERATOR_OR || - $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + || $operator->getType() === ISearchBinaryOperator::OPERATOR_AND ) ) { $newArguments = []; diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php index cb04351534a..7e99c04f197 100644 --- a/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php +++ b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php @@ -16,11 +16,11 @@ class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep { public function processOperator(ISearchOperator &$operator): bool { parent::processOperator($operator); if ( - $operator instanceof ISearchBinaryOperator && - count($operator->getArguments()) === 1 && - ( - $operator->getType() === ISearchBinaryOperator::OPERATOR_OR || - $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + $operator instanceof ISearchBinaryOperator + && count($operator->getArguments()) === 1 + && ( + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + || $operator->getType() === ISearchBinaryOperator::OPERATOR_AND ) ) { $operator = $operator->getArguments()[0]; diff --git a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php index 3f2b36823ab..6df35c9c9a2 100644 --- a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php +++ b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php @@ -18,8 +18,8 @@ use OCP\Files\Search\ISearchOperator; class OrEqualsToIn extends ReplacingOptimizerStep { public function processOperator(ISearchOperator &$operator): bool { if ( - $operator instanceof ISearchBinaryOperator && - $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + $operator instanceof ISearchBinaryOperator + && $operator->getType() === ISearchBinaryOperator::OPERATOR_OR ) { $groups = $this->groupEqualsComparisonsByField($operator->getArguments()); $newParts = array_map(function (array $group) { @@ -32,7 +32,7 @@ class OrEqualsToIn extends ReplacingOptimizerStep { $value = $comparison->getValue(); return $value; }, $group); - $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values); + $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values, $group[0]->getExtra()); $pathEqHash = array_reduce($group, function ($pathEqHash, ISearchComparison $comparison) { return $comparison->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true) && $pathEqHash; }, true); diff --git a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php index 9d50746d5d1..2994a9365a7 100644 --- a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php +++ b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php @@ -52,9 +52,9 @@ class PathPrefixOptimizer extends QueryOptimizerStep { private function operatorPairIsPathPrefix(ISearchOperator $like, ISearchOperator $equal): bool { return ( - $like instanceof ISearchComparison && $equal instanceof ISearchComparison && - !$like->getExtra() && !$equal->getExtra() && $like->getField() === 'path' && $equal->getField() === 'path' && - $like->getType() === ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE && $equal->getType() === ISearchComparison::COMPARE_EQUAL + $like instanceof ISearchComparison && $equal instanceof ISearchComparison + && !$like->getExtra() && !$equal->getExtra() && $like->getField() === 'path' && $equal->getField() === 'path' + && $like->getType() === ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE && $equal->getType() === ISearchComparison::COMPARE_EQUAL && $like->getValue() === SearchComparison::escapeLikeParameter($equal->getValue()) . '/%' ); } diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php index 5c7655b49c1..8aee1975708 100644 --- a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php +++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php @@ -18,13 +18,13 @@ use OCP\Files\Search\ISearchOperator; class SplitLargeIn extends ReplacingOptimizerStep { public function processOperator(ISearchOperator &$operator): bool { if ( - $operator instanceof ISearchComparison && - $operator->getType() === ISearchComparison::COMPARE_IN && - count($operator->getValue()) > 1000 + $operator instanceof ISearchComparison + && $operator->getType() === ISearchComparison::COMPARE_IN + && count($operator->getValue()) > 1000 ) { $chunks = array_chunk($operator->getValue(), 1000); $chunkComparisons = array_map(function (array $values) use ($operator) { - return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values); + return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values, $operator->getExtra()); }, $chunks); $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons); diff --git a/lib/private/Files/Search/SearchBinaryOperator.php b/lib/private/Files/Search/SearchBinaryOperator.php index 59a1ba2dfaf..49f599933f4 100644 --- a/lib/private/Files/Search/SearchBinaryOperator.php +++ b/lib/private/Files/Search/SearchBinaryOperator.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Search/SearchOrder.php b/lib/private/Files/Search/SearchOrder.php index 3dcbc02bc1a..5a036653f4e 100644 --- a/lib/private/Files/Search/SearchOrder.php +++ b/lib/private/Files/Search/SearchOrder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Search/SearchQuery.php b/lib/private/Files/Search/SearchQuery.php index 1793e42d349..592749cf4a0 100644 --- a/lib/private/Files/Search/SearchQuery.php +++ b/lib/private/Files/Search/SearchQuery.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/SetupManager.php b/lib/private/Files/SetupManager.php index 2b8121a529d..b92c608a81d 100644 --- a/lib/private/Files/SetupManager.php +++ b/lib/private/Files/SetupManager.php @@ -21,19 +21,19 @@ use OC\Files\Storage\Wrapper\Quota; use OC\Lockdown\Filesystem\NullStorage; use OC\Share\Share; use OC\Share20\ShareDisableChecker; -use OC_App; use OC_Hook; -use OC_Util; use OCA\Files_External\Config\ExternalMountPoint; use OCA\Files_Sharing\External\Mount; use OCA\Files_Sharing\ISharedMountPoint; use OCA\Files_Sharing\SharedMount; +use OCP\App\IAppManager; use OCP\Constants; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IHomeMountProvider; use OCP\Files\Config\IMountProvider; +use OCP\Files\Config\IRootMountProvider; use OCP\Files\Config\IUserMountCache; use OCP\Files\Events\BeforeFileSystemSetupEvent; use OCP\Files\Events\InvalidateMountCacheEvent; @@ -81,6 +81,7 @@ class SetupManager { private LoggerInterface $logger, private IConfig $config, private ShareDisableChecker $shareDisableChecker, + private IAppManager $appManager, ) { $this->cache = $cacheFactory->createDistributed('setupmanager::'); $this->listeningForProviders = false; @@ -104,7 +105,7 @@ class SetupManager { $this->setupBuiltinWrappersDone = true; // load all filesystem apps before, so no setup-hook gets lost - OC_App::loadApps(['filesystem']); + $this->appManager->loadApps(['filesystem']); $prevLogging = Filesystem::logWarningWhenAddingStorageWrapper(false); Filesystem::addStorageWrapper('mount_options', function ($mountPoint, IStorage $storage, IMountPoint $mount) { @@ -156,7 +157,7 @@ class SetupManager { if ($mount instanceof HomeMountPoint) { $user = $mount->getUser(); return new Quota(['storage' => $storage, 'quotaCallback' => function () use ($user) { - return OC_Util::getUserQuota($user); + return $user->getQuotaBytes(); }, 'root' => 'files', 'include_external_storage' => $quotaIncludeExternal]); } @@ -171,9 +172,9 @@ class SetupManager { return new PermissionsMask([ 'storage' => $storage, 'mask' => Constants::PERMISSION_ALL & ~( - Constants::PERMISSION_UPDATE | - Constants::PERMISSION_CREATE | - Constants::PERMISSION_DELETE + Constants::PERMISSION_UPDATE + | Constants::PERMISSION_CREATE + | Constants::PERMISSION_DELETE ), ]); } @@ -280,14 +281,18 @@ class SetupManager { $mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot) { return str_starts_with($mount->getMountPoint(), $userRoot); }); - $allProviders = array_map(function (IMountProvider $provider) { + $allProviders = array_map(function (IMountProvider|IHomeMountProvider|IRootMountProvider $provider) { return get_class($provider); - }, $this->mountProviderCollection->getProviders()); + }, array_merge( + $this->mountProviderCollection->getProviders(), + $this->mountProviderCollection->getHomeProviders(), + $this->mountProviderCollection->getRootProviders(), + )); $newProviders = array_diff($allProviders, $previouslySetupProviders); $mounts = array_filter($mounts, function (IMountPoint $mount) use ($previouslySetupProviders) { return !in_array($mount->getMountProvider(), $previouslySetupProviders); }); - $this->userMountCache->registerMounts($user, $mounts, $newProviders); + $this->registerMounts($user, $mounts, $newProviders); $cacheDuration = $this->config->getSystemValueInt('fs_mount_cache_duration', 5 * 60); if ($cacheDuration > 0) { @@ -452,7 +457,7 @@ class SetupManager { } if (count($mounts)) { - $this->userMountCache->registerMounts($user, $mounts, $currentProviders); + $this->registerMounts($user, $mounts, $currentProviders); $this->setupForUserWith($user, function () use ($mounts) { array_walk($mounts, [$this->mountManager, 'addMount']); }); @@ -523,7 +528,7 @@ class SetupManager { $mounts = $this->mountProviderCollection->getUserMountsForProviderClasses($user, $providers); } - $this->userMountCache->registerMounts($user, $mounts, $providers); + $this->registerMounts($user, $mounts, $providers); $this->setupForUserWith($user, function () use ($mounts) { array_walk($mounts, [$this->mountManager, 'addMount']); }); @@ -595,4 +600,10 @@ class SetupManager { }); } } + + private function registerMounts(IUser $user, array $mounts, ?array $mountProviderClasses = null): void { + if ($this->lockdownManager->canAccessFilesystem()) { + $this->userMountCache->registerMounts($user, $mounts, $mountProviderClasses); + } + } } diff --git a/lib/private/Files/SetupManagerFactory.php b/lib/private/Files/SetupManagerFactory.php index 6ae38c6fdbc..d2fe978fa9e 100644 --- a/lib/private/Files/SetupManagerFactory.php +++ b/lib/private/Files/SetupManagerFactory.php @@ -9,6 +9,7 @@ declare(strict_types=1); namespace OC\Files; use OC\Share20\ShareDisableChecker; +use OCP\App\IAppManager; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Config\IMountProviderCollection; @@ -36,6 +37,7 @@ class SetupManagerFactory { private LoggerInterface $logger, private IConfig $config, private ShareDisableChecker $shareDisableChecker, + private IAppManager $appManager, ) { $this->setupManager = null; } @@ -55,6 +57,7 @@ class SetupManagerFactory { $this->logger, $this->config, $this->shareDisableChecker, + $this->appManager, ); } return $this->setupManager; diff --git a/lib/private/Files/SimpleFS/SimpleFile.php b/lib/private/Files/SimpleFS/SimpleFile.php index cbb3af4db29..d9c1b47d2f1 100644 --- a/lib/private/Files/SimpleFS/SimpleFile.php +++ b/lib/private/Files/SimpleFS/SimpleFile.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -6,9 +7,11 @@ namespace OC\Files\SimpleFS; use OCP\Files\File; +use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\ISimpleFile; +use OCP\Lock\LockedException; class SimpleFile implements ISimpleFile { private File $file; @@ -48,8 +51,10 @@ class SimpleFile implements ISimpleFile { /** * Get the content * - * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException * @throws NotFoundException + * @throws NotPermittedException */ public function getContent(): string { $result = $this->file->getContent(); @@ -65,8 +70,10 @@ class SimpleFile implements ISimpleFile { * Overwrite the file * * @param string|resource $data - * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException * @throws NotFoundException + * @throws NotPermittedException */ public function putContent($data): void { try { diff --git a/lib/private/Files/SimpleFS/SimpleFolder.php b/lib/private/Files/SimpleFS/SimpleFolder.php index 0da552e402b..62f3db25e9b 100644 --- a/lib/private/Files/SimpleFS/SimpleFolder.php +++ b/lib/private/Files/SimpleFS/SimpleFolder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php index ca0e38774c8..2dc359169d7 100644 --- a/lib/private/Files/Storage/Common.php +++ b/lib/private/Files/Storage/Common.php @@ -19,6 +19,7 @@ use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\Storage\Wrapper\Encryption; use OC\Files\Storage\Wrapper\Jail; use OC\Files\Storage\Wrapper\Wrapper; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IScanner; @@ -205,7 +206,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, } else { $sourceStream = $this->fopen($source, 'r'); $targetStream = $this->fopen($target, 'w'); - [, $result] = \OC_Helper::streamCopy($sourceStream, $targetStream); + [, $result] = Files::streamCopy($sourceStream, $targetStream, true); if (!$result) { Server::get(LoggerInterface::class)->warning("Failed to write data while copying $source to $target"); } @@ -549,8 +550,8 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { if ( - !$sourceStorage->instanceOfStorage(Encryption::class) && - $this->isSameStorage($sourceStorage) + !$sourceStorage->instanceOfStorage(Encryption::class) + && $this->isSameStorage($sourceStorage) ) { // resolve any jailed paths while ($sourceStorage->instanceOfStorage(Jail::class)) { @@ -734,7 +735,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, throw new GenericFileException("Failed to open $path for writing"); } try { - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + [$count, $result] = Files::streamCopy($stream, $target, true); if (!$result) { throw new GenericFileException('Failed to copy stream'); } diff --git a/lib/private/Files/Storage/DAV.php b/lib/private/Files/Storage/DAV.php index 10670d6331a..2d166b5438d 100644 --- a/lib/private/Files/Storage/DAV.php +++ b/lib/private/Files/Storage/DAV.php @@ -116,7 +116,7 @@ class DAV extends Common { // inject mock for testing $this->certManager = \OC::$server->getCertificateManager(); } - $this->root = $parameters['root'] ?? '/'; + $this->root = rawurldecode($parameters['root'] ?? '/'); $this->root = '/' . ltrim($this->root, '/'); $this->root = rtrim($this->root, '/') . '/'; } else { @@ -191,7 +191,7 @@ class DAV extends Common { if ($this->secure) { $baseUri .= 's'; } - $baseUri .= '://' . $this->host . $this->root; + $baseUri .= '://' . $this->host . $this->encodePath($this->root); return $baseUri; } @@ -350,7 +350,13 @@ class DAV extends Common { } } - return $response->getBody(); + $content = $response->getBody(); + + if ($content === null || is_string($content)) { + return false; + } + + return $content; case 'w': case 'wb': case 'a': @@ -390,6 +396,8 @@ class DAV extends Common { $this->writeBack($tmpFile, $path); }); } + + return false; } public function writeBack(string $tmpFile, string $path): void { diff --git a/lib/private/Files/Storage/LocalTempFileTrait.php b/lib/private/Files/Storage/LocalTempFileTrait.php index c8e3588f2e8..fffc3e789f3 100644 --- a/lib/private/Files/Storage/LocalTempFileTrait.php +++ b/lib/private/Files/Storage/LocalTempFileTrait.php @@ -7,6 +7,8 @@ */ namespace OC\Files\Storage; +use OCP\Files; + /** * Storage backend class for providing common filesystem operation methods * which are not storage-backend specific. @@ -45,7 +47,7 @@ trait LocalTempFileTrait { } $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($extension); $target = fopen($tmpFile, 'w'); - \OC_Helper::streamCopy($source, $target); + Files::streamCopy($source, $target); fclose($target); return $tmpFile; } diff --git a/lib/private/Files/Storage/Temporary.php b/lib/private/Files/Storage/Temporary.php index ff7a816930d..ecf8a1315a9 100644 --- a/lib/private/Files/Storage/Temporary.php +++ b/lib/private/Files/Storage/Temporary.php @@ -7,16 +7,20 @@ */ namespace OC\Files\Storage; +use OCP\Files; +use OCP\ITempManager; +use OCP\Server; + /** * local storage backend in temporary folder for testing purpose */ class Temporary extends Local { public function __construct(array $parameters = []) { - parent::__construct(['datadir' => \OC::$server->getTempManager()->getTemporaryFolder()]); + parent::__construct(['datadir' => Server::get(ITempManager::class)->getTemporaryFolder()]); } public function cleanUp(): void { - \OC_Helper::rmdirr($this->datadir); + Files::rmdirr($this->datadir); } public function __destruct() { diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php index eb073d24087..58bd4dfddcf 100644 --- a/lib/private/Files/Storage/Wrapper/Encryption.php +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -8,7 +8,6 @@ namespace OC\Files\Storage\Wrapper; use OC\Encryption\Exceptions\ModuleDoesNotExistsException; -use OC\Encryption\Update; use OC\Encryption\Util; use OC\Files\Cache\CacheEntry; use OC\Files\Filesystem; @@ -18,10 +17,13 @@ use OC\Files\Storage\Common; use OC\Files\Storage\LocalTempFileTrait; use OC\Memcache\ArrayCache; use OCP\Cache\CappedMemoryCache; +use OCP\Encryption\Exceptions\InvalidHeaderException; use OCP\Encryption\IFile; use OCP\Encryption\IManager; use OCP\Encryption\Keys\IStorage; +use OCP\Files; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\GenericFileException; use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage; use Psr\Log\LoggerInterface; @@ -49,7 +51,6 @@ class Encryption extends Wrapper { private IFile $fileHelper, private ?string $uid, private IStorage $keyStorage, - private Update $update, private Manager $mountManager, private ArrayCache $arrayCache, ) { @@ -64,7 +65,8 @@ class Encryption extends Wrapper { $info = $this->getCache()->get($path); if ($info === false) { - return false; + /* Pass call to wrapped storage, it may be a special file like a part file */ + return $this->storage->filesize($path); } if (isset($this->unencryptedSize[$fullPath])) { $size = $this->unencryptedSize[$fullPath]; @@ -184,11 +186,11 @@ class Encryption extends Wrapper { public function rename(string $source, string $target): bool { $result = $this->storage->rename($source, $target); - if ($result && + if ($result // versions always use the keys from the original file, so we can skip // this step for versions - $this->isVersion($target) === false && - $this->encryptionManager->isEnabled()) { + && $this->isVersion($target) === false + && $this->encryptionManager->isEnabled()) { $sourcePath = $this->getFullPath($source); if (!$this->util->isExcluded($sourcePath)) { $targetPath = $this->getFullPath($target); @@ -209,9 +211,9 @@ class Encryption extends Wrapper { public function rmdir(string $path): bool { $result = $this->storage->rmdir($path); $fullPath = $this->getFullPath($path); - if ($result && - $this->util->isExcluded($fullPath) === false && - $this->encryptionManager->isEnabled() + if ($result + && $this->util->isExcluded($fullPath) === false + && $this->encryptionManager->isEnabled() ) { $this->keyStorage->deleteAllFileKeys($fullPath); } @@ -224,9 +226,9 @@ class Encryption extends Wrapper { $metaData = $this->getMetaData($path); if ( - !$this->is_dir($path) && - isset($metaData['encrypted']) && - $metaData['encrypted'] === true + !$this->is_dir($path) + && isset($metaData['encrypted']) + && $metaData['encrypted'] === true ) { $fullPath = $this->getFullPath($path); $module = $this->getEncryptionModule($path); @@ -318,7 +320,7 @@ class Encryption extends Wrapper { if (!empty($encryptionModuleId)) { $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); $shouldEncrypt = true; - } elseif (empty($encryptionModuleId) && $info['encrypted'] === true) { + } elseif ($info !== false && $info['encrypted'] === true) { // we come from a old installation. No header and/or no module defined // but the file is encrypted. In this case we need to use the // OC_DEFAULT_MODULE to read the file @@ -344,6 +346,16 @@ class Encryption extends Wrapper { if ($shouldEncrypt === true && $encryptionModule !== null) { $this->encryptedPaths->set($this->util->stripPartialFileExtension($path), true); $headerSize = $this->getHeaderSize($path); + if ($mode === 'r' && $headerSize === 0) { + $firstBlock = $this->readFirstBlock($path); + if (!$firstBlock) { + throw new InvalidHeaderException("Unable to get header block for $path"); + } elseif (!str_starts_with($firstBlock, Util::HEADER_START)) { + throw new InvalidHeaderException("Unable to get header size for $path, file doesn't start with encryption header"); + } else { + throw new InvalidHeaderException("Unable to get header size for $path, even though file does start with encryption header"); + } + } $source = $this->storage->fopen($path, $mode); if (!is_resource($source)) { return false; @@ -373,9 +385,9 @@ class Encryption extends Wrapper { $size = $this->storage->filesize($path); $result = $unencryptedSize; - if ($unencryptedSize < 0 || - ($size > 0 && $unencryptedSize === $size) || - $unencryptedSize > $size + if ($unencryptedSize < 0 + || ($size > 0 && $unencryptedSize === $size) + || $unencryptedSize > $size ) { // check if we already calculate the unencrypted size for the // given path to avoid recursions @@ -526,10 +538,22 @@ class Encryption extends Wrapper { $result = $this->copyBetweenStorage($sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, true); if ($result) { - if ($sourceStorage->is_dir($sourceInternalPath)) { - $result = $sourceStorage->rmdir($sourceInternalPath); - } else { - $result = $sourceStorage->unlink($sourceInternalPath); + $setPreserveCacheOnDelete = $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && !$this->instanceOfStorage(ObjectStoreStorage::class); + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + try { + if ($sourceStorage->is_dir($sourceInternalPath)) { + $result = $sourceStorage->rmdir($sourceInternalPath); + } else { + $result = $sourceStorage->unlink($sourceInternalPath); + } + } finally { + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } } } return $result; @@ -611,8 +635,8 @@ class Encryption extends Wrapper { ): bool { // for versions we have nothing to do, because versions should always use the // key from the original file. Just create a 1:1 copy and done - if ($this->isVersion($targetInternalPath) || - $this->isVersion($sourceInternalPath)) { + if ($this->isVersion($targetInternalPath) + || $this->isVersion($sourceInternalPath)) { // remember that we try to create a version so that we can detect it during // fopen($sourceInternalPath) and by-pass the encryption in order to // create a 1:1 copy of the file @@ -628,6 +652,7 @@ class Encryption extends Wrapper { $info->getUnencryptedSize() ); } + $this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true); } return $result; } @@ -653,7 +678,7 @@ class Encryption extends Wrapper { if (is_resource($dh)) { while ($result && ($file = readdir($dh)) !== false) { if (!Filesystem::isIgnoredDir($file)) { - $result &= $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file, false, $isRename); + $result = $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file, $preserveMtime, $isRename); } } } @@ -661,12 +686,16 @@ class Encryption extends Wrapper { try { $source = $sourceStorage->fopen($sourceInternalPath, 'r'); $target = $this->fopen($targetInternalPath, 'w'); - [, $result] = \OC_Helper::streamCopy($source, $target); + if ($source === false || $target === false) { + $result = false; + } else { + [, $result] = Files::streamCopy($source, $target, true); + } } finally { - if (is_resource($source)) { + if (isset($source) && $source !== false) { fclose($source); } - if (is_resource($target)) { + if (isset($target) && $target !== false) { fclose($target); } } @@ -716,6 +745,9 @@ class Encryption extends Wrapper { public function hash(string $type, string $path, bool $raw = false): string|false { $fh = $this->fopen($path, 'rb'); + if ($fh === false) { + return false; + } $ctx = hash_init($type); hash_update_stream($ctx, $fh); fclose($fh); @@ -740,6 +772,9 @@ class Encryption extends Wrapper { $firstBlock = ''; if ($this->storage->is_file($path)) { $handle = $this->storage->fopen($path, 'r'); + if ($handle === false) { + return ''; + } $firstBlock = fread($handle, $this->util->getHeaderSize()); fclose($handle); } @@ -870,7 +905,10 @@ class Encryption extends Wrapper { public function writeStream(string $path, $stream, ?int $size = null): int { // always fall back to fopen $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + if ($target === false) { + throw new GenericFileException("Failed to open $path for writing"); + } + [$count, $result] = Files::streamCopy($stream, $target, true); fclose($stream); fclose($target); @@ -893,4 +931,16 @@ class Encryption extends Wrapper { public function setEnabled(bool $enabled): void { $this->enabled = $enabled; } + + /** + * Check if the on-disk data for a file has a valid encrypted header + * + * @param string $path + * @return bool + */ + public function hasValidHeader(string $path): bool { + $firstBlock = $this->readFirstBlock($path); + $header = $this->util->parseRawHeader($firstBlock); + return (count($header) > 0); + } } diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php index c8db0e94e59..38b113cef88 100644 --- a/lib/private/Files/Storage/Wrapper/Jail.php +++ b/lib/private/Files/Storage/Wrapper/Jail.php @@ -11,6 +11,7 @@ use OC\Files\Cache\Wrapper\CacheJail; use OC\Files\Cache\Wrapper\JailPropagator; use OC\Files\Cache\Wrapper\JailWatcher; use OC\Files\Filesystem; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IWatcher; @@ -253,7 +254,7 @@ class Jail extends Wrapper { return $storage->writeStream($this->getUnjailedPath($path), $stream, $size); } else { $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + $count = Files::streamCopy($stream, $target); fclose($stream); fclose($target); return $count; diff --git a/lib/private/Files/Storage/Wrapper/Quota.php b/lib/private/Files/Storage/Wrapper/Quota.php index 3be77ba1b37..35a265f8c8e 100644 --- a/lib/private/Files/Storage/Wrapper/Quota.php +++ b/lib/private/Files/Storage/Wrapper/Quota.php @@ -21,6 +21,7 @@ class Quota extends Wrapper { protected string $sizeRoot; private SystemConfig $config; private bool $quotaIncludeExternalStorage; + private bool $enabled = true; /** * @param array $parameters @@ -46,6 +47,9 @@ class Quota extends Wrapper { } private function hasQuota(): bool { + if (!$this->enabled) { + return false; + } return $this->getQuota() !== FileInfo::SPACE_UNLIMITED; } @@ -197,4 +201,8 @@ class Quota extends Wrapper { return parent::touch($path, $mtime); } + + public function enableQuota(bool $enabled): void { + $this->enabled = $enabled; + } } diff --git a/lib/private/Files/Storage/Wrapper/Wrapper.php b/lib/private/Files/Storage/Wrapper/Wrapper.php index 6dea439fe25..7af11dd5ef7 100644 --- a/lib/private/Files/Storage/Wrapper/Wrapper.php +++ b/lib/private/Files/Storage/Wrapper/Wrapper.php @@ -9,6 +9,7 @@ namespace OC\Files\Storage\Wrapper; use OC\Files\Storage\FailedStorage; use OC\Files\Storage\Storage; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IScanner; @@ -322,7 +323,7 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea return $storage->writeStream($path, $stream, $size); } else { $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + $count = Files::streamCopy($stream, $target); fclose($stream); fclose($target); return $count; diff --git a/lib/private/Files/Stream/Encryption.php b/lib/private/Files/Stream/Encryption.php index 0d55385820c..ef147ec421f 100644 --- a/lib/private/Files/Stream/Encryption.php +++ b/lib/private/Files/Stream/Encryption.php @@ -312,8 +312,8 @@ class Encryption extends Wrapper { // for seekable streams the pointer is moved back to the beginning of the encrypted block // flush will start writing there when the position moves to another block - $positionInFile = (int)floor($this->position / $this->unencryptedBlockSize) * - $this->util->getBlockSize() + $this->headerSize; + $positionInFile = (int)floor($this->position / $this->unencryptedBlockSize) + * $this->util->getBlockSize() + $this->headerSize; $resultFseek = $this->parentStreamSeek($positionInFile); // only allow writes on seekable streams, or at the end of the encrypted stream @@ -336,8 +336,8 @@ class Encryption extends Wrapper { // if $data doesn't fit the current block, the fill the current block and reiterate // after the block is filled, it is flushed and $data is updatedxxx } else { - $this->cache = substr($this->cache, 0, $blockPosition) . - substr($data, 0, $this->unencryptedBlockSize - $blockPosition); + $this->cache = substr($this->cache, 0, $blockPosition) + . substr($data, 0, $this->unencryptedBlockSize - $blockPosition); $this->flush(); $this->position += ($this->unencryptedBlockSize - $blockPosition); $length += ($this->unencryptedBlockSize - $blockPosition); diff --git a/lib/private/Files/Stream/SeekableHttpStream.php b/lib/private/Files/Stream/SeekableHttpStream.php index 5ed04ed9066..6ce0a880e8d 100644 --- a/lib/private/Files/Stream/SeekableHttpStream.php +++ b/lib/private/Files/Stream/SeekableHttpStream.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/Files/Template/TemplateManager.php b/lib/private/Files/Template/TemplateManager.php index 5a3f16872e6..80ef5a14786 100644 --- a/lib/private/Files/Template/TemplateManager.php +++ b/lib/private/Files/Template/TemplateManager.php @@ -19,6 +19,7 @@ use OCP\Files\IRootFolder; use OCP\Files\Node; use OCP\Files\NotFoundException; use OCP\Files\Template\BeforeGetTemplatesEvent; +use OCP\Files\Template\Field; use OCP\Files\Template\FileCreatedFromTemplateEvent; use OCP\Files\Template\ICustomTemplateProvider; use OCP\Files\Template\ITemplateManager; @@ -125,6 +126,19 @@ class TemplateManager implements ITemplateManager { }, $this->listCreators())); } + public function listTemplateFields(int $fileId): array { + foreach ($this->listCreators() as $creator) { + $fields = $this->getTemplateFields($creator, $fileId); + if (empty($fields)) { + continue; + } + + return $fields; + } + + return []; + } + /** * @param string $filePath * @param string $templateId @@ -179,7 +193,6 @@ class TemplateManager implements ITemplateManager { if ($path instanceof Folder) { return $path; } - $this->logger->warning('Template folder ' . $path . ' not found or invalid', ['app' => 'files_templates']); } throw new NotFoundException(); } @@ -188,6 +201,20 @@ class TemplateManager implements ITemplateManager { * @return list<Template> */ private function getTemplateFiles(TemplateFileCreator $type): array { + $templates = array_merge( + $this->getProviderTemplates($type), + $this->getUserTemplates($type) + ); + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($templates, false)); + + return $templates; + } + + /** + * @return list<Template> + */ + private function getProviderTemplates(TemplateFileCreator $type): array { $templates = []; foreach ($this->getRegisteredProviders() as $provider) { foreach ($type->getMimetypes() as $mimetype) { @@ -196,11 +223,22 @@ class TemplateManager implements ITemplateManager { } } } + + return $templates; + } + + /** + * @return list<Template> + */ + private function getUserTemplates(TemplateFileCreator $type): array { + $templates = []; + try { $userTemplateFolder = $this->getTemplateFolder(); } catch (\Exception $e) { return $templates; } + foreach ($type->getMimetypes() as $mimetype) { foreach ($userTemplateFolder->searchByMime($mimetype) as $templateFile) { $template = new Template( @@ -213,11 +251,33 @@ class TemplateManager implements ITemplateManager { } } - $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($templates)); - return $templates; } + /* + * @return list<Field> + */ + private function getTemplateFields(TemplateFileCreator $type, int $fileId): array { + $providerTemplates = $this->getProviderTemplates($type); + $userTemplates = $this->getUserTemplates($type); + + $matchedTemplates = array_filter( + array_merge($providerTemplates, $userTemplates), + function (Template $template) use ($fileId) { + return $template->jsonSerialize()['fileid'] === $fileId; + }); + + if (empty($matchedTemplates)) { + return []; + } + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($matchedTemplates, true)); + + return array_values(array_map(function (Template $template) { + return $template->jsonSerialize()['fields'] ?? []; + }, $matchedTemplates)); + } + /** * @param Node|File $file * @return array diff --git a/lib/private/Files/Type/Detection.php b/lib/private/Files/Type/Detection.php index b1e4c098e54..6af6ce1a0b1 100644 --- a/lib/private/Files/Type/Detection.php +++ b/lib/private/Files/Type/Detection.php @@ -9,6 +9,8 @@ declare(strict_types=1); namespace OC\Files\Type; use OCP\Files\IMimeTypeDetector; +use OCP\IBinaryFinder; +use OCP\ITempManager; use OCP\IURLGenerator; use Psr\Log\LoggerInterface; @@ -22,13 +24,17 @@ use Psr\Log\LoggerInterface; class Detection implements IMimeTypeDetector { private const CUSTOM_MIMETYPEMAPPING = 'mimetypemapping.json'; private const CUSTOM_MIMETYPEALIASES = 'mimetypealiases.json'; + private const CUSTOM_MIMETYPENAMES = 'mimetypenames.json'; - protected array $mimetypes = []; + /** @var array<list{string, string|null}> */ + protected array $mimeTypes = []; protected array $secureMimeTypes = []; - protected array $mimetypeIcons = []; + protected array $mimeTypeIcons = []; /** @var array<string,string> */ protected array $mimeTypeAlias = []; + /** @var array<string,string> */ + protected array $mimeTypesNames = []; public function __construct( private IURLGenerator $urlGenerator, @@ -39,40 +45,47 @@ class Detection implements IMimeTypeDetector { } /** - * Add an extension -> mimetype mapping + * Add an extension -> MIME type mapping * - * $mimetype is the assumed correct mime type + * $mimeType is the assumed correct mime type * The optional $secureMimeType is an alternative to send to send * to avoid potential XSS. * * @param string $extension - * @param string $mimetype + * @param string $mimeType * @param string|null $secureMimeType */ - public function registerType(string $extension, - string $mimetype, + public function registerType( + string $extension, + string $mimeType, ?string $secureMimeType = null): void { - $this->mimetypes[$extension] = [$mimetype, $secureMimeType]; - $this->secureMimeTypes[$mimetype] = $secureMimeType ?: $mimetype; + // Make sure the extension is a string + // https://github.com/nextcloud/server/issues/42902 + $this->mimeTypes[$extension] = [$mimeType, $secureMimeType]; + $this->secureMimeTypes[$mimeType] = $secureMimeType ?? $mimeType; } /** - * Add an array of extension -> mimetype mappings + * Add an array of extension -> MIME type mappings * - * The mimetype value is in itself an array where the first index is - * the assumed correct mimetype and the second is either a secure alternative + * The mimeType value is in itself an array where the first index is + * the assumed correct mimeType and the second is either a secure alternative * or null if the correct is considered secure. * * @param array $types */ public function registerTypeArray(array $types): void { - $this->mimetypes = array_merge($this->mimetypes, $types); + // Register the types, + foreach ($types as $extension => $mimeType) { + $this->registerType((string)$extension, $mimeType[0], $mimeType[1] ?? null); + } - // Update the alternative mimetypes to avoid having to look them up each time. - foreach ($this->mimetypes as $extension => $mimeType) { - if (str_starts_with($extension, '_comment')) { + // Update the alternative mimeTypes to avoid having to look them up each time. + foreach ($this->mimeTypes as $extension => $mimeType) { + if (str_starts_with((string)$extension, '_comment')) { continue; } + $this->secureMimeTypes[$mimeType[0]] = $mimeType[1] ?? $mimeType[0]; if (isset($mimeType[1])) { $this->secureMimeTypes[$mimeType[1]] = $mimeType[1]; @@ -93,7 +106,7 @@ class Detection implements IMimeTypeDetector { } /** - * Add the mimetype aliases if they are not yet present + * Add the MIME type aliases if they are not yet present */ private function loadAliases(): void { if (!empty($this->mimeTypeAlias)) { @@ -119,29 +132,48 @@ class Detection implements IMimeTypeDetector { } /** - * Add mimetype mappings if they are not yet present + * Add MIME type mappings if they are not yet present */ private function loadMappings(): void { - if (!empty($this->mimetypes)) { + if (!empty($this->mimeTypes)) { return; } - $mimetypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypemapping.dist.json'), true); - $mimetypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPEMAPPING, $mimetypeMapping); + $mimeTypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypemapping.dist.json'), true); + $mimeTypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPEMAPPING, $mimeTypeMapping); - $this->registerTypeArray($mimetypeMapping); + $this->registerTypeArray($mimeTypeMapping); } /** - * @return array + * @return array<list{string, string|null}> */ public function getAllMappings(): array { $this->loadMappings(); - return $this->mimetypes; + return $this->mimeTypes; + } + + private function loadNamings(): void { + if (!empty($this->mimeTypesNames)) { + return; + } + + $mimeTypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypenames.dist.json'), true); + $mimeTypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPENAMES, $mimeTypeMapping); + + $this->mimeTypesNames = $mimeTypeMapping; + } + + /** + * @return array<string,string> + */ + public function getAllNamings(): array { + $this->loadNamings(); + return $this->mimeTypesNames; } /** - * detect mimetype only based on filename, content of file is not used + * detect MIME type only based on filename, content of file is not used * * @param string $path * @return string @@ -163,8 +195,8 @@ class Detection implements IMimeTypeDetector { $extension = strrchr($fileName, '.'); if ($extension !== false) { $extension = strtolower($extension); - $extension = substr($extension, 1); //remove leading . - return $this->mimetypes[$extension][0] ?? 'application/octet-stream'; + $extension = substr($extension, 1); // remove leading . + return $this->mimeTypes[$extension][0] ?? 'application/octet-stream'; } } @@ -172,7 +204,8 @@ class Detection implements IMimeTypeDetector { } /** - * detect mimetype only based on the content of file + * Detect MIME type only based on the content of file. + * * @param string $path * @return string * @since 18.0.0 @@ -185,14 +218,10 @@ class Detection implements IMimeTypeDetector { return 'httpd/unix-directory'; } - if (function_exists('finfo_open') - && function_exists('finfo_file') - && $finfo = finfo_open(FILEINFO_MIME)) { - $info = @finfo_file($finfo, $path); - finfo_close($finfo); - if ($info) { - $info = strtolower($info); - $mimeType = str_contains($info, ';') ? substr($info, 0, strpos($info, ';')) : $info; + if (class_exists(finfo::class)) { + $finfo = new finfo(FILEINFO_MIME_TYPE); + $mimeType = @$finfo->file($path); + if ($mimeType) { $mimeType = $this->getSecureMimeType($mimeType); if ($mimeType !== 'application/octet-stream') { return $mimeType; @@ -208,7 +237,7 @@ class Detection implements IMimeTypeDetector { if (function_exists('mime_content_type')) { // use mime magic extension if available $mimeType = mime_content_type($path); - if ($mimeType !== false) { + if ($mimeType) { $mimeType = $this->getSecureMimeType($mimeType); if ($mimeType !== 'application/octet-stream') { return $mimeType; @@ -216,28 +245,30 @@ class Detection implements IMimeTypeDetector { } } - if (\OC_Helper::canExecute('file')) { + $binaryFinder = \OCP\Server::get(IBinaryFinder::class); + $program = $binaryFinder->findBinaryPath('file'); + if ($program !== false) { // it looks like we have a 'file' command, // lets see if it does have mime support $path = escapeshellarg($path); - $fp = popen("test -f $path && file -b --mime-type $path", 'r'); - $mimeType = fgets($fp); - pclose($fp); - - if ($mimeType !== false) { - //trim the newline - $mimeType = trim($mimeType); - $mimeType = $this->getSecureMimeType($mimeType); - if ($mimeType !== 'application/octet-stream') { + $fp = popen("test -f $path && $program -b --mime-type $path", 'r'); + if ($fp !== false) { + $mimeType = fgets($fp); + pclose($fp); + if ($mimeType) { + //trim the newline + $mimeType = trim($mimeType); + $mimeType = $this->getSecureMimeType($mimeType); return $mimeType; } } } + return 'application/octet-stream'; } /** - * detect mimetype based on both filename and content + * Detect MIME type based on both filename and content * * @param string $path * @return string @@ -253,29 +284,31 @@ class Detection implements IMimeTypeDetector { } /** - * detect mimetype based on the content of a string + * Detect MIME type based on the content of a string * * @param string $data * @return string */ public function detectString($data): string { - if (function_exists('finfo_open') && function_exists('finfo_file')) { - $finfo = finfo_open(FILEINFO_MIME); - $info = finfo_buffer($finfo, $data); - return str_contains($info, ';') ? substr($info, 0, strpos($info, ';')) : $info; + if (class_exists(finfo::class)) { + $finfo = new finfo(FILEINFO_MIME_TYPE); + $mimeType = $finfo->buffer($data); + if ($mimeType) { + return $mimeType; + } } - $tmpFile = \OC::$server->getTempManager()->getTemporaryFile(); + $tmpFile = \OCP\Server::get(ITempManager::class)->getTemporaryFile(); $fh = fopen($tmpFile, 'wb'); fwrite($fh, $data, 8024); fclose($fh); - $mime = $this->detect($tmpFile); + $mimeType = $this->detect($tmpFile); unset($tmpFile); - return $mime; + return $mimeType; } /** - * Get a secure mimetype that won't expose potential XSS. + * Get a secure MIME type that won't expose potential XSS. * * @param string $mimeType * @return string @@ -288,57 +321,56 @@ class Detection implements IMimeTypeDetector { /** * Get path to the icon of a file type - * @param string $mimetype the MIME type + * @param string $mimeType the MIME type * @return string the url */ - public function mimeTypeIcon($mimetype): string { + public function mimeTypeIcon($mimeType): string { $this->loadAliases(); - while (isset($this->mimeTypeAlias[$mimetype])) { - $mimetype = $this->mimeTypeAlias[$mimetype]; + while (isset($this->mimeTypeAlias[$mimeType])) { + $mimeType = $this->mimeTypeAlias[$mimeType]; } - if (isset($this->mimetypeIcons[$mimetype])) { - return $this->mimetypeIcons[$mimetype]; + if (isset($this->mimeTypeIcons[$mimeType])) { + return $this->mimeTypeIcons[$mimeType]; } // Replace slash and backslash with a minus - $icon = str_replace(['/', '\\'], '-', $mimetype); + $icon = str_replace(['/', '\\'], '-', $mimeType); // Is it a dir? - if ($mimetype === 'dir') { - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/folder.svg'); - return $this->mimetypeIcons[$mimetype]; + if ($mimeType === 'dir') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder.svg'); + return $this->mimeTypeIcons[$mimeType]; } - if ($mimetype === 'dir-shared') { - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/folder-shared.svg'); - return $this->mimetypeIcons[$mimetype]; + if ($mimeType === 'dir-shared') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder-shared.svg'); + return $this->mimeTypeIcons[$mimeType]; } - if ($mimetype === 'dir-external') { - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/folder-external.svg'); - return $this->mimetypeIcons[$mimetype]; + if ($mimeType === 'dir-external') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder-external.svg'); + return $this->mimeTypeIcons[$mimeType]; } // Icon exists? try { - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/' . $icon . '.svg'); - return $this->mimetypeIcons[$mimetype]; + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/' . $icon . '.svg'); + return $this->mimeTypeIcons[$mimeType]; } catch (\RuntimeException $e) { // Specified image not found } // Try only the first part of the filetype - if (strpos($icon, '-')) { $mimePart = substr($icon, 0, strpos($icon, '-')); try { - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/' . $mimePart . '.svg'); - return $this->mimetypeIcons[$mimetype]; + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/' . $mimePart . '.svg'); + return $this->mimeTypeIcons[$mimeType]; } catch (\RuntimeException $e) { - // Image for the first part of the mimetype not found + // Image for the first part of the MIME type not found } } - $this->mimetypeIcons[$mimetype] = $this->urlGenerator->imagePath('core', 'filetypes/file.svg'); - return $this->mimetypeIcons[$mimetype]; + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/file.svg'); + return $this->mimeTypeIcons[$mimeType]; } } diff --git a/lib/private/Files/Utils/PathHelper.php b/lib/private/Files/Utils/PathHelper.php index a6ae029b957..db1294bcc10 100644 --- a/lib/private/Files/Utils/PathHelper.php +++ b/lib/private/Files/Utils/PathHelper.php @@ -37,6 +37,8 @@ class PathHelper { if ($path === '' or $path === '/') { return '/'; } + // No null bytes + $path = str_replace(chr(0), '', $path); //no windows style slashes $path = str_replace('\\', '/', $path); //add leading slash diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 40201d10b99..576cb66b3cf 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -23,11 +23,13 @@ use OCP\Files\Events\FileScannedEvent; use OCP\Files\Events\FolderScannedEvent; use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\NodeRemovedFromCache; +use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\Storage\IStorage; use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; use Psr\Log\LoggerInterface; /** @@ -85,7 +87,7 @@ class Scanner extends PublicEmitter { * get all storages for $dir * * @param string $dir - * @return \OC\Files\Mount\MountPoint[] + * @return array<string, IMountPoint> */ protected function getMounts($dir) { //TODO: move to the node based fileapi once that's done @@ -96,8 +98,9 @@ class Scanner extends PublicEmitter { $mounts = $mountManager->findIn($dir); $mounts[] = $mountManager->find($dir); $mounts = array_reverse($mounts); //start with the mount of $dir + $mountPoints = array_map(fn ($mount) => $mount->getMountPoint(), $mounts); - return $mounts; + return array_combine($mountPoints, $mounts); } /** @@ -202,7 +205,10 @@ class Scanner extends PublicEmitter { foreach (['', 'files'] as $path) { if (!$storage->isCreatable($path)) { $fullPath = $storage->getSourcePath($path); - if (!$storage->is_dir($path) && $storage->getCache()->inCache($path)) { + if (isset($mounts[$mount->getMountPoint() . $path . '/'])) { + // /<user>/files is overwritten by a mountpoint, so this check is irrelevant + break; + } elseif (!$storage->is_dir($path) && $storage->getCache()->inCache($path)) { throw new NotFoundException("User folder $fullPath exists in cache but not on disk"); } elseif ($storage->is_dir($path)) { $ownerUid = fileowner($fullPath); @@ -255,7 +261,15 @@ class Scanner extends PublicEmitter { try { $propagator = $storage->getPropagator(); $propagator->beginBatch(); - $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + try { + $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + } catch (LockedException $e) { + if (is_string($e->getReadablePath()) && str_starts_with($e->getReadablePath(), 'scanner::')) { + throw new LockedException("scanner::$dir", $e, $e->getExistingLock()); + } else { + throw $e; + } + } $cache = $storage->getCache(); if ($cache instanceof Cache) { // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index e97cd75251d..63eecf5e1d6 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -10,12 +10,14 @@ namespace OC\Files; use Icewind\Streams\CallbackWrapper; use OC\Files\Mount\MoveableMount; use OC\Files\Storage\Storage; +use OC\Files\Storage\Wrapper\Quota; use OC\Share\Share; use OC\User\LazyUser; use OC\User\Manager as UserManager; use OC\User\User; use OCA\Files_Sharing\SharedMount; use OCP\Constants; +use OCP\Files; use OCP\Files\Cache\ICacheEntry; use OCP\Files\ConnectionLostException; use OCP\Files\EmptyFileNameException; @@ -24,11 +26,13 @@ use OCP\Files\ForbiddenException; use OCP\Files\InvalidCharacterInPathException; use OCP\Files\InvalidDirectoryException; use OCP\Files\InvalidPathException; +use OCP\Files\Mount\IMountManager; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\ReservedWordException; use OCP\IUser; use OCP\IUserManager; +use OCP\L10N\IFactory; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; use OCP\Server; @@ -605,13 +609,13 @@ class View { $this->lockFile($path, ILockingProvider::LOCK_SHARED); $exists = $this->file_exists($path); - $run = true; if ($this->shouldEmitHooks($path)) { + $run = true; $this->emit_file_hooks_pre($exists, $path, $run); - } - if (!$run) { - $this->unlockFile($path, ILockingProvider::LOCK_SHARED); - return false; + if (!$run) { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + return false; + } } try { @@ -626,7 +630,7 @@ class View { [$storage, $internalPath] = $this->resolvePath($path); $target = $storage->fopen($internalPath, 'w'); if ($target) { - [, $result] = \OC_Helper::streamCopy($data, $target); + [, $result] = Files::streamCopy($data, $target, true); fclose($target); fclose($data); @@ -695,11 +699,14 @@ class View { * * @param string $source source path * @param string $target target path + * @param array $options * * @return bool|mixed * @throws LockedException */ - public function rename($source, $target) { + public function rename($source, $target, array $options = []) { + $checkSubMounts = $options['checkSubMounts'] ?? true; + $absolutePath1 = Filesystem::normalizePath($this->getAbsolutePath($source)); $absolutePath2 = Filesystem::normalizePath($this->getAbsolutePath($target)); @@ -707,6 +714,9 @@ class View { throw new ForbiddenException('Moving a folder into a child folder is forbidden', false); } + /** @var IMountManager $mountManager */ + $mountManager = \OC::$server->get(IMountManager::class); + $targetParts = explode('/', $absolutePath2); $targetUser = $targetParts[1] ?? null; $result = false; @@ -764,24 +774,28 @@ class View { try { $this->changeLock($target, ILockingProvider::LOCK_EXCLUSIVE, true); + if ($checkSubMounts) { + $movedMounts = $mountManager->findIn($this->getAbsolutePath($source)); + } else { + $movedMounts = []; + } + if ($internalPath1 === '') { - if ($mount1 instanceof MoveableMount) { - $sourceParentMount = $this->getMount(dirname($source)); - if ($sourceParentMount === $mount2 && $this->targetIsNotShared($targetUser, $absolutePath2)) { - /** - * @var \OC\Files\Mount\MountPoint | \OC\Files\Mount\MoveableMount $mount1 - */ - $sourceMountPoint = $mount1->getMountPoint(); - $result = $mount1->moveMount($absolutePath2); - $manager->moveMount($sourceMountPoint, $mount1->getMountPoint()); - } else { - $result = false; - } - } else { - $result = false; - } + $sourceParentMount = $this->getMount(dirname($source)); + $movedMounts[] = $mount1; + $this->validateMountMove($movedMounts, $sourceParentMount, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + /** + * @var \OC\Files\Mount\MountPoint | \OC\Files\Mount\MoveableMount $mount1 + */ + $sourceMountPoint = $mount1->getMountPoint(); + $result = $mount1->moveMount($absolutePath2); + $manager->moveMount($sourceMountPoint, $mount1->getMountPoint()); + // moving a file/folder within the same mount point } elseif ($storage1 === $storage2) { + if (count($movedMounts) > 0) { + $this->validateMountMove($movedMounts, $mount1, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + } if ($storage1) { $result = $storage1->rename($internalPath1, $internalPath2); } else { @@ -789,6 +803,9 @@ class View { } // moving a file/folder between storages (from $storage1 to $storage2) } else { + if (count($movedMounts) > 0) { + $this->validateMountMove($movedMounts, $mount1, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + } $result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2); } @@ -839,6 +856,56 @@ class View { } /** + * @throws ForbiddenException + */ + private function validateMountMove(array $mounts, IMountPoint $sourceMount, IMountPoint $targetMount, bool $targetIsShared): void { + $targetPath = $this->getRelativePath($targetMount->getMountPoint()); + if ($targetPath) { + $targetPath = trim($targetPath, '/'); + } else { + $targetPath = $targetMount->getMountPoint(); + } + + $l = \OC::$server->get(IFactory::class)->get('files'); + foreach ($mounts as $mount) { + $sourcePath = $this->getRelativePath($mount->getMountPoint()); + if ($sourcePath) { + $sourcePath = trim($sourcePath, '/'); + } else { + $sourcePath = $mount->getMountPoint(); + } + + if (!$mount instanceof MoveableMount) { + throw new ForbiddenException($l->t('Storage %s cannot be moved', [$sourcePath]), false); + } + + if ($targetIsShared) { + if ($sourceMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a share (%s) into a shared folder is not allowed', [$sourcePath]), false); + } else { + throw new ForbiddenException($l->t('Moving a storage (%s) into a shared folder is not allowed', [$sourcePath]), false); + } + } + + if ($sourceMount !== $targetMount) { + if ($sourceMount instanceof SharedMount) { + if ($targetMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a share (%s) into another share (%s) is not allowed', [$sourcePath, $targetPath]), false); + } else { + throw new ForbiddenException($l->t('Moving a share (%s) into another storage (%s) is not allowed', [$sourcePath, $targetPath]), false); + } + } else { + if ($targetMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a storage (%s) into a share (%s) is not allowed', [$sourcePath, $targetPath]), false); + } else { + throw new ForbiddenException($l->t('Moving a storage (%s) into another storage (%s) is not allowed', [$sourcePath, $targetPath]), false); + } + } + } + } + } + + /** * Copy a file/folder from the source path to target path * * @param string $source source path @@ -871,7 +938,7 @@ class View { try { $exists = $this->file_exists($target); - if ($this->shouldEmitHooks()) { + if ($this->shouldEmitHooks($target)) { \OC_Hook::emit( Filesystem::CLASSNAME, Filesystem::signal_copy, @@ -911,7 +978,7 @@ class View { $this->changeLock($target, ILockingProvider::LOCK_SHARED); $lockTypePath2 = ILockingProvider::LOCK_SHARED; - if ($this->shouldEmitHooks() && $result !== false) { + if ($this->shouldEmitHooks($target) && $result !== false) { \OC_Hook::emit( Filesystem::CLASSNAME, Filesystem::signal_post_copy, @@ -1401,8 +1468,7 @@ class View { public function addSubMounts(FileInfo $info, $extOnly = false): void { $mounts = Filesystem::getMountManager()->findIn($info->getPath()); $info->setSubMounts(array_filter($mounts, function (IMountPoint $mount) use ($extOnly) { - $subStorage = $mount->getStorage(); - return !($extOnly && $subStorage instanceof \OCA\Files_Sharing\SharedStorage); + return !($extOnly && $mount instanceof SharedMount); })); } @@ -1514,12 +1580,22 @@ class View { // Create parent folders if the mountpoint is inside a subfolder that doesn't exist yet if (!isset($files[$entryName])) { try { + [$storage, ] = $this->resolvePath($path . '/' . $entryName); + // make sure we can create the mountpoint folder, even if the user has a quota of 0 + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(false); + } + if ($this->mkdir($path . '/' . $entryName) !== false) { $info = $this->getFileInfo($path . '/' . $entryName); if ($info !== false) { $files[$entryName] = $info; } } + + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(true); + } } catch (\Exception $e) { // Creating the parent folder might not be possible, for example due to a lack of permissions. $this->logger->debug('Failed to create non-existent parent', ['exception' => $e, 'path' => $path . '/' . $entryName]); @@ -1995,9 +2071,9 @@ class View { ); } } catch (LockedException $e) { - // rethrow with the a human-readable path + // rethrow with the human-readable path throw new LockedException( - $this->getPathRelativeToFiles($absolutePath), + $path, $e, $e->getExistingLock() ); @@ -2035,20 +2111,12 @@ class View { ); } } catch (LockedException $e) { - try { - // rethrow with the a human-readable path - throw new LockedException( - $this->getPathRelativeToFiles($absolutePath), - $e, - $e->getExistingLock() - ); - } catch (\InvalidArgumentException $ex) { - throw new LockedException( - $absolutePath, - $ex, - $e->getExistingLock() - ); - } + // rethrow with the a human-readable path + throw new LockedException( + $path, + $e, + $e->getExistingLock() + ); } return true; |