diff options
Diffstat (limited to 'lib/private/Files')
126 files changed, 26751 insertions, 0 deletions
diff --git a/lib/private/Files/AppData/AppData.php b/lib/private/Files/AppData/AppData.php new file mode 100644 index 00000000000..c13372ae1d9 --- /dev/null +++ b/lib/private/Files/AppData/AppData.php @@ -0,0 +1,154 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\AppData; + +use OC\Files\SimpleFS\SimpleFolder; +use OC\SystemConfig; +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Folder; +use OCP\Files\IAppData; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Files\SimpleFS\ISimpleFolder; + +class AppData implements IAppData { + private IRootFolder $rootFolder; + private SystemConfig $config; + private string $appId; + private ?Folder $folder = null; + /** @var CappedMemoryCache<ISimpleFolder|NotFoundException> */ + private CappedMemoryCache $folders; + + /** + * AppData constructor. + * + * @param IRootFolder $rootFolder + * @param SystemConfig $systemConfig + * @param string $appId + */ + public function __construct(IRootFolder $rootFolder, + SystemConfig $systemConfig, + string $appId) { + $this->rootFolder = $rootFolder; + $this->config = $systemConfig; + $this->appId = $appId; + $this->folders = new CappedMemoryCache(); + } + + private function getAppDataFolderName() { + $instanceId = $this->config->getValue('instanceid', null); + if ($instanceId === null) { + throw new \RuntimeException('no instance id!'); + } + + return 'appdata_' . $instanceId; + } + + protected function getAppDataRootFolder(): Folder { + $name = $this->getAppDataFolderName(); + + try { + /** @var Folder $node */ + $node = $this->rootFolder->get($name); + return $node; + } catch (NotFoundException $e) { + try { + return $this->rootFolder->newFolder($name); + } catch (NotPermittedException $e) { + throw new \RuntimeException('Could not get appdata folder'); + } + } + } + + /** + * @return Folder + * @throws \RuntimeException + */ + private function getAppDataFolder(): Folder { + if ($this->folder === null) { + $name = $this->getAppDataFolderName(); + + try { + $this->folder = $this->rootFolder->get($name . '/' . $this->appId); + } catch (NotFoundException $e) { + $appDataRootFolder = $this->getAppDataRootFolder(); + + try { + $this->folder = $appDataRootFolder->get($this->appId); + } catch (NotFoundException $e) { + try { + $this->folder = $appDataRootFolder->newFolder($this->appId); + } catch (NotPermittedException $e) { + throw new \RuntimeException('Could not get appdata folder for ' . $this->appId); + } + } + } + } + + return $this->folder; + } + + public function getFolder(string $name): ISimpleFolder { + $key = $this->appId . '/' . $name; + if ($cachedFolder = $this->folders->get($key)) { + if ($cachedFolder instanceof \Exception) { + throw $cachedFolder; + } else { + return $cachedFolder; + } + } + try { + // Hardening if somebody wants to retrieve '/' + if ($name === '/') { + $node = $this->getAppDataFolder(); + } else { + $path = $this->getAppDataFolderName() . '/' . $this->appId . '/' . $name; + $node = $this->rootFolder->get($path); + } + } catch (NotFoundException $e) { + $this->folders->set($key, $e); + throw $e; + } + + /** @var Folder $node */ + $folder = new SimpleFolder($node); + $this->folders->set($key, $folder); + return $folder; + } + + public function newFolder(string $name): ISimpleFolder { + $key = $this->appId . '/' . $name; + $folder = $this->getAppDataFolder()->newFolder($name); + + $simpleFolder = new SimpleFolder($folder); + $this->folders->set($key, $simpleFolder); + return $simpleFolder; + } + + public function getDirectoryListing(): array { + $listing = $this->getAppDataFolder()->getDirectoryListing(); + + $fileListing = array_map(function (Node $folder) { + if ($folder instanceof Folder) { + return new SimpleFolder($folder); + } + return null; + }, $listing); + + $fileListing = array_filter($fileListing); + + return array_values($fileListing); + } + + public function getId(): int { + return $this->getAppDataFolder()->getId(); + } +} diff --git a/lib/private/Files/AppData/Factory.php b/lib/private/Files/AppData/Factory.php new file mode 100644 index 00000000000..38b73f370b8 --- /dev/null +++ b/lib/private/Files/AppData/Factory.php @@ -0,0 +1,35 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\AppData; + +use OC\SystemConfig; +use OCP\Files\AppData\IAppDataFactory; +use OCP\Files\IAppData; +use OCP\Files\IRootFolder; + +class Factory implements IAppDataFactory { + private IRootFolder $rootFolder; + private SystemConfig $config; + + /** @var array<string, IAppData> */ + private array $folders = []; + + public function __construct(IRootFolder $rootFolder, + SystemConfig $systemConfig) { + $this->rootFolder = $rootFolder; + $this->config = $systemConfig; + } + + public function get(string $appId): IAppData { + if (!isset($this->folders[$appId])) { + $this->folders[$appId] = new AppData($this->rootFolder, $this->config, $appId); + } + return $this->folders[$appId]; + } +} diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php new file mode 100644 index 00000000000..329466e682d --- /dev/null +++ b/lib/private/Files/Cache/Cache.php @@ -0,0 +1,1284 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception\UniqueConstraintViolationException; +use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery; +use OC\Files\Storage\Wrapper\Encryption; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\CacheEntryInsertedEvent; +use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntryUpdatedEvent; +use OCP\Files\Cache\CacheInsertEvent; +use OCP\Files\Cache\CacheUpdateEvent; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\FileInfo; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use OCP\Util; +use Psr\Log\LoggerInterface; + +/** + * Metadata cache for a storage + * + * The cache stores the metadata for all files and folders in a storage and is kept up to date through the following mechanisms: + * + * - Scanner: scans the storage and updates the cache where needed + * - Watcher: checks for changes made to the filesystem outside of the Nextcloud instance and rescans files and folder when a change is detected + * - Updater: listens to changes made to the filesystem inside of the Nextcloud instance and updates the cache where needed + * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater + */ +class Cache implements ICache { + use MoveFromCacheTrait { + MoveFromCacheTrait::moveFromCache as moveFromCacheFallback; + } + + /** + * @var array partial data for the cache + */ + protected array $partial = []; + protected string $storageId; + protected Storage $storageCache; + protected IMimeTypeLoader $mimetypeLoader; + protected IDBConnection $connection; + protected SystemConfig $systemConfig; + protected LoggerInterface $logger; + protected QuerySearchHelper $querySearchHelper; + protected IEventDispatcher $eventDispatcher; + protected IFilesMetadataManager $metadataManager; + + public function __construct( + private IStorage $storage, + // this constructor is used in to many pleases to easily do proper di + // so instead we group it all together + ?CacheDependencies $dependencies = null, + ) { + $this->storageId = $storage->getId(); + if (strlen($this->storageId) > 64) { + $this->storageId = md5($this->storageId); + } + if (!$dependencies) { + $dependencies = \OCP\Server::get(CacheDependencies::class); + } + $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->systemConfig = $dependencies->getSystemConfig(); + $this->logger = $dependencies->getLogger(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + $this->eventDispatcher = $dependencies->getEventDispatcher(); + $this->metadataManager = $dependencies->getMetadataManager(); + } + + protected function getQueryBuilder() { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getStorageCache(): Storage { + return $this->storageCache; + } + + /** + * Get the numeric storage id for this cache's storage + * + * @return int + */ + public function getNumericStorageId() { + return $this->storageCache->getNumericId(); + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file either the path of a file or folder or the file id for a file or folder + * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache + */ + public function get($file) { + $query = $this->getQueryBuilder(); + $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); + + if (is_string($file) || $file == '') { + // normalize file + $file = $this->normalize($file); + + $query->wherePath($file); + } else { //file id + $query->whereFileId($file); + } + $query->whereStorageId($this->getNumericStorageId()); + + $result = $query->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + + if ($data !== false) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } + } + + return false; + } + + /** + * Create a CacheEntry from database row + * + * @param array $data + * @param IMimeTypeLoader $mimetypeLoader + * @return CacheEntry + */ + public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) { + //fix types + $data['name'] = (string)$data['name']; + $data['path'] = (string)$data['path']; + $data['fileid'] = (int)$data['fileid']; + $data['parent'] = (int)$data['parent']; + $data['size'] = Util::numericToNumber($data['size']); + $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0); + $data['mtime'] = (int)$data['mtime']; + $data['storage_mtime'] = (int)$data['storage_mtime']; + $data['encryptedVersion'] = (int)$data['encrypted']; + $data['encrypted'] = (bool)$data['encrypted']; + $data['storage_id'] = $data['storage']; + $data['storage'] = (int)$data['storage']; + $data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']); + $data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']); + if ($data['storage_mtime'] == 0) { + $data['storage_mtime'] = $data['mtime']; + } + if (isset($data['f_permissions'])) { + $data['scan_permissions'] = $data['f_permissions']; + } + $data['permissions'] = (int)$data['permissions']; + if (isset($data['creation_time'])) { + $data['creation_time'] = (int)$data['creation_time']; + } + if (isset($data['upload_time'])) { + $data['upload_time'] = (int)$data['upload_time']; + } + return new CacheEntry($data); + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return ICacheEntry[] + */ + public function getFolderContentsById($fileId) { + if ($fileId > -1) { + $query = $this->getQueryBuilder(); + $query->selectFileCache() + ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) + ->orderBy('name', 'ASC'); + + $metadataQuery = $query->selectMetadata(); + + $result = $query->executeQuery(); + $files = $result->fetchAll(); + $result->closeCursor(); + + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return self::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); + } + return []; + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + // normalize file + $file = $this->normalize($file); + + if (isset($this->partial[$file])) { //add any saved partial data + $data = array_merge($this->partial[$file]->getData(), $data); + unset($this->partial[$file]); + } + + $requiredFields = ['size', 'mtime', 'mimetype']; + foreach ($requiredFields as $field) { + if (!isset($data[$field])) { //data not complete save as partial and return + $this->partial[$file] = new CacheEntry($data); + return -1; + } + } + + $data['path'] = $file; + if (!isset($data['parent'])) { + $data['parent'] = $this->getParentId($file); + } + if ($data['parent'] === -1 && $file !== '') { + throw new \Exception('Parent folder not in filecache for ' . $file); + } + $data['name'] = basename($file); + + [$values, $extensionValues] = $this->normalizeData($data); + $storageId = $this->getNumericStorageId(); + $values['storage'] = $storageId; + + try { + $builder = $this->connection->getQueryBuilder(); + $builder->insert('filecache'); + + foreach ($values as $column => $value) { + $builder->setValue($column, $builder->createNamedParameter($value)); + } + + if ($builder->execute()) { + $fileId = $builder->getLastInsertId(); + + if (count($extensionValues)) { + $query = $this->getQueryBuilder(); + $query->insert('filecache_extended'); + $query->hintShardKey('storage', $storageId); + + $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + foreach ($extensionValues as $column => $value) { + $query->setValue($column, $query->createNamedParameter($value)); + } + $query->executeStatement(); + } + + $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId); + $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + return $fileId; + } + } catch (UniqueConstraintViolationException $e) { + // entry exists already + if ($this->connection->inTransaction()) { + $this->connection->commit(); + $this->connection->beginTransaction(); + } + } + + // The file was created in the mean time + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + throw new \RuntimeException('File entry could not be inserted but could also not be selected with getId() in order to perform an update. Please try again.'); + } + } + + /** + * update the metadata of an existing file or folder in the cache + * + * @param int $id the fileid of the existing file or folder + * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged + */ + public function update($id, array $data) { + if (isset($data['path'])) { + // normalize path + $data['path'] = $this->normalize($data['path']); + } + + if (isset($data['name'])) { + // normalize path + $data['name'] = $this->normalize($data['name']); + } + + [$values, $extensionValues] = $this->normalizeData($data); + + if (count($values)) { + $query = $this->getQueryBuilder(); + + $query->update('filecache') + ->whereFileId($id) + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { + return $query->expr()->orX( + $query->expr()->neq($key, $query->createNamedParameter($value)), + $query->expr()->isNull($key) + ); + }, array_keys($values), array_values($values)))); + + foreach ($values as $key => $value) { + $query->set($key, $query->createNamedParameter($value)); + } + + $query->executeStatement(); + } + + if (count($extensionValues)) { + try { + $query = $this->getQueryBuilder(); + $query->insert('filecache_extended'); + $query->hintShardKey('storage', $this->getNumericStorageId()); + + $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); + foreach ($extensionValues as $column => $value) { + $query->setValue($column, $query->createNamedParameter($value)); + } + + $query->execute(); + } catch (UniqueConstraintViolationException $e) { + $query = $this->getQueryBuilder(); + $query->update('filecache_extended') + ->whereFileId($id) + ->hintShardKey('storage', $this->getNumericStorageId()) + ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { + return $query->expr()->orX( + $query->expr()->neq($key, $query->createNamedParameter($value)), + $query->expr()->isNull($key) + ); + }, array_keys($extensionValues), array_values($extensionValues)))); + + foreach ($extensionValues as $key => $value) { + $query->set($key, $query->createNamedParameter($value)); + } + + $query->executeStatement(); + } + } + + $path = $this->getPathById($id); + // path can still be null if the file doesn't exist + if ($path !== null) { + $event = new CacheEntryUpdatedEvent($this->storage, $path, $id, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } + } + + /** + * extract query parts and params array from data array + * + * @param array $data + * @return array + */ + protected function normalizeData(array $data): array { + $fields = [ + 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', + 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; + $extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; + + $doNotCopyStorageMTime = false; + if (array_key_exists('mtime', $data) && $data['mtime'] === null) { + // this horrific magic tells it to not copy storage_mtime to mtime + unset($data['mtime']); + $doNotCopyStorageMTime = true; + } + + $params = []; + $extensionParams = []; + foreach ($data as $name => $value) { + if (in_array($name, $fields)) { + if ($name === 'path') { + $params['path_hash'] = md5($value); + } elseif ($name === 'mimetype') { + $params['mimepart'] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/'))); + $value = $this->mimetypeLoader->getId($value); + } elseif ($name === 'storage_mtime') { + if (!$doNotCopyStorageMTime && !isset($data['mtime'])) { + $params['mtime'] = $value; + } + } elseif ($name === 'encrypted') { + if (isset($data['encryptedVersion'])) { + $value = $data['encryptedVersion']; + } else { + // Boolean to integer conversion + $value = $value ? 1 : 0; + } + } + $params[$name] = $value; + } + if (in_array($name, $extensionFields)) { + $extensionParams[$name] = $value; + } + } + return [$params, array_filter($extensionParams)]; + } + + /** + * get the file id for a file + * + * A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file + * + * File ids are easiest way for apps to store references to a file since unlike paths they are not affected by renames or sharing + * + * @param string $file + * @return int + */ + public function getId($file) { + // normalize file + $file = $this->normalize($file); + + $query = $this->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->wherePath($file); + + $result = $query->executeQuery(); + $id = $result->fetchOne(); + $result->closeCursor(); + + return $id === false ? -1 : (int)$id; + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + if ($file === '') { + return -1; + } else { + $parent = $this->getParentPath($file); + return (int)$this->getId($parent); + } + } + + private function getParentPath($path) { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + return $parent; + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getId($file) != -1; + } + + /** + * remove a file or folder from the cache + * + * when removing a folder from the cache all files and folders inside the folder will be removed as well + * + * @param string $file + */ + public function remove($file) { + $entry = $this->get($file); + + if ($entry instanceof ICacheEntry) { + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereFileId($entry->getId()); + $query->executeStatement(); + + $query = $this->getQueryBuilder(); + $query->delete('filecache_extended') + ->whereFileId($entry->getId()) + ->hintShardKey('storage', $this->getNumericStorageId()); + $query->executeStatement(); + + if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $this->removeChildren($entry); + } + + $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $entry->getPath(), $entry->getId(), $this->getNumericStorageId())); + } + } + + /** + * Remove all children of a folder + * + * @param ICacheEntry $entry the cache entry of the folder to remove the children of + * @throws \OC\DatabaseException + */ + private function removeChildren(ICacheEntry $entry) { + $parentIds = [$entry->getId()]; + $queue = [$entry->getId()]; + $deletedIds = []; + $deletedPaths = []; + + // we walk depth first through the file tree, removing all filecache_extended attributes while we walk + // and collecting all folder ids to later use to delete the filecache entries + while ($entryId = array_pop($queue)) { + $children = $this->getFolderContentsById($entryId); + $childIds = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getId(); + }, $children); + $childPaths = array_map(function (ICacheEntry $cacheEntry) { + return $cacheEntry->getPath(); + }, $children); + + foreach ($childIds as $childId) { + $deletedIds[] = $childId; + } + + foreach ($childPaths as $childPath) { + $deletedPaths[] = $childPath; + } + + $query = $this->getQueryBuilder(); + $query->delete('filecache_extended') + ->where($query->expr()->in('fileid', $query->createParameter('childIds'))) + ->hintShardKey('storage', $this->getNumericStorageId()); + + foreach (array_chunk($childIds, 1000) as $childIdChunk) { + $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + + /** @var ICacheEntry[] $childFolders */ + $childFolders = []; + foreach ($children as $child) { + if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) { + $childFolders[] = $child; + } + } + foreach ($childFolders as $folder) { + $parentIds[] = $folder->getId(); + $queue[] = $folder->getId(); + } + } + + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereParentInParameter('parentIds'); + + // Sorting before chunking allows the db to find the entries close to each + // other in the index + sort($parentIds, SORT_NUMERIC); + foreach (array_chunk($parentIds, 1000) as $parentIdChunk) { + $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + + foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { + $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( + $this->storage, + $filePath, + $fileId, + $this->getNumericStorageId() + ); + $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); + } + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->moveFromCache($this, $source, $target); + } + + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $path]; + } + + protected function hasEncryptionWrapper(): bool { + return $this->storage->instanceOfStorage(Encryption::class); + } + + /** + * Move a file or folder in the cache + * + * @param ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + * @throws \OC\DatabaseException + * @throws \Exception if the given storages have an invalid id + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache instanceof Cache) { + // normalize source and target + $sourcePath = $this->normalize($sourcePath); + $targetPath = $this->normalize($targetPath); + + $sourceData = $sourceCache->get($sourcePath); + if (!$sourceData) { + throw new \Exception('Source path not found in cache: ' . $sourcePath); + } + + $shardDefinition = $this->connection->getShardDefinition('filecache'); + if ( + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + ) { + $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); + return; + } + + $sourceId = $sourceData['fileid']; + $newParentId = $this->getParentId($targetPath); + + [$sourceStorageId, $sourcePath] = $sourceCache->getMoveInfo($sourcePath); + [$targetStorageId, $targetPath] = $this->getMoveInfo($targetPath); + + if (is_null($sourceStorageId) || $sourceStorageId === false) { + throw new \Exception('Invalid source storage id: ' . $sourceStorageId); + } + if (is_null($targetStorageId) || $targetStorageId === false) { + throw new \Exception('Invalid target storage id: ' . $targetStorageId); + } + + if ($sourceData['mimetype'] === 'httpd/unix-directory') { + //update all child entries + $sourceLength = mb_strlen($sourcePath); + + $childIds = $this->getChildIds($sourceStorageId, $sourcePath); + + $childChunks = array_chunk($childIds, 1000); + + $query = $this->getQueryBuilder(); + + $fun = $query->func(); + $newPathFunction = $fun->concat( + $query->createNamedParameter($targetPath), + $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash + ); + $query->update('filecache') + ->set('path_hash', $fun->md5($newPathFunction)) + ->set('path', $newPathFunction) + ->whereStorageId($sourceStorageId) + ->andWhere($query->expr()->in('fileid', $query->createParameter('files'))); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + // Retry transaction in case of RetryableException like deadlocks. + // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend + $retryLimit = 4; + for ($i = 1; $i <= $retryLimit; $i++) { + try { + $this->connection->beginTransaction(); + foreach ($childChunks as $chunk) { + $query->setParameter('files', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + break; + } catch (\OC\DatabaseException $e) { + $this->connection->rollBack(); + throw $e; + } catch (DbalException $e) { + $this->connection->rollBack(); + + if (!$e->isRetryable()) { + throw $e; + } + + // Simply throw if we already retried 4 times. + if ($i === $retryLimit) { + throw $e; + } + + // Sleep a bit to give some time to the other transaction to finish. + usleep(100 * 1000 * $i); + } + } + } else { + $this->connection->beginTransaction(); + } + + $query = $this->getQueryBuilder(); + $query->update('filecache') + ->set('path', $query->createNamedParameter($targetPath)) + ->set('path_hash', $query->createNamedParameter(md5($targetPath))) + ->set('name', $query->createNamedParameter(basename($targetPath))) + ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT)) + ->whereStorageId($sourceStorageId) + ->whereFileId($sourceId); + + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); + } + + $query->executeStatement(); + + $this->connection->commit(); + + if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) { + $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId())); + $event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } else { + $event = new CacheEntryUpdatedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId()); + $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event); + $this->eventDispatcher->dispatchTyped($event); + } + } else { + $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath); + } + } + + private function getChildIds(int $storageId, string $path): array { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($path) . '/%'))); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $query = $this->getQueryBuilder(); + $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()); + $query->executeStatement(); + + $query = $this->connection->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); + $query->executeStatement(); + } + + /** + * Get the scan status of a file + * + * - Cache::NOT_FOUND: File is not in the cache + * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known + * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned + * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned + * + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + // normalize file + $file = $this->normalize($file); + + $query = $this->getQueryBuilder(); + $query->select('size') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->wherePath($file); + + $result = $query->executeQuery(); + $size = $result->fetchOne(); + $result->closeCursor(); + + if ($size !== false) { + if ((int)$size === -1) { + return self::SHALLOW; + } else { + return self::COMPLETE; + } + } else { + if (isset($this->partial[$file])) { + return self::PARTIAL; + } else { + return self::NOT_FOUND; + } + } + } + + /** + * search for files matching $pattern + * + * @param string $pattern the search pattern using SQL search syntax (e.g. '%searchstring%') + * @return ICacheEntry[] an array of cache entries where the name matches the search pattern + */ + public function search($pattern) { + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', $pattern); + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); + } + + /** + * search for files by mimetype + * + * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') + * where it will search for all mimetypes in the group ('image/*') + * @return ICacheEntry[] an array of cache entries where the mimetype matches the search + */ + public function searchByMime($mimetype) { + if (!str_contains($mimetype, '/')) { + $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'); + } else { + $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype); + } + return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); + } + + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); + } + + /** + * Re-calculate the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $this->calculateFolderSize($path, $data); + + if ($path !== '') { + $parent = dirname($path); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + + if ($isBackgroundScan) { + $parentData = $this->get($parent); + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { + $this->correctFolderSize($parent, $parentData, $isBackgroundScan); + } + } else { + $this->correctFolderSize($parent); + } + } + } + + /** + * get the incomplete count that shares parent $folder + * + * @param int $fileId the file id of the folder + * @return int + */ + public function getIncompleteChildrenCount($fileId) { + if ($fileId > -1) { + $query = $this->getQueryBuilder(); + $query->select($query->func()->count()) + ->from('filecache') + ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $size = (int)$result->fetchOne(); + $result->closeCursor(); + + return $size; + } + return -1; + } + + /** + * calculate the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + return $this->calculateFolderSizeInner($path, $entry); + } + + + /** + * inner function because we can't add new params to the public function without breaking any child classes + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown + * @return int|float + */ + protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) { + $totalSize = 0; + if (is_null($entry) || !isset($entry['fileid'])) { + $entry = $this->get($path); + } + if (isset($entry['mimetype']) && $entry['mimetype'] === FileInfo::MIMETYPE_FOLDER) { + $id = $entry['fileid']; + + $query = $this->getQueryBuilder(); + $query->select('size', 'unencrypted_size') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereParent($id); + if ($ignoreUnknown) { + $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); + } + + $result = $query->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + + if ($rows) { + $sizes = array_map(function (array $row) { + return Util::numericToNumber($row['size']); + }, $rows); + $unencryptedOnlySizes = array_map(function (array $row) { + return Util::numericToNumber($row['unencrypted_size']); + }, $rows); + $unencryptedSizes = array_map(function (array $row) { + return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']); + }, $rows); + + $sum = array_sum($sizes); + $min = min($sizes); + + $unencryptedSum = array_sum($unencryptedSizes); + $unencryptedMin = min($unencryptedSizes); + $unencryptedMax = max($unencryptedOnlySizes); + + $sum = 0 + $sum; + $min = 0 + $min; + if ($min === -1) { + $totalSize = $min; + } else { + $totalSize = $sum; + } + if ($unencryptedMin === -1 || $min === -1) { + $unencryptedTotal = $unencryptedMin; + } else { + $unencryptedTotal = $unencryptedSum; + } + } else { + $totalSize = 0; + $unencryptedTotal = 0; + $unencryptedMax = 0; + } + + // only set unencrypted size for a folder if any child entries have it set, or the folder is empty + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + if ($shouldWriteUnEncryptedSize) { + // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes + if ($unencryptedMax === 0) { + $unencryptedTotal = 0; + } + + $this->update($id, [ + 'size' => $totalSize, + 'unencrypted_size' => $unencryptedTotal, + ]); + } else { + $this->update($id, [ + 'size' => $totalSize, + ]); + } + } + } + return $totalSize; + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + $query = $this->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()); + + $result = $query->executeQuery(); + $files = $result->fetchAll(\PDO::FETCH_COLUMN); + $result->closeCursor(); + + return array_map(function ($id) { + return (int)$id; + }, $files); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + $query = $this->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->orderBy('fileid', 'DESC') + ->setMaxResults(1); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + return $path === false ? false : (string)$path; + } + + /** + * get the path of a file on this storage by it's file id + * + * @param int $id the file id of the file or folder to search + * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache + */ + public function getPathById($id) { + $query = $this->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) + ->whereFileId($id); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + if ($path === false) { + return null; + } + + return (string)$path; + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @return array first element holding the storage id, second the path + * @deprecated 17.0.0 use getPathById() instead + */ + public static function getById($id) { + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->select('path', 'storage') + ->from('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $numericId = $row['storage']; + $path = $row['path']; + } else { + return null; + } + + if ($id = Storage::getStorageId($numericId)) { + return [$id, $path]; + } else { + return null; + } + } + + /** + * normalize the given path + * + * @param string $path + * @return string + */ + public function normalize($path) { + return trim(\OC_Util::normalizeUnicode($path), '/'); + } + + /** + * Copy a file or folder in the cache + * + * @param ICache $sourceCache + * @param ICacheEntry $sourceEntry + * @param string $targetPath + * @return int fileId of copied entry + */ + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + if ($sourceEntry->getId() < 0) { + throw new \RuntimeException('Invalid source cache entry on copyFromCache'); + } + $data = $this->cacheEntryToArray($sourceEntry); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { + $data['encrypted'] = 0; + } + + $fileId = $this->put($targetPath, $data); + if ($fileId <= 0) { + throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' '); + } + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); + foreach ($folderContent as $subEntry) { + $subTargetPath = $targetPath . '/' . $subEntry->getName(); + $this->copyFromCache($sourceCache, $subEntry, $subTargetPath); + } + } + return $fileId; + } + + private function cacheEntryToArray(ICacheEntry $entry): array { + $data = [ + 'size' => $entry->getSize(), + 'mtime' => $entry->getMTime(), + 'storage_mtime' => $entry->getStorageMTime(), + 'mimetype' => $entry->getMimeType(), + 'mimepart' => $entry->getMimePart(), + 'etag' => $entry->getEtag(), + 'permissions' => $entry->getPermissions(), + 'encrypted' => $entry->isEncrypted(), + 'creation_time' => $entry->getCreationTime(), + 'upload_time' => $entry->getUploadTime(), + 'metadata_etag' => $entry->getMetadataEtag(), + ]; + if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) { + $data['permissions'] = $entry['scan_permissions']; + } + return $data; + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', $this->getNumericStorageId()); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($rawEntry->getStorageId() === $this->getNumericStorageId()) { + return $rawEntry; + } else { + return null; + } + } + + private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void { + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath()); + } else { + $fileIds = []; + } + $fileIds[] = $sourceEntry->getId(); + + $helper = $this->connection->getCrossShardMoveHelper(); + + $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId()); + $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId()); + + $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper(); + + $sourcePathLength = strlen($sourceEntry->getPath()); + foreach ($cacheItems as &$cacheItem) { + if ($cacheItem['path'] === $sourceEntry->getPath()) { + $cacheItem['path'] = $targetPath; + $cacheItem['parent'] = $this->getParentId($targetPath); + $cacheItem['name'] = basename($cacheItem['path']); + } else { + $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash + } + $cacheItem['path_hash'] = md5($cacheItem['path']); + $cacheItem['storage'] = $this->getNumericStorageId(); + if ($removeEncryptedFlag) { + $cacheItem['encrypted'] = 0; + } + } + + $targetConnection->beginTransaction(); + + try { + $helper->saveItems($targetConnection, 'filecache', $cacheItems); + $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems); + $helper->saveItems($targetConnection, 'files_metadata', $metadataItems); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + + $sourceConnection->beginTransaction(); + + try { + $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + } catch (\Exception $e) { + $targetConnection->rollback(); + $sourceConnection->rollBack(); + throw $e; + } + + try { + $sourceConnection->commit(); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + $targetConnection->commit(); + } +} diff --git a/lib/private/Files/Cache/CacheDependencies.php b/lib/private/Files/Cache/CacheDependencies.php new file mode 100644 index 00000000000..61d2a2f9646 --- /dev/null +++ b/lib/private/Files/Cache/CacheDependencies.php @@ -0,0 +1,61 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +class CacheDependencies { + public function __construct( + private IMimeTypeLoader $mimeTypeLoader, + private IDBConnection $connection, + private IEventDispatcher $eventDispatcher, + private QuerySearchHelper $querySearchHelper, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private IFilesMetadataManager $metadataManager, + private DisplayNameCache $displayNameCache, + ) { + } + + public function getMimeTypeLoader(): IMimeTypeLoader { + return $this->mimeTypeLoader; + } + + public function getConnection(): IDBConnection { + return $this->connection; + } + + public function getEventDispatcher(): IEventDispatcher { + return $this->eventDispatcher; + } + + public function getQuerySearchHelper(): QuerySearchHelper { + return $this->querySearchHelper; + } + + public function getSystemConfig(): SystemConfig { + return $this->systemConfig; + } + + public function getLogger(): LoggerInterface { + return $this->logger; + } + + public function getDisplayNameCache(): DisplayNameCache { + return $this->displayNameCache; + } + + public function getMetadataManager(): IFilesMetadataManager { + return $this->metadataManager; + } +} diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php new file mode 100644 index 00000000000..ab5bae316f4 --- /dev/null +++ b/lib/private/Files/Cache/CacheEntry.php @@ -0,0 +1,132 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +/** + * meta data for a file or folder + */ +class CacheEntry implements ICacheEntry { + /** + * @var array + */ + private $data; + + public function __construct(array $data) { + $this->data = $data; + } + + public function offsetSet($offset, $value): void { + $this->data[$offset] = $value; + } + + public function offsetExists($offset): bool { + return isset($this->data[$offset]); + } + + public function offsetUnset($offset): void { + unset($this->data[$offset]); + } + + /** + * @return mixed + */ + #[\ReturnTypeWillChange] + public function offsetGet($offset) { + if (isset($this->data[$offset])) { + return $this->data[$offset]; + } else { + return null; + } + } + + public function getId() { + return (int)$this->data['fileid']; + } + + public function getStorageId() { + return $this->data['storage']; + } + + + public function getPath() { + return (string)$this->data['path']; + } + + + public function getName() { + return $this->data['name']; + } + + + public function getMimeType() { + return $this->data['mimetype']; + } + + + public function getMimePart() { + return $this->data['mimepart']; + } + + public function getSize() { + return $this->data['size']; + } + + public function getMTime() { + return $this->data['mtime']; + } + + public function getStorageMTime() { + return $this->data['storage_mtime']; + } + + public function getEtag() { + return $this->data['etag']; + } + + public function getPermissions() { + return $this->data['permissions']; + } + + public function isEncrypted() { + return isset($this->data['encrypted']) && $this->data['encrypted']; + } + + public function getMetadataEtag(): ?string { + return $this->data['metadata_etag'] ?? null; + } + + public function getCreationTime(): ?int { + return $this->data['creation_time'] ?? null; + } + + public function getUploadTime(): ?int { + return $this->data['upload_time'] ?? null; + } + + public function getParentId(): int { + return $this->data['parent']; + } + + public function getData() { + return $this->data; + } + + public function __clone() { + $this->data = array_merge([], $this->data); + } + + public function getUnencryptedSize(): int { + if ($this->data['encrypted'] && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return $this->data['size'] ?? 0; + } + } +} diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php new file mode 100644 index 00000000000..5492452273b --- /dev/null +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -0,0 +1,125 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\DB\QueryBuilder\ExtendedQueryBuilder; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; + +/** + * Query builder with commonly used helpers for filecache queries + */ +class CacheQueryBuilder extends ExtendedQueryBuilder { + private ?string $alias = null; + + public function __construct( + IQueryBuilder $queryBuilder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($queryBuilder); + } + + public function selectTagUsage(): self { + $this + ->select('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable', 'systemtag.etag', 'systemtag.color') + ->selectAlias($this->createFunction('COUNT(filecache.fileid)'), 'number_files') + ->selectAlias($this->createFunction('MAX(filecache.fileid)'), 'ref_file_id') + ->from('filecache', 'filecache') + ->leftJoin('filecache', 'systemtag_object_mapping', 'systemtagmap', $this->expr()->andX( + $this->expr()->eq('filecache.fileid', $this->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $this->expr()->eq('systemtagmap.objecttype', $this->createNamedParameter('files')) + )) + ->leftJoin('systemtagmap', 'systemtag', 'systemtag', $this->expr()->andX( + $this->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'), + $this->expr()->eq('systemtag.visibility', $this->createNamedParameter(true)) + )) + ->groupBy('systemtag.name', 'systemtag.id', 'systemtag.visibility', 'systemtag.editable'); + + return $this; + } + + public function selectFileCache(?string $alias = null, bool $joinExtendedCache = true) { + $name = $alias ?: 'filecache'; + $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', + 'storage_mtime', 'encrypted', "$name.etag", "$name.permissions", 'checksum', 'unencrypted_size') + ->from('filecache', $name); + + if ($joinExtendedCache) { + $this->addSelect('metadata_etag', 'creation_time', 'upload_time'); + $this->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); + } + + $this->alias = $name; + + return $this; + } + + public function whereStorageId(int $storageId) { + $this->andWhere($this->expr()->eq('storage', $this->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function whereFileId(int $fileId) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->eq("{$alias}fileid", $this->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function wherePath(string $path) { + $this->andWhere($this->expr()->eq('path_hash', $this->createNamedParameter(md5($path)))); + + return $this; + } + + public function whereParent(int $parent) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->eq("{$alias}parent", $this->createNamedParameter($parent, IQueryBuilder::PARAM_INT))); + + return $this; + } + + public function whereParentInParameter(string $parameter) { + $alias = $this->alias; + if ($alias) { + $alias .= '.'; + } else { + $alias = ''; + } + + $this->andWhere($this->expr()->in("{$alias}parent", $this->createParameter($parameter))); + + return $this; + } + + /** + * join metadata to current query builder and returns an helper + * + * @return IMetadataQuery + */ + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } +} diff --git a/lib/private/Files/Cache/FailedCache.php b/lib/private/Files/Cache/FailedCache.php new file mode 100644 index 00000000000..44c1016ca8e --- /dev/null +++ b/lib/private/Files/Cache/FailedCache.php @@ -0,0 +1,138 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OC\Files\Search\SearchComparison; +use OCP\Constants; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; + +/** + * Storage placeholder to represent a missing precondition, storage unavailable + */ +class FailedCache implements ICache { + /** @var bool whether to show the failed storage in the ui */ + private $visible; + + /** + * FailedCache constructor. + * + * @param bool $visible + */ + public function __construct($visible = true) { + $this->visible = $visible; + } + + + public function getNumericStorageId() { + return -1; + } + + public function get($file) { + if ($file === '') { + return new CacheEntry([ + 'fileid' => -1, + 'size' => 0, + 'mimetype' => 'httpd/unix-directory', + 'mimepart' => 'httpd', + 'permissions' => $this->visible ? Constants::PERMISSION_READ : 0, + 'mtime' => time() + ]); + } else { + return false; + } + } + + public function getFolderContents($folder) { + return []; + } + + public function getFolderContentsById($fileId) { + return []; + } + + public function put($file, array $data) { + } + + public function insert($file, array $data) { + } + + public function update($id, array $data) { + } + + public function getId($file) { + return -1; + } + + public function getParentId($file) { + return -1; + } + + public function inCache($file) { + return false; + } + + public function remove($file) { + } + + public function move($source, $target) { + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + } + + public function clear() { + } + + public function getStatus($file) { + return ICache::NOT_FOUND; + } + + public function search($pattern) { + return []; + } + + public function searchByMime($mimetype) { + return []; + } + + public function searchQuery(ISearchQuery $query) { + return []; + } + + public function getAll() { + return []; + } + + public function getIncomplete() { + return []; + } + + public function getPathById($id) { + return null; + } + + public function normalize($path) { + return $path; + } + + public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { + throw new \Exception('Invalid cache'); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', -1); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + return null; + } +} diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php new file mode 100644 index 00000000000..c3f3614f3ca --- /dev/null +++ b/lib/private/Files/Cache/FileAccess.php @@ -0,0 +1,222 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\SystemConfig; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IFileAccess; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Low level access to the file cache + */ +class FileAccess implements IFileAccess { + public function __construct( + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private FilesMetadataManager $metadataManager, + private IMimeTypeLoader $mimeTypeLoader, + ) { + } + + private function getQuery(): CacheQueryBuilder { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->metadataManager, + ); + } + + public function getByFileIdInStorage(int $fileId, int $storageId): ?CacheEntry { + $items = array_values($this->getByFileIdsInStorage([$fileId], $storageId)); + return $items[0] ?? null; + } + + public function getByPathInStorage(string $path, int $storageId): ?CacheEntry { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->eq('filecache.path_hash', $query->createNamedParameter(md5($path)))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $row = $query->executeQuery()->fetch(); + return $row ? Cache::cacheEntryFromData($row, $this->mimeTypeLoader) : null; + } + + public function getByFileId(int $fileId): ?CacheEntry { + $items = array_values($this->getByFileIds([$fileId])); + return $items[0] ?? null; + } + + /** + * @param array[] $rows + * @return array<int, CacheEntry> + */ + private function rowsToEntries(array $rows): array { + $result = []; + foreach ($rows as $row) { + $entry = Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + $result[$entry->getId()] = $entry; + } + return $result; + } + + /** + * @param int[] $fileIds + * @return array<int, CacheEntry> + */ + public function getByFileIds(array $fileIds): array { + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + /** + * @param int[] $fileIds + * @param int $storageId + * @return array<int, CacheEntry> + */ + public function getByFileIdsInStorage(array $fileIds, int $storageId): array { + $fileIds = array_values($fileIds); + $query = $this->getQuery()->selectFileCache(); + $query->andWhere($query->expr()->in('filecache.fileid', $query->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->andWhere($query->expr()->eq('filecache.storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + + $rows = $query->executeQuery()->fetchAll(); + return $this->rowsToEntries($rows); + } + + public function getByAncestorInStorage(int $storageId, int $folderId, int $fileIdCursor = 0, int $maxResults = 100, array $mimeTypeIds = [], bool $endToEndEncrypted = true, bool $serverSideEncrypted = true): \Generator { + $qb = $this->getQuery(); + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($folderId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var array{path:string}|false $root */ + $root = $result->fetch(); + $result->closeCursor(); + + if ($root === false) { + throw new Exception('Could not fetch storage root'); + } + + $qb = $this->getQuery(); + + $path = $root['path'] === '' ? '' : $root['path'] . '/'; + + $qb->selectDistinct('*') + ->from('filecache', 'f') + ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) + ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + + if (!$endToEndEncrypted) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // Use a subquery to check the `encrypted` status of the parent folder + $subQuery = $this->getQuery()->select('p.encrypted') + ->from('filecache', 'p') + ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) + ->getSQL(); + + $qb->andWhere( + $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + ); + } + + if (!$serverSideEncrypted) { + // Server side encrypted files have encrypted=1 directly + $qb->andWhere($qb->expr()->eq('f.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + } + + if (count($mimeTypeIds) > 0) { + $qb->andWhere($qb->expr()->in('f.mimetype', $qb->createNamedParameter($mimeTypeIds, IQueryBuilder::PARAM_INT_ARRAY))); + } + + if ($maxResults !== 0) { + $qb->setMaxResults($maxResults); + } + $qb->orderBy('f.fileid', 'ASC'); + $files = $qb->executeQuery(); + + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); + } + + $files->closeCursor(); + } + + public function getDistinctMounts(array $mountProviders = [], bool $onlyUserFilesMounts = true): \Generator { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct(['root_id', 'storage_id', 'mount_provider_class']) + ->from('mounts'); + if ($onlyUserFilesMounts) { + $qb->andWhere( + $qb->expr()->orX( + $qb->expr()->like('mount_point', $qb->createNamedParameter('/%/files/%')), + $qb->expr()->in('mount_provider_class', $qb->createNamedParameter([ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], IQueryBuilder::PARAM_STR_ARRAY)) + ) + ); + } + if (count($mountProviders) > 0) { + $qb->andWhere($qb->expr()->in('mount_provider_class', $qb->createNamedParameter($mountProviders, IQueryBuilder::PARAM_STR_ARRAY))); + } + $qb->orderBy('root_id', 'ASC'); + $result = $qb->executeQuery(); + + while ( + /** @var array{storage_id:int, root_id:int,mount_provider_class:string} $row */ + $row = $result->fetch() + ) { + $storageId = (int)$row['storage_id']; + $rootId = (int)$row['root_id']; + $overrideRoot = $rootId; + // LocalHomeMountProvider is the default provider for user home directories + // ObjectHomeMountProvider is the home directory provider for when S3 primary storage is used + if ($onlyUserFilesMounts && in_array($row['mount_provider_class'], [ + \OC\Files\Mount\LocalHomeMountProvider::class, + \OC\Files\Mount\ObjectHomeMountProvider::class, + ], true)) { + // Only crawl files, not cache or trashbin + $qb = $this->getQuery(); + try { + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('path', $qb->createNamedParameter('files'))); + /** @var array|false $root */ + $root = $qb->executeQuery()->fetch(); + if ($root !== false) { + $overrideRoot = (int)$root['fileid']; + } + } catch (Exception $e) { + $this->logger->error('Could not fetch home storage files root for storage ' . $storageId, ['exception' => $e]); + continue; + } + } + // Reference to root_id is still necessary even if we have the overridden_root_id, because storage_id and root_id uniquely identify a mount + yield [ + 'storage_id' => $storageId, + 'root_id' => $rootId, + 'overridden_root' => $overrideRoot, + ]; + } + $result->closeCursor(); + } +} diff --git a/lib/private/Files/Cache/HomeCache.php b/lib/private/Files/Cache/HomeCache.php new file mode 100644 index 00000000000..248cdc818f0 --- /dev/null +++ b/lib/private/Files/Cache/HomeCache.php @@ -0,0 +1,47 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; + +class HomeCache extends Cache { + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + if ($path !== '/' and $path !== '' and $path !== 'files' and $path !== 'files_trashbin' and $path !== 'files_versions') { + return parent::calculateFolderSize($path, $entry); + } elseif ($path === '' or $path === '/') { + // since the size of / isn't used (the size of /files is used instead) there is no use in calculating it + return 0; + } else { + return $this->calculateFolderSizeInner($path, $entry, true); + } + } + + /** + * @param string $file + * @return ICacheEntry + */ + public function get($file) { + $data = parent::get($file); + if ($file === '' or $file === '/') { + // only the size of the "files" dir counts + $filesData = parent::get('files'); + + if (isset($filesData['size'])) { + $data['size'] = $filesData['size']; + } + } + return $data; + } +} diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php new file mode 100644 index 00000000000..d4ac8a7c8e3 --- /dev/null +++ b/lib/private/Files/Cache/HomePropagator.php @@ -0,0 +1,37 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\IDBConnection; + +class HomePropagator extends Propagator { + private $ignoredBaseFolders; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection) { + parent::__construct($storage, $connection); + $this->ignoredBaseFolders = ['files_encryption']; + } + + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + [$baseFolder] = explode('/', $internalPath, 2); + if (in_array($baseFolder, $this->ignoredBaseFolders)) { + return []; + } else { + parent::propagateChange($internalPath, $time, $sizeDifference); + } + } +} diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php new file mode 100644 index 00000000000..3f4f70b865b --- /dev/null +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -0,0 +1,32 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +class LocalRootScanner extends Scanner { + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + if ($this->shouldScanPath($file)) { + return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); + } else { + return null; + } + } + + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + if ($this->shouldScanPath($path)) { + return parent::scan($path, $recursive, $reuse, $lock); + } else { + return null; + } + } + + private function shouldScanPath(string $path): bool { + $path = trim($path, '/'); + return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); + } +} diff --git a/lib/private/Files/Cache/MoveFromCacheTrait.php b/lib/private/Files/Cache/MoveFromCacheTrait.php new file mode 100644 index 00000000000..db35c6bb7f8 --- /dev/null +++ b/lib/private/Files/Cache/MoveFromCacheTrait.php @@ -0,0 +1,44 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; + +/** + * Fallback implementation for moveFromCache + */ +trait MoveFromCacheTrait { + /** + * store meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + abstract public function put($file, array $data); + + abstract public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int; + + /** + * Move a file or folder in the cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $sourceEntry = $sourceCache->get($sourcePath); + + $this->copyFromCache($sourceCache, $sourceEntry, $targetPath); + + $sourceCache->remove($sourcePath); + } +} diff --git a/lib/private/Files/Cache/NullWatcher.php b/lib/private/Files/Cache/NullWatcher.php new file mode 100644 index 00000000000..e3659214849 --- /dev/null +++ b/lib/private/Files/Cache/NullWatcher.php @@ -0,0 +1,38 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +class NullWatcher extends Watcher { + private $policy; + + public function __construct() { + } + + public function setPolicy($policy) { + $this->policy = $policy; + } + + public function getPolicy() { + return $this->policy; + } + + public function checkUpdate($path, $cachedEntry = null) { + return false; + } + + public function update($path, $cachedData) { + } + + public function needsUpdate($path, $cachedData) { + return false; + } + + public function cleanFolder($path) { + } +} diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php new file mode 100644 index 00000000000..a6ba87896f4 --- /dev/null +++ b/lib/private/Files/Cache/Propagator.php @@ -0,0 +1,223 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OC\DB\Exceptions\DbalException; +use OC\Files\Storage\Wrapper\Encryption; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; +use OCP\Server; +use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; + +/** + * Propagate etags and mtimes within the storage + */ +class Propagator implements IPropagator { + public const MAX_RETRIES = 3; + private $inBatch = false; + + private $batch = []; + + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @var IDBConnection + */ + private $connection; + + /** + * @var array + */ + private $ignore = []; + + private ClockInterface $clock; + + public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) { + $this->storage = $storage; + $this->connection = $connection; + $this->ignore = $ignore; + $this->clock = Server::get(ClockInterface::class); + } + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference number of bytes the file has grown + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + // Do not propagate changes in ignored paths + foreach ($this->ignore as $ignore) { + if (str_starts_with($internalPath, $ignore)) { + return; + } + } + + $time = min((int)$time, $this->clock->now()->getTimestamp()); + + $storageId = $this->storage->getStorageCache()->getNumericId(); + + $parents = $this->getParents($internalPath); + + if ($this->inBatch) { + foreach ($parents as $parent) { + $this->addToBatch($parent, $time, $sizeDifference); + } + return; + } + + $parentHashes = array_map('md5', $parents); + $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag + + $builder = $this->connection->getQueryBuilder(); + $hashParams = array_map(function ($hash) use ($builder) { + return $builder->expr()->literal($hash); + }, $parentHashes); + + $builder->update('filecache') + ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT))) + ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($builder->expr()->in('path_hash', $hashParams)); + if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); + } + + if ($sizeDifference !== 0) { + $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $newSize = $builder->func()->greatest( + $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + ); + + // Only update if row had a previously calculated size + $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); + + if ($this->storage->instanceOfStorage(Encryption::class)) { + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); + $newUnencryptedSize = $builder->func()->greatest( + $builder->func()->add( + $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), + $builder->createNamedParameter($sizeDifference) + ), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + ); + + // Only update if row had a previously calculated size + $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); + } + } + + for ($i = 0; $i < self::MAX_RETRIES; $i++) { + try { + $builder->executeStatement(); + break; + } catch (DbalException $e) { + if (!$e->isRetryable()) { + throw $e; + } + + /** @var LoggerInterface $loggerInterface */ + $loggerInterface = \OCP\Server::get(LoggerInterface::class); + $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); + } + } + } + + protected function getParents($path) { + $parts = explode('/', $path); + $parent = ''; + $parents = []; + foreach ($parts as $part) { + $parents[] = $parent; + $parent = trim($parent . '/' . $part, '/'); + } + return $parents; + } + + /** + * Mark the beginning of a propagation batch + * + * Note that not all cache setups support propagation in which case this will be a noop + * + * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent + * before the batch is committed. + */ + public function beginBatch() { + $this->inBatch = true; + } + + private function addToBatch($internalPath, $time, $sizeDifference) { + if (!isset($this->batch[$internalPath])) { + $this->batch[$internalPath] = [ + 'hash' => md5($internalPath), + 'time' => $time, + 'size' => $sizeDifference, + ]; + } else { + $this->batch[$internalPath]['size'] += $sizeDifference; + if ($time > $this->batch[$internalPath]['time']) { + $this->batch[$internalPath]['time'] = $time; + } + } + } + + /** + * Commit the active propagation batch + */ + public function commitBatch() { + if (!$this->inBatch) { + throw new \BadMethodCallException('Not in batch'); + } + $this->inBatch = false; + + $this->connection->beginTransaction(); + + $query = $this->connection->getQueryBuilder(); + $storageId = (int)$this->storage->getStorageCache()->getNumericId(); + + $query->update('filecache') + ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) + ->set('etag', $query->expr()->literal(uniqid())) + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); + + $sizeQuery = $this->connection->getQueryBuilder(); + $sizeQuery->update('filecache') + ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size'))) + ->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash'))) + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); + + foreach ($this->batch as $item) { + $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); + $query->setParameter('hash', $item['hash']); + + $query->executeStatement(); + + if ($item['size']) { + $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); + $sizeQuery->setParameter('hash', $item['hash']); + + $sizeQuery->executeStatement(); + } + } + + $this->batch = []; + + $this->connection->commit(); + } +} diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php new file mode 100644 index 00000000000..3ddcf1ca4e6 --- /dev/null +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -0,0 +1,240 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\QueryOptimizer\QueryOptimizer; +use OC\Files\Search\SearchBinaryOperator; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\IUser; +use Psr\Log\LoggerInterface; + +class QuerySearchHelper { + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + protected function getQueryBuilder() { + return new CacheQueryBuilder( + $this->connection->getQueryBuilder(), + $this->filesMetadataManager, + ); + } + + /** + * @param CacheQueryBuilder $query + * @param ISearchQuery $searchQuery + * @param array $caches + * @param IMetadataQuery|null $metadataQuery + */ + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null, + ): void { + $storageFilters = array_values(array_map(function (ICache $cache) { + return $cache->getQueryFilterForStorage(); + }, $caches)); + $storageFilter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters); + $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); + $this->queryOptimizer->processOperator($filter); + + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); + if ($searchExpr) { + $query->andWhere($searchExpr); + } + + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); + + if ($searchQuery->getLimit()) { + $query->setMaxResults($searchQuery->getLimit()); + } + if ($searchQuery->getOffset()) { + $query->setFirstResult($searchQuery->getOffset()); + } + } + + + /** + * @return array<array-key, array{id: int, name: string, visibility: int, editable: int, ref_file_id: int, number_files: int}> + */ + public function findUsedTagsInCaches(ISearchQuery $searchQuery, array $caches): array { + $query = $this->getQueryBuilder(); + $query->selectTagUsage(); + + $this->applySearchConstraints($query, $searchQuery, $caches); + + $result = $query->executeQuery(); + $tags = $result->fetchAll(); + $result->closeCursor(); + return $tags; + } + + protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user): void { + $query->leftJoin('file', 'systemtag_object_mapping', 'systemtagmap', $query->expr()->andX( + $query->expr()->eq('file.fileid', $query->expr()->castColumn('systemtagmap.objectid', IQueryBuilder::PARAM_INT)), + $query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')) + )); + $on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid')); + if (!$this->groupManager->isAdmin($user->getUID())) { + $on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true))); + } + $query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on); + } + + protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void { + $query + ->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid')) + ->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX( + $query->expr()->eq('tagmap.categoryid', 'tag.id'), + $query->expr()->eq('tag.type', $query->createNamedParameter('files')), + $query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID())) + )); + } + + + protected function equipQueryForShares(CacheQueryBuilder $query): void { + $query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source')); + } + + /** + * Perform a file system search in multiple caches + * + * the results will be grouped by the same array keys as the $caches argument to allow + * post-processing based on which cache the result came from + * + * @template T of array-key + * @param ISearchQuery $searchQuery + * @param array<T, ICache> $caches + * @return array<T, ICacheEntry[]> + */ + public function searchInCaches(ISearchQuery $searchQuery, array $caches): array { + // search in multiple caches at once by creating one query in the following format + // SELECT ... FROM oc_filecache WHERE + // <filter expressions from the search query> + // AND ( + // <filter expression for storage1> OR + // <filter expression for storage2> OR + // ... + // ); + // + // This gives us all the files matching the search query from all caches + // + // while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id) + // we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result. + + $builder = $this->getQueryBuilder(); + + $query = $builder->selectFileCache('file', false); + + $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + + if (in_array('systemtag', $requestedFields)) { + $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); + } + if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) { + $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); + } + if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) { + $this->equipQueryForShares($query); + } + + $metadataQuery = $query->selectMetadata(); + + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); + + $result = $query->executeQuery(); + $files = $result->fetchAll(); + + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); + return Cache::cacheEntryFromData($data, $this->mimetypeLoader); + }, $files); + + $result->closeCursor(); + + // loop through all caches for each result to see if the result matches that storage + // results are grouped by the same array keys as the caches argument to allow the caller to distinguish the source of the results + $results = array_fill_keys(array_keys($caches), []); + foreach ($rawEntries as $rawEntry) { + foreach ($caches as $cacheKey => $cache) { + $entry = $cache->getCacheEntryFromSearchResult($rawEntry); + if ($entry) { + $results[$cacheKey][] = $entry; + } + } + } + return $results; + } + + protected function requireUser(ISearchQuery $searchQuery): IUser { + $user = $searchQuery->getUser(); + if ($user === null) { + throw new \InvalidArgumentException('This search operation requires the user to be set in the query'); + } + return $user; + } + + /** + * @return list{0?: array<array-key, ICache>, 1?: array<array-key, IMountPoint>} + */ + public function getCachesAndMountPointsForSearch(IRootFolder $root, string $path, bool $limitToHome = false): array { + $rootLength = strlen($path); + $mount = $root->getMount($path); + $storage = $mount->getStorage(); + if ($storage === null) { + return []; + } + $internalPath = $mount->getInternalPath($path); + + if ($internalPath !== '') { + // a temporary CacheJail is used to handle filtering down the results to within this folder + /** @var ICache[] $caches */ + $caches = ['' => new CacheJail($storage->getCache(''), $internalPath)]; + } else { + /** @var ICache[] $caches */ + $caches = ['' => $storage->getCache('')]; + } + /** @var IMountPoint[] $mountByMountPoint */ + $mountByMountPoint = ['' => $mount]; + + if (!$limitToHome) { + $mounts = $root->getMountsIn($path); + foreach ($mounts as $mount) { + $storage = $mount->getStorage(); + if ($storage) { + $relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/'); + $caches[$relativeMountPoint] = $storage->getCache(''); + $mountByMountPoint[$relativeMountPoint] = $mount; + } + } + } + + return [$caches, $mountByMountPoint]; + } +} diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php new file mode 100644 index 00000000000..b067f70b8cb --- /dev/null +++ b/lib/private/Files/Cache/Scanner.php @@ -0,0 +1,623 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OC\Hooks\BasicEmitter; +use OC\SystemConfig; +use OCP\Files\Cache\IScanner; +use OCP\Files\ForbiddenException; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IDBConnection; +use OCP\Lock\ILockingProvider; +use Psr\Log\LoggerInterface; + +/** + * Class Scanner + * + * Hooks available in scope \OC\Files\Cache\Scanner: + * - scanFile(string $path, string $storageId) + * - scanFolder(string $path, string $storageId) + * - postScanFile(string $path, string $storageId) + * - postScanFolder(string $path, string $storageId) + * + * @package OC\Files\Cache + */ +class Scanner extends BasicEmitter implements IScanner { + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var string $storageId + */ + protected $storageId; + + /** + * @var \OC\Files\Cache\Cache $cache + */ + protected $cache; + + /** + * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache + */ + protected $cacheActive; + + /** + * @var bool $useTransactions whether to use transactions + */ + protected $useTransactions = true; + + /** + * @var \OCP\Lock\ILockingProvider + */ + protected $lockingProvider; + + protected IDBConnection $connection; + + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->storageId = $this->storage->getId(); + $this->cache = $storage->getCache(); + /** @var SystemConfig $config */ + $config = \OC::$server->get(SystemConfig::class); + $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); + $this->lockingProvider = \OC::$server->get(ILockingProvider::class); + $this->connection = \OC::$server->get(IDBConnection::class); + } + + /** + * Whether to wrap the scanning of a folder in a database transaction + * On default transactions are used + * + * @param bool $useTransactions + */ + public function setUseTransactions($useTransactions): void { + $this->useTransactions = $useTransactions; + } + + /** + * get all the metadata of a file or folder + * * + * + * @param string $path + * @return array|null an array of metadata of the file + */ + protected function getData($path) { + $data = $this->storage->getMetaData($path); + if (is_null($data)) { + \OC::$server->get(LoggerInterface::class)->debug("!!! Path '$path' is not accessible or present !!!", ['app' => 'core']); + } + return $data; + } + + /** + * scan a single file and store it in the cache + * + * @param string $file + * @param int $reuseExisting + * @param int $parentId + * @param array|CacheEntry|null|false $cacheData existing data in the cache for the file to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @param array|null $data the metadata for the file, as returned by the storage + * @return array|null an array of metadata of the scanned file + * @throws \OCP\Lock\LockedException + */ + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + if ($file !== '') { + try { + $this->storage->verifyPath(dirname($file), basename($file)); + } catch (\Exception $e) { + return null; + } + } + + // only proceed if $file is not a partial file, blacklist is handled by the storage + if (self::isPartialFile($file)) { + return null; + } + + // acquire a lock + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + try { + $data = $data ?? $this->getData($file); + } catch (ForbiddenException $e) { + if ($lock) { + if ($this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + return null; + } + + try { + if ($data === null) { + $this->removeFromCache($file); + } else { + // pre-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'scanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + + $parent = dirname($file); + if ($parent === '.' || $parent === '/') { + $parent = ''; + } + if ($parentId === -1) { + $parentId = $this->cache->getParentId($file); + } + + // scan the parent if it's not in the cache (id -1) and the current file is not the root folder + if ($file && $parentId === -1) { + $parentData = $this->scanFile($parent); + if ($parentData === null) { + return null; + } + + $parentId = $parentData['fileid']; + } + if ($parent) { + $data['parent'] = $parentId; + } + + $cacheData = $cacheData ?? $this->cache->get($file); + if ($reuseExisting && $cacheData !== false && isset($cacheData['fileid'])) { + // prevent empty etag + $etag = empty($cacheData['etag']) ? $data['etag'] : $cacheData['etag']; + $fileId = $cacheData['fileid']; + $data['fileid'] = $fileId; + // only reuse data if the file hasn't explicitly changed + $mtimeUnchanged = isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']; + // if the folder is marked as unscanned, never reuse etags + if ($mtimeUnchanged && $cacheData['size'] !== -1) { + $data['mtime'] = $cacheData['mtime']; + if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) { + $data['size'] = $cacheData['size']; + } + if ($reuseExisting & self::REUSE_ETAG && !$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { + $data['etag'] = $etag; + } + } + + // we only updated unencrypted_size if it's already set + if (isset($cacheData['unencrypted_size']) && $cacheData['unencrypted_size'] === 0) { + unset($data['unencrypted_size']); + } + + /** + * Only update metadata that has changed. + * i.e. get all the values in $data that are not present in the cache already + * + * We need the OC implementation for usage of "getData" method below. + * @var \OC\Files\Cache\CacheEntry $cacheData + */ + $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); + + // make it known to the caller that etag has been changed and needs propagation + if (isset($newData['etag'])) { + $data['etag_changed'] = true; + } + } else { + unset($data['unencrypted_size']); + $newData = $data; + $fileId = -1; + } + if (!empty($newData)) { + // Reset the checksum if the data has changed + $newData['checksum'] = ''; + $newData['parent'] = $parentId; + $data['fileid'] = $this->addToCache($file, $newData, $fileId); + } + + if ($cacheData !== false) { + $data['oldSize'] = $cacheData['size'] ?? 0; + $data['encrypted'] = $cacheData['encrypted'] ?? false; + } + + // post-emit only if it was a file. By that we avoid counting/treating folders as files + if ($data['mimetype'] !== 'httpd/unix-directory') { + $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', [$file, $this->storageId]); + \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', ['path' => $file, 'storage' => $this->storageId]); + } + } + } finally { + // release the acquired lock + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($file, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + } + + return $data; + } + + protected function removeFromCache($path) { + \OC_Hook::emit('Scanner', 'removeFromCache', ['file' => $path]); + $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', [$path]); + if ($this->cacheActive) { + $this->cache->remove($path); + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + * @return int the id of the added file + */ + protected function addToCache($path, $data, $fileId = -1) { + if (isset($data['scan_permissions'])) { + $data['permissions'] = $data['scan_permissions']; + } + \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', [$path, $this->storageId, $data, $fileId]); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + return $fileId; + } else { + return $this->cache->insert($path, $data); + } + } else { + return -1; + } + } + + /** + * @param string $path + * @param array $data + * @param int $fileId + */ + protected function updateCache($path, $data, $fileId = -1) { + \OC_Hook::emit('Scanner', 'addToCache', ['file' => $path, 'data' => $data]); + $this->emit('\OC\Files\Cache\Scanner', 'updateCache', [$path, $this->storageId, $data]); + if ($this->cacheActive) { + if ($fileId !== -1) { + $this->cache->update($fileId, $data); + } else { + $this->cache->put($path, $data); + } + } + } + + /** + * scan a folder and all it's children + * + * @param string $path + * @param bool $recursive + * @param int $reuse + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @return array|null an array of the meta data of the scanned file or folder + */ + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->acquireLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + $this->storage->acquireLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + } + + try { + $data = $this->scanFile($path, $reuse, -1, lock: $lock); + + if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); + $data['size'] = $size; + } + } catch (NotFoundException $e) { + $this->removeFromCache($path); + return null; + } finally { + if ($lock && $this->storage->instanceOfStorage(ILockingStorage::class)) { + $this->storage->releaseLock($path, ILockingProvider::LOCK_SHARED, $this->lockingProvider); + $this->storage->releaseLock('scanner::' . $path, ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + } + } + return $data; + } + + /** + * Compares $array1 against $array2 and returns all the values in $array1 that are not in $array2 + * Note this is a one-way check - i.e. we don't care about things that are in $array2 that aren't in $array1 + * + * Supports multi-dimensional arrays + * Also checks keys/indexes + * Comparisons are strict just like array_diff_assoc + * Order of keys/values does not matter + * + * @param array $array1 + * @param array $array2 + * @return array with the differences between $array1 and $array1 + * @throws \InvalidArgumentException if $array1 isn't an actual array + * + */ + protected function array_diff_assoc_multi(array $array1, array $array2) { + + $result = []; + + foreach ($array1 as $key => $value) { + + // if $array2 doesn't have the same key, that's a result + if (!array_key_exists($key, $array2)) { + $result[$key] = $value; + continue; + } + + // if $array2's value for the same key is different, that's a result + if ($array2[$key] !== $value && !is_array($value)) { + $result[$key] = $value; + continue; + } + + if (is_array($value)) { + $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); + if (!empty($nestedDiff)) { + $result[$key] = $nestedDiff; + continue; + } + } + } + return $result; + } + + /** + * Get the children currently in the cache + * + * @param int $folderId + * @return array<string, \OCP\Files\Cache\ICacheEntry> + */ + protected function getExistingChildren($folderId): array { + $existingChildren = []; + $children = $this->cache->getFolderContentsById($folderId); + foreach ($children as $child) { + $existingChildren[$child['name']] = $child; + } + return $existingChildren; + } + + /** + * scan all the files and folders in a folder + * + * @param string $path + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + * @param int $reuse a combination of self::REUSE_* + * @param int $folderId id for the folder to be scanned + * @param bool $lock set to false to disable getting an additional read lock during scanning + * @param int|float $oldSize the size of the folder before (re)scanning the children + * @return int|float the size of the scanned folder or -1 if the size is unknown at this stage + */ + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + if ($reuse === -1) { + $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; + } + $this->emit('\OC\Files\Cache\Scanner', 'scanFolder', [$path, $this->storageId]); + $size = 0; + $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + + foreach ($childQueue as $child => [$childId, $childSize]) { + // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed + $childEtagChanged = false; + $childSize = $this->scanChildren($child, $recursive, $reuse, $childId, $lock, $childSize, $childEtagChanged); + $etagChanged |= $childEtagChanged; + + if ($childSize === -1) { + $size = -1; + } elseif ($size !== -1) { + $size += $childSize; + } + } + + // for encrypted storages, we trigger a regular folder size calculation instead of using the calculated size + // to make sure we also updated the unencrypted-size where applicable + if ($this->storage->instanceOfStorage(Encryption::class)) { + $this->cache->calculateFolderSize($path); + } else { + if ($this->cacheActive) { + $updatedData = []; + if ($oldSize !== $size) { + $updatedData['size'] = $size; + } + if ($etagChanged) { + $updatedData['etag'] = uniqid(); + } + if ($updatedData) { + $this->cache->update($folderId, $updatedData); + } + } + } + $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); + return $size; + } + + /** + * @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive + */ + private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array { + // we put this in it's own function so it cleans up the memory before we start recursing + $existingChildren = $this->getExistingChildren($folderId); + $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); + + if (count($existingChildren) === 0 && count($newChildren) === 0) { + // no need to do a transaction + return []; + } + + if ($this->useTransactions) { + $this->connection->beginTransaction(); + } + + $exceptionOccurred = false; + $childQueue = []; + $newChildNames = []; + foreach ($newChildren as $fileMeta) { + $permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions']; + if ($permissions === 0) { + continue; + } + $originalFile = $fileMeta['name']; + $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/'); + if (trim($originalFile, '/') !== $file) { + // encoding mismatch, might require compatibility wrapper + \OC::$server->get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".', ['app' => 'core']); + $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]); + // skip this entry + continue; + } + + $newChildNames[] = $file; + $child = $path ? $path . '/' . $file : $file; + try { + $existingData = $existingChildren[$file] ?? false; + $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta); + if ($data) { + if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) { + $childQueue[$child] = [$data['fileid'], $data['size']]; + } elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) { + // only recurse into folders which aren't fully scanned + $childQueue[$child] = [$data['fileid'], $data['size']]; + } elseif ($data['size'] === -1) { + $size = -1; + } elseif ($size !== -1) { + $size += $data['size']; + } + + if (isset($data['etag_changed']) && $data['etag_changed']) { + $etagChanged = true; + } + } + } catch (Exception $ex) { + // might happen if inserting duplicate while a scanning + // process is running in parallel + // log and ignore + if ($this->useTransactions) { + $this->connection->rollback(); + $this->connection->beginTransaction(); + } + \OC::$server->get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [ + 'app' => 'core', + 'exception' => $ex, + ]); + $exceptionOccurred = true; + } catch (\OCP\Lock\LockedException $e) { + if ($this->useTransactions) { + $this->connection->rollback(); + } + throw $e; + } + } + $removedChildren = \array_diff(array_keys($existingChildren), $newChildNames); + foreach ($removedChildren as $childName) { + $child = $path ? $path . '/' . $childName : $childName; + $this->removeFromCache($child); + } + if ($this->useTransactions) { + $this->connection->commit(); + } + if ($exceptionOccurred) { + // It might happen that the parallel scan process has already + // inserted mimetypes but those weren't available yet inside the transaction + // To make sure to have the updated mime types in such cases, + // we reload them here + \OC::$server->getMimeTypeLoader()->reset(); + } + return $childQueue; + } + + /** + * check if the file should be ignored when scanning + * NOTE: files with a '.part' extension are ignored as well! + * prevents unfinished put requests to be scanned + * + * @param string $file + * @return boolean + */ + public static function isPartialFile($file) { + if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { + return true; + } + if (str_contains($file, '.part/')) { + return true; + } + + return false; + } + + /** + * walk over any folders that are not fully scanned yet and scan them + */ + public function backgroundScan() { + if ($this->storage->instanceOfStorage(Jail::class)) { + // for jail storage wrappers (shares, groupfolders) we run the background scan on the source storage + // this is mainly done because the jail wrapper doesn't implement `getIncomplete` (because it would be inefficient). + // + // Running the scan on the source storage might scan more than "needed", but the unscanned files outside the jail will + // have to be scanned at some point anyway. + $unJailedScanner = $this->storage->getUnjailedStorage()->getScanner(); + $unJailedScanner->backgroundScan(); + } else { + if (!$this->cache->inCache('')) { + // if the storage isn't in the cache yet, just scan the root completely + $this->runBackgroundScanJob(function () { + $this->scan('', self::SCAN_RECURSIVE, self::REUSE_ETAG); + }, ''); + } else { + $lastPath = null; + // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck) + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function () use ($path) { + $this->scan($path, self::SCAN_RECURSIVE_INCOMPLETE, self::REUSE_ETAG | self::REUSE_SIZE); + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } + } + } + } + + protected function runBackgroundScanJob(callable $callback, $path) { + try { + $callback(); + \OC_Hook::emit('Scanner', 'correctFolderSize', ['path' => $path]); + if ($this->cacheActive && $this->cache instanceof Cache) { + $this->cache->correctFolderSize($path, null, true); + } + } catch (\OCP\Files\StorageInvalidException $e) { + // skip unavailable storages + } catch (\OCP\Files\StorageNotAvailableException $e) { + // skip unavailable storages + } catch (\OCP\Files\ForbiddenException $e) { + // skip forbidden storages + } catch (\OCP\Lock\LockedException $e) { + // skip unavailable storages + } + } + + /** + * Set whether the cache is affected by scan operations + * + * @param boolean $active The active state of the cache + */ + public function setCacheActive($active) { + $this->cacheActive = $active; + } +} diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php new file mode 100644 index 00000000000..e1d3c42a8a2 --- /dev/null +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -0,0 +1,355 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\IMetadataQuery; + +/** + * Tools for transforming search queries into database queries + * + * @psalm-import-type ParamSingleValue from ISearchComparison + * @psalm-import-type ParamValue from ISearchComparison + */ +class SearchBuilder { + /** @var array<string, string> */ + protected static $searchOperatorMap = [ + ISearchComparison::COMPARE_LIKE => 'iLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like', + ISearchComparison::COMPARE_EQUAL => 'eq', + ISearchComparison::COMPARE_GREATER_THAN => 'gt', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte', + ISearchComparison::COMPARE_LESS_THAN => 'lt', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte', + ISearchComparison::COMPARE_DEFINED => 'isNotNull', + ISearchComparison::COMPARE_IN => 'in', + ]; + + /** @var array<string, string> */ + protected static $searchOperatorNegativeMap = [ + ISearchComparison::COMPARE_LIKE => 'notLike', + ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike', + ISearchComparison::COMPARE_EQUAL => 'neq', + ISearchComparison::COMPARE_GREATER_THAN => 'lte', + ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt', + ISearchComparison::COMPARE_LESS_THAN => 'gte', + ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt', + ISearchComparison::COMPARE_DEFINED => 'isNull', + ISearchComparison::COMPARE_IN => 'notIn', + ]; + + /** @var array<string, string> */ + protected static $fieldTypes = [ + 'mimetype' => 'string', + 'mtime' => 'integer', + 'name' => 'string', + 'path' => 'string', + 'size' => 'integer', + 'tagname' => 'string', + 'systemtag' => 'string', + 'favorite' => 'boolean', + 'fileid' => 'integer', + 'storage' => 'integer', + 'share_with' => 'string', + 'share_type' => 'integer', + 'owner' => 'string', + ]; + + /** @var array<string, int|string> */ + protected static $paramTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR, + 'integer' => IQueryBuilder::PARAM_INT, + 'boolean' => IQueryBuilder::PARAM_BOOL, + ]; + + /** @var array<string, int> */ + protected static $paramArrayTypeMap = [ + 'string' => IQueryBuilder::PARAM_STR_ARRAY, + 'integer' => IQueryBuilder::PARAM_INT_ARRAY, + 'boolean' => IQueryBuilder::PARAM_INT_ARRAY, + ]; + + public const TAG_FAVORITE = '_$!<Favorite>!$_'; + + public function __construct( + private IMimeTypeLoader $mimetypeLoader, + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @return string[] + */ + public function extractRequestedFields(ISearchOperator $operator): array { + if ($operator instanceof ISearchBinaryOperator) { + return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { + return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); + }, []); + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { + return [$operator->getField()]; + } + return []; + } + + /** + * @param IQueryBuilder $builder + * @param ISearchOperator[] $operators + */ + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null, + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); + }, $operators)); + } + + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null, + ) { + $expr = $builder->expr(); + + if ($operator instanceof ISearchBinaryOperator) { + if (count($operator->getArguments()) === 0) { + return null; + } + + switch ($operator->getType()) { + case ISearchBinaryOperator::OPERATOR_NOT: + $negativeOperator = $operator->getArguments()[0]; + if ($negativeOperator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); + } + // no break + case ISearchBinaryOperator::OPERATOR_AND: + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + case ISearchBinaryOperator::OPERATOR_OR: + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); + default: + throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); + } + } elseif ($operator instanceof ISearchComparison) { + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); + } + } + + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null, + ) { + if ($comparison->getExtra()) { + [$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison); + } + + if (isset($operatorMap[$type])) { + $queryOperator = $operatorMap[$type]; + return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType)); + } else { + throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType()); + } + } + + /** + * @param ISearchComparison $operator + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValue(ISearchComparison $operator): array { + $this->validateComparison($operator); + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + $pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true); + return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash); + } + + /** + * @param string $field + * @param ParamValue $value + * @param string $type + * @return list{string, ParamValue, string, string} + */ + private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array { + $paramType = self::$fieldTypes[$field]; + if ($type === ISearchComparison::COMPARE_IN) { + $resultField = $field; + $values = []; + foreach ($value as $arrayValue) { + /** @var ParamSingleValue $arrayValue */ + [$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash); + $resultField = $arrayField; + $values[] = $arrayValue; + } + return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType]; + } + if ($field === 'mimetype') { + $value = (string)$value; + if ($type === ISearchComparison::COMPARE_EQUAL) { + $value = $this->mimetypeLoader->getId($value); + } elseif ($type === ISearchComparison::COMPARE_LIKE) { + // transform "mimetype='foo/%'" to "mimepart='foo'" + if (preg_match('|(.+)/%|', $value, $matches)) { + $field = 'mimepart'; + $value = $this->mimetypeLoader->getId($matches[1]); + $type = ISearchComparison::COMPARE_EQUAL; + } elseif (str_contains($value, '%')) { + throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported'); + } else { + $field = 'mimetype'; + $value = $this->mimetypeLoader->getId($value); + $type = ISearchComparison::COMPARE_EQUAL; + } + } + } elseif ($field === 'favorite') { + $field = 'tag.category'; + $value = self::TAG_FAVORITE; + $paramType = 'string'; + } elseif ($field === 'name') { + $field = 'file.name'; + } elseif ($field === 'tagname') { + $field = 'tag.category'; + } elseif ($field === 'systemtag') { + $field = 'systemtag.name'; + } elseif ($field === 'fileid') { + $field = 'file.fileid'; + } elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) { + $field = 'path_hash'; + $value = md5((string)$value); + } elseif ($field === 'owner') { + $field = 'uid_owner'; + } + return [$field, $value, $type, $paramType]; + } + + private function validateComparison(ISearchComparison $operator) { + $comparisons = [ + 'mimetype' => ['eq', 'like', 'in'], + 'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'name' => ['eq', 'like', 'clike', 'in'], + 'path' => ['eq', 'like', 'clike', 'in'], + 'size' => ['eq', 'gt', 'lt', 'gte', 'lte'], + 'tagname' => ['eq', 'like'], + 'systemtag' => ['eq', 'like'], + 'favorite' => ['eq'], + 'fileid' => ['eq', 'in'], + 'storage' => ['eq', 'in'], + 'share_with' => ['eq'], + 'share_type' => ['eq'], + 'owner' => ['eq'], + ]; + + if (!isset(self::$fieldTypes[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField()); + } + $type = self::$fieldTypes[$operator->getField()]; + if ($operator->getType() === ISearchComparison::COMPARE_IN) { + if (!is_array($operator->getValue())) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + foreach ($operator->getValue() as $arrayValue) { + if (gettype($arrayValue) !== $type) { + throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField()); + } + } + } else { + if (gettype($operator->getValue()) !== $type) { + throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField()); + } + } + if (!in_array($operator->getType(), $comparisons[$operator->getField()])) { + throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType()); + } + } + + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + $knownMetadata = $this->filesMetadataManager->getKnownMetadata(); + $isIndex = $knownMetadata->isIndex($field); + $paramType = $knownMetadata->getType($field) === 'int' ? 'integer' : 'string'; + + if (!$isIndex) { + throw new \InvalidArgumentException('Cannot search non indexed metadata key'); + } + + switch ($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type, $paramType]; + } + + private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) { + if ($value instanceof \DateTime) { + $value = $value->getTimestamp(); + } + if (is_array($value)) { + $type = self::$paramArrayTypeMap[$paramType]; + } else { + $type = self::$paramTypeMap[$paramType]; + } + return $builder->createNamedParameter($value, $type); + } + + /** + * @param IQueryBuilder $query + * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery + */ + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { + foreach ($orders as $order) { + $field = $order->getField(); + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; + + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } + $query->addOrderBy($field, $order->getDirection()); + } + } +} diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php new file mode 100644 index 00000000000..1a3bda58e6a --- /dev/null +++ b/lib/private/Files/Cache/Storage.php @@ -0,0 +1,235 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Storage\IStorage; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * Handle the mapping between the string and numeric storage ids + * + * Each storage has 2 different ids + * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') + * and a numeric storage id which is referenced in the file cache + * + * A mapping between the two storage ids is stored in the database and accessible through this class + * + * @package OC\Files\Cache + */ +class Storage { + /** @var StorageGlobal|null */ + private static $globalCache = null; + private $storageId; + private $numericId; + + /** + * @return StorageGlobal + */ + public static function getGlobalCache() { + if (is_null(self::$globalCache)) { + self::$globalCache = new StorageGlobal(\OC::$server->getDatabaseConnection()); + } + return self::$globalCache; + } + + /** + * @param \OC\Files\Storage\Storage|string $storage + * @param bool $isAvailable + * @throws \RuntimeException + */ + public function __construct($storage, $isAvailable, IDBConnection $connection) { + if ($storage instanceof IStorage) { + $this->storageId = $storage->getId(); + } else { + $this->storageId = $storage; + } + $this->storageId = self::adjustStorageId($this->storageId); + + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = (int)$row['numeric_id']; + } else { + $available = $isAvailable ? 1 : 0; + if ($connection->insertIfNotExist('*PREFIX*storages', ['id' => $this->storageId, 'available' => $available])) { + $this->numericId = $connection->lastInsertId('*PREFIX*storages'); + } else { + if ($row = self::getStorageById($this->storageId)) { + $this->numericId = (int)$row['numeric_id']; + } else { + throw new \RuntimeException('Storage could neither be inserted nor be selected from the database: ' . $this->storageId); + } + } + } + } + + /** + * @param string $storageId + * @return array + */ + public static function getStorageById($storageId) { + return self::getGlobalCache()->getStorageInfo($storageId); + } + + /** + * Adjusts the storage id to use md5 if too long + * @param string $storageId storage id + * @return string unchanged $storageId if its length is less than 64 characters, + * else returns the md5 of $storageId + */ + public static function adjustStorageId($storageId) { + if (strlen($storageId) > 64) { + return md5($storageId); + } + return $storageId; + } + + /** + * Get the numeric id for the storage + * + * @return int + */ + public function getNumericId() { + return $this->numericId; + } + + /** + * Get the string id for the storage + * + * @param int $numericId + * @return string|null either the storage id string or null if the numeric id is not known + */ + public static function getStorageId(int $numericId): ?string { + $storage = self::getGlobalCache()->getStorageInfoByNumericId($numericId); + return $storage['id'] ?? null; + } + + /** + * Get the numeric of the storage with the provided string id + * + * @param $storageId + * @return int|null either the numeric storage id or null if the storage id is not known + */ + public static function getNumericStorageId($storageId) { + $storageId = self::adjustStorageId($storageId); + + if ($row = self::getStorageById($storageId)) { + return (int)$row['numeric_id']; + } else { + return null; + } + } + + /** + * @return array [ available, last_checked ] + */ + public function getAvailability() { + if ($row = self::getStorageById($this->storageId)) { + return [ + 'available' => (int)$row['available'] === 1, + 'last_checked' => $row['last_checked'] + ]; + } else { + return [ + 'available' => true, + 'last_checked' => time(), + ]; + } + } + + /** + * @param bool $isAvailable + * @param int $delay amount of seconds to delay reconsidering that storage further + */ + public function setAvailability($isAvailable, int $delay = 0) { + $available = $isAvailable ? 1 : 0; + if (!$isAvailable) { + \OCP\Server::get(LoggerInterface::class)->info('Storage with ' . $this->storageId . ' marked as unavailable', ['app' => 'lib']); + } + + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->update('storages') + ->set('available', $query->createNamedParameter($available)) + ->set('last_checked', $query->createNamedParameter(time() + $delay)) + ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); + $query->executeStatement(); + } + + /** + * Check if a string storage id is known + * + * @param string $storageId + * @return bool + */ + public static function exists($storageId) { + return !is_null(self::getNumericStorageId($storageId)); + } + + /** + * remove the entry for the storage + * + * @param string $storageId + */ + public static function remove($storageId) { + $storageId = self::adjustStorageId($storageId); + $numericId = self::getNumericStorageId($storageId); + + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->eq('id', $query->createNamedParameter($storageId))); + $query->executeStatement(); + + if (!is_null($numericId)) { + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($numericId))); + $query->executeStatement(); + } + } + + /** + * remove the entry for the storage by the mount id + * + * @param int $mountId + */ + public static function cleanByMountId(int $mountId) { + $db = \OC::$server->getDatabaseConnection(); + + try { + $db->beginTransaction(); + + $query = $db->getQueryBuilder(); + $query->select('storage_id') + ->from('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $storageIds = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + $storageIds = array_unique($storageIds); + + $query = $db->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('storages') + ->where($query->expr()->in('numeric_id', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->executeStatement(); + + $query = $db->getQueryBuilder(); + $query->delete('mounts') + ->where($query->expr()->eq('mount_id', $query->createNamedParameter($mountId, IQueryBuilder::PARAM_INT))); + $query->executeStatement(); + + $db->commit(); + } catch (\Exception $e) { + $db->rollBack(); + throw $e; + } + } +} diff --git a/lib/private/Files/Cache/StorageGlobal.php b/lib/private/Files/Cache/StorageGlobal.php new file mode 100644 index 00000000000..bab31b1db91 --- /dev/null +++ b/lib/private/Files/Cache/StorageGlobal.php @@ -0,0 +1,99 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Handle the mapping between the string and numeric storage ids + * + * Each storage has 2 different ids + * a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share') + * and a numeric storage id which is referenced in the file cache + * + * A mapping between the two storage ids is stored in the database and accessible through this class + * + * @package OC\Files\Cache + */ +class StorageGlobal { + /** @var array<string, array> */ + private $cache = []; + /** @var array<int, array> */ + private $numericIdCache = []; + + public function __construct( + private IDBConnection $connection, + ) { + } + + /** + * @param string[] $storageIds + */ + public function loadForStorageIds(array $storageIds) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->in('id', $builder->createNamedParameter(array_values($storageIds), IQueryBuilder::PARAM_STR_ARRAY))); + + $result = $query->executeQuery(); + while ($row = $result->fetch()) { + $this->cache[$row['id']] = $row; + } + $result->closeCursor(); + } + + /** + * @param string $storageId + * @return array|null + */ + public function getStorageInfo(string $storageId): ?array { + if (!isset($this->cache[$storageId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->eq('id', $builder->createNamedParameter($storageId))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $this->cache[$storageId] = $row; + $this->numericIdCache[(int)$row['numeric_id']] = $row; + } + } + return $this->cache[$storageId] ?? null; + } + + /** + * @param int $numericId + * @return array|null + */ + public function getStorageInfoByNumericId(int $numericId): ?array { + if (!isset($this->numericIdCache[$numericId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select(['id', 'numeric_id', 'available', 'last_checked']) + ->from('storages') + ->where($builder->expr()->eq('numeric_id', $builder->createNamedParameter($numericId))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ($row) { + $this->numericIdCache[$numericId] = $row; + $this->cache[$row['id']] = $row; + } + } + return $this->numericIdCache[$numericId] ?? null; + } + + public function clearCache() { + $this->cache = []; + } +} diff --git a/lib/private/Files/Cache/Updater.php b/lib/private/Files/Cache/Updater.php new file mode 100644 index 00000000000..03681036aa2 --- /dev/null +++ b/lib/private/Files/Cache/Updater.php @@ -0,0 +1,292 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use Doctrine\DBAL\Exception\DeadlockException; +use OC\Files\FileInfo; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; + +/** + * Update the cache and propagate changes + * + */ +class Updater implements IUpdater { + /** + * @var bool + */ + protected $enabled = true; + + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @var \OC\Files\Cache\Propagator + */ + protected $propagator; + + /** + * @var Scanner + */ + protected $scanner; + + /** + * @var Cache + */ + protected $cache; + + private LoggerInterface $logger; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->propagator = $storage->getPropagator(); + $this->scanner = $storage->getScanner(); + $this->cache = $storage->getCache(); + $this->logger = \OC::$server->get(LoggerInterface::class); + } + + /** + * Disable updating the cache through this updater + */ + public function disable() { + $this->enabled = false; + } + + /** + * Re-enable the updating of the cache through this updater + */ + public function enable() { + $this->enabled = true; + } + + /** + * Get the propagator for etags and mtime for the view the updater works on + * + * @return Propagator + */ + public function getPropagator() { + return $this->propagator; + } + + /** + * Propagate etag and mtime changes for the parent folders of $path up to the root of the filesystem + * + * @param string $path the path of the file to propagate the changes for + * @param int|null $time the timestamp to set as mtime for the parent folders, if left out the current time is used + */ + public function propagate($path, $time = null) { + if (Scanner::isPartialFile($path)) { + return; + } + $this->propagator->propagateChange($path, $time); + } + + /** + * Update the cache for $path and update the size, etag and mtime of the parent folders + * + * @param string $path + * @param int $time + */ + public function update($path, $time = null, ?int $sizeDifference = null) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + if (is_null($time)) { + $time = time(); + } + + $data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false); + + if (isset($data['oldSize']) && isset($data['size'])) { + $sizeDifference = $data['size'] - $data['oldSize']; + } + + // encryption is a pita and touches the cache itself + if (isset($data['encrypted']) && (bool)$data['encrypted']) { + $sizeDifference = null; + } + + // scanner didn't provide size info, fallback to full size calculation + if ($this->cache instanceof Cache && $sizeDifference === null) { + $this->cache->correctFolderSize($path, $data); + } + $this->correctParentStorageMtime($path); + $this->propagator->propagateChange($path, $time, $sizeDifference ?? 0); + } + + /** + * Remove $path from the cache and update the size, etag and mtime of the parent folders + * + * @param string $path + */ + public function remove($path) { + if (!$this->enabled or Scanner::isPartialFile($path)) { + return; + } + + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + + $entry = $this->cache->get($path); + + $this->cache->remove($path); + + $this->correctParentStorageMtime($path); + if ($entry instanceof ICacheEntry) { + $this->propagator->propagateChange($path, time(), -$entry->getSize()); + } else { + $this->propagator->propagateChange($path, time()); + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($parent); + } + } + } + + /** + * Rename a file or folder in the cache. + * + * @param IStorage $sourceStorage + * @param string $source + * @param string $target + */ + public function renameFromStorage(IStorage $sourceStorage, $source, $target) { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache) use ($sourceStorage, $source, $target) { + // Remove existing cache entry to no reuse the fileId. + if ($this->cache->inCache($target)) { + $this->cache->remove($target); + } + + if ($sourceStorage === $this->storage) { + $this->cache->move($source, $target); + } else { + $this->cache->moveFromCache($sourceCache, $source, $target); + } + }); + } + + /** + * Copy a file or folder in the cache. + */ + public function copyFromStorage(IStorage $sourceStorage, string $source, string $target): void { + $this->copyOrRenameFromStorage($sourceStorage, $source, $target, function (ICache $sourceCache, ICacheEntry $sourceInfo) use ($target) { + $parent = dirname($target); + if ($parent === '.') { + $parent = ''; + } + $parentInCache = $this->cache->inCache($parent); + if (!$parentInCache) { + $parentData = $this->scanner->scan($parent, Scanner::SCAN_SHALLOW, -1, false); + $parentInCache = $parentData !== null; + } + if ($parentInCache) { + $this->cache->copyFromCache($sourceCache, $sourceInfo, $target); + } + }); + } + + /** + * Utility to copy or rename a file or folder in the cache and update the size, etag and mtime of the parent folders + */ + private function copyOrRenameFromStorage(IStorage $sourceStorage, string $source, string $target, callable $operation): void { + if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) { + return; + } + + $time = time(); + + $sourceCache = $sourceStorage->getCache(); + $sourceUpdater = $sourceStorage->getUpdater(); + $sourcePropagator = $sourceStorage->getPropagator(); + + $sourceInfo = $sourceCache->get($source); + + $sourceExtension = pathinfo($source, PATHINFO_EXTENSION); + $targetExtension = pathinfo($target, PATHINFO_EXTENSION); + $targetIsTrash = preg_match("/^d\d+$/", $targetExtension); + + if ($sourceInfo !== false) { + if (!$this->storage->instanceOfStorage(ObjectStoreStorage::class)) { + $operation($sourceCache, $sourceInfo); + } + + $isDir = $sourceInfo->getMimeType() === FileInfo::MIMETYPE_FOLDER; + } else { + $isDir = $this->storage->is_dir($target); + } + + if ($sourceExtension !== $targetExtension && !$isDir && !$targetIsTrash) { + // handle mime type change + $mimeType = $this->storage->getMimeType($target); + $fileId = $this->cache->getId($target); + $this->cache->update($fileId, ['mimetype' => $mimeType]); + } + + if ($sourceCache instanceof Cache) { + $sourceCache->correctFolderSize($source); + } + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($target); + } + if ($sourceUpdater instanceof Updater) { + $sourceUpdater->correctParentStorageMtime($source); + } + $this->correctParentStorageMtime($target); + $this->updateStorageMTimeOnly($target); + $sourcePropagator->propagateChange($source, $time); + $this->propagator->propagateChange($target, $time); + } + + private function updateStorageMTimeOnly($internalPath) { + $fileId = $this->cache->getId($internalPath); + if ($fileId !== -1) { + $mtime = $this->storage->filemtime($internalPath); + if ($mtime !== false) { + $this->cache->update( + $fileId, [ + 'mtime' => null, // this magic tells it to not overwrite mtime + 'storage_mtime' => $mtime + ] + ); + } + } + } + + /** + * update the storage_mtime of the direct parent in the cache to the mtime from the storage + * + * @param string $internalPath + */ + private function correctParentStorageMtime($internalPath) { + $parentId = $this->cache->getParentId($internalPath); + $parent = dirname($internalPath); + if ($parentId != -1) { + $mtime = $this->storage->filemtime($parent); + if ($mtime !== false) { + try { + $this->cache->update($parentId, ['storage_mtime' => $mtime]); + } catch (DeadlockException $e) { + // ignore the failure. + // with failures concurrent updates, someone else would have already done it. + // in the worst case the `storage_mtime` isn't updated, which should at most only trigger an extra rescan + $this->logger->warning('Error while updating parent storage_mtime, should be safe to ignore', ['exception' => $e]); + } + } + } + } +} diff --git a/lib/private/Files/Cache/Watcher.php b/lib/private/Files/Cache/Watcher.php new file mode 100644 index 00000000000..f1de5d3cfb8 --- /dev/null +++ b/lib/private/Files/Cache/Watcher.php @@ -0,0 +1,146 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache; + +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IWatcher; + +/** + * check the storage backends for updates and change the cache accordingly + */ +class Watcher implements IWatcher { + protected $watchPolicy = self::CHECK_ONCE; + + protected $checkedPaths = []; + + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + /** + * @var Cache $cache + */ + protected $cache; + + /** + * @var Scanner $scanner ; + */ + protected $scanner; + + /** @var callable[] */ + protected $onUpdate = []; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + $this->cache = $storage->getCache(); + $this->scanner = $storage->getScanner(); + } + + /** + * @param int $policy either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function setPolicy($policy) { + $this->watchPolicy = $policy; + } + + /** + * @return int either \OC\Files\Cache\Watcher::CHECK_NEVER, \OC\Files\Cache\Watcher::CHECK_ONCE, \OC\Files\Cache\Watcher::CHECK_ALWAYS + */ + public function getPolicy() { + return $this->watchPolicy; + } + + /** + * check $path for updates and update if needed + * + * @param string $path + * @param ICacheEntry|null $cachedEntry + * @return boolean true if path was updated + */ + public function checkUpdate($path, $cachedEntry = null) { + if (is_null($cachedEntry)) { + $cachedEntry = $this->cache->get($path); + } + if ($cachedEntry === false || $this->needsUpdate($path, $cachedEntry)) { + $this->update($path, $cachedEntry); + + if ($cachedEntry === false) { + return true; + } else { + // storage backends can sometimes return false positives, only return true if the scanner actually found a change + $newEntry = $this->cache->get($path); + return $newEntry->getStorageMTime() > $cachedEntry->getStorageMTime(); + } + } else { + return false; + } + } + + /** + * Update the cache for changes to $path + * + * @param string $path + * @param ICacheEntry $cachedData + */ + public function update($path, $cachedData) { + if ($this->storage->is_dir($path)) { + $this->scanner->scan($path, Scanner::SCAN_SHALLOW); + } else { + $this->scanner->scanFile($path); + } + if (is_array($cachedData) && $cachedData['mimetype'] === 'httpd/unix-directory') { + $this->cleanFolder($path); + } + if ($this->cache instanceof Cache) { + $this->cache->correctFolderSize($path); + } + foreach ($this->onUpdate as $callback) { + $callback($path); + } + } + + /** + * Check if the cache for $path needs to be updated + * + * @param string $path + * @param ICacheEntry $cachedData + * @return bool + */ + public function needsUpdate($path, $cachedData) { + if ($this->watchPolicy === self::CHECK_ALWAYS or ($this->watchPolicy === self::CHECK_ONCE and !in_array($path, $this->checkedPaths))) { + $this->checkedPaths[] = $path; + return $cachedData['storage_mtime'] === null || $this->storage->hasUpdated($path, $cachedData['storage_mtime']); + } + return false; + } + + /** + * remove deleted files in $path from the cache + * + * @param string $path + */ + public function cleanFolder($path) { + $cachedContent = $this->cache->getFolderContents($path); + foreach ($cachedContent as $entry) { + if (!$this->storage->file_exists($entry['path'])) { + $this->cache->remove($entry['path']); + } + } + } + + /** + * register a callback to be called whenever the watcher triggers and update + */ + public function onUpdate(callable $callback): void { + $this->onUpdate[] = $callback; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php new file mode 100644 index 00000000000..5bc4ee8529d --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -0,0 +1,329 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * Jail to a subdirectory of the wrapped cache + */ +class CacheJail extends CacheWrapper { + + protected string $unjailedRoot; + + public function __construct( + ?ICache $cache, + protected string $root, + ?CacheDependencies $dependencies = null, + ) { + parent::__construct($cache, $dependencies); + + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); + } + } + + /** + * @return string + */ + protected function getRoot() { + return $this->root; + } + + /** + * Get the root path with any nested jails resolved + * + * @return string + */ + public function getGetUnjailedRoot() { + return $this->unjailedRoot; + } + + /** + * @return string + */ + protected function getSourcePath(string $path) { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + /** + * @param string $path + * @param null|string $root + * @return null|string the jailed path or null if the path is outside the jail + */ + protected function getJailedPath(string $path, ?string $root = null) { + if ($root === null) { + $root = $this->getRoot(); + } + if ($root === '') { + return $path; + } + $rootLength = strlen($root) + 1; + if ($path === $root) { + return ''; + } elseif (substr($path, 0, $rootLength) === $root . '/') { + return substr($path, $rootLength); + } else { + return null; + } + } + + protected function formatCacheEntry($entry) { + if (isset($entry['path'])) { + $entry['path'] = $this->getJailedPath($entry['path']); + } + return $entry; + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file + * @return ICacheEntry|false + */ + public function get($file) { + if (is_string($file) or $file == '') { + $file = $this->getSourcePath($file); + } + return parent::get($file); + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->getCache()->insert($this->getSourcePath($file), $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->getCache()->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->getCache()->getId($this->getSourcePath($file)); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + return $this->getCache()->getParentId($this->getSourcePath($file)); + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getCache()->inCache($this->getSourcePath($file)); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->getCache()->remove($this->getSourcePath($file)); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->getCache()->move($this->getSourcePath($source), $this->getSourcePath($target)); + } + + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $this->getSourcePath($path)]; + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->getCache()->remove($this->getRoot()); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->getCache()->getStatus($this->getSourcePath($file)); + } + + /** + * update the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($this->getSourcePath($path), $data, $isBackgroundScan); + } + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($this->getSourcePath($path), $entry); + } else { + return 0; + } + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + // not supported + return []; + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiply incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + // not supported + return false; + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + $path = $this->getCache()->getPathById($id); + if ($path === null) { + return null; + } + + return $this->getJailedPath($path); + } + + /** + * Move a file or folder in the cache + * + * Note that this should make sure the entries are removed from the source cache + * + * @param \OCP\Files\Cache\ICache $sourceCache + * @param string $sourcePath + * @param string $targetPath + */ + public function moveFromCache(\OCP\Files\Cache\ICache $sourceCache, $sourcePath, $targetPath) { + if ($sourceCache === $this) { + return $this->move($sourcePath, $targetPath); + } + return $this->getCache()->moveFromCache($sourceCache, $sourcePath, $this->getSourcePath($targetPath)); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->addJailFilterQuery($this->getCache()->getQueryFilterForStorage()); + } + + protected function addJailFilterQuery(ISearchOperator $filter): ISearchOperator { + if ($this->getGetUnjailedRoot() !== '' && $this->getGetUnjailedRoot() !== '/') { + return new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, + [ + $filter, + new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, + [ + new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'path', $this->getGetUnjailedRoot()), + new SearchComparison(ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE, 'path', SearchComparison::escapeLikeParameter($this->getGetUnjailedRoot()) . '/%'), + ], + ) + ] + ); + } else { + return $filter; + } + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + if ($this->getGetUnjailedRoot() === '' || str_starts_with($rawEntry->getPath(), $this->getGetUnjailedRoot())) { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $jailedPath = $this->getJailedPath($rawEntry->getPath()); + if ($jailedPath !== null) { + return $this->formatCacheEntry(clone $rawEntry) ?: null; + } + } + } + + return null; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php new file mode 100644 index 00000000000..ff17cb79ac7 --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CachePermissionsMask.php @@ -0,0 +1,32 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +class CachePermissionsMask extends CacheWrapper { + /** + * @var int + */ + protected $mask; + + /** + * @param \OCP\Files\Cache\ICache $cache + * @param int $mask + */ + public function __construct($cache, $mask) { + parent::__construct($cache); + $this->mask = $mask; + } + + protected function formatCacheEntry($entry) { + if (isset($entry['permissions'])) { + $entry['scan_permissions'] = $entry['permissions']; + $entry['permissions'] &= $this->mask; + } + return $entry; + } +} diff --git a/lib/private/Files/Cache/Wrapper/CacheWrapper.php b/lib/private/Files/Cache/Wrapper/CacheWrapper.php new file mode 100644 index 00000000000..f2f1036d6a3 --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/CacheWrapper.php @@ -0,0 +1,315 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchQuery; +use OCP\Server; + +class CacheWrapper extends Cache { + /** + * @var ?ICache + */ + protected $cache; + + public function __construct(?ICache $cache, ?CacheDependencies $dependencies = null) { + $this->cache = $cache; + if (!$dependencies && $cache instanceof Cache) { + $this->mimetypeLoader = $cache->mimetypeLoader; + $this->connection = $cache->connection; + $this->querySearchHelper = $cache->querySearchHelper; + } else { + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); + $this->connection = $dependencies->getConnection(); + $this->querySearchHelper = $dependencies->getQuerySearchHelper(); + } + } + + protected function getCache() { + return $this->cache; + } + + protected function hasEncryptionWrapper(): bool { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->hasEncryptionWrapper(); + } else { + return false; + } + } + + /** + * Make it easy for wrappers to modify every returned cache entry + * + * @param ICacheEntry $entry + * @return ICacheEntry|false + */ + protected function formatCacheEntry($entry) { + return $entry; + } + + /** + * get the stored metadata of a file or folder + * + * @param string|int $file + * @return ICacheEntry|false + */ + public function get($file) { + $result = $this->getCache()->get($file); + if ($result instanceof ICacheEntry) { + $result = $this->formatCacheEntry($result); + } + return $result; + } + + /** + * get the metadata of all files stored in $folder + * + * @param string $folder + * @return ICacheEntry[] + */ + public function getFolderContents($folder) { + // can't do a simple $this->getCache()->.... call here since getFolderContentsById needs to be called on this + // and not the wrapped cache + $fileId = $this->getId($folder); + return $this->getFolderContentsById($fileId); + } + + /** + * get the metadata of all files stored in $folder + * + * @param int $fileId the file id of the folder + * @return array + */ + public function getFolderContentsById($fileId) { + $results = $this->getCache()->getFolderContentsById($fileId); + return array_map([$this, 'formatCacheEntry'], $results); + } + + /** + * insert or update meta data for a file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function put($file, array $data) { + if (($id = $this->getId($file)) > -1) { + $this->update($id, $data); + return $id; + } else { + return $this->insert($file, $data); + } + } + + /** + * insert meta data for a new file or folder + * + * @param string $file + * @param array $data + * + * @return int file id + * @throws \RuntimeException + */ + public function insert($file, array $data) { + return $this->getCache()->insert($file, $data); + } + + /** + * update the metadata in the cache + * + * @param int $id + * @param array $data + */ + public function update($id, array $data) { + $this->getCache()->update($id, $data); + } + + /** + * get the file id for a file + * + * @param string $file + * @return int + */ + public function getId($file) { + return $this->getCache()->getId($file); + } + + /** + * get the id of the parent folder of a file + * + * @param string $file + * @return int + */ + public function getParentId($file) { + return $this->getCache()->getParentId($file); + } + + /** + * check if a file is available in the cache + * + * @param string $file + * @return bool + */ + public function inCache($file) { + return $this->getCache()->inCache($file); + } + + /** + * remove a file or folder from the cache + * + * @param string $file + */ + public function remove($file) { + $this->getCache()->remove($file); + } + + /** + * Move a file or folder in the cache + * + * @param string $source + * @param string $target + */ + public function move($source, $target) { + $this->getCache()->move($source, $target); + } + + protected function getMoveInfo($path) { + /** @var Cache $cache */ + $cache = $this->getCache(); + return $cache->getMoveInfo($path); + } + + public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { + $this->getCache()->moveFromCache($sourceCache, $sourcePath, $targetPath); + } + + /** + * remove all entries for files that are stored on the storage from the cache + */ + public function clear() { + $this->getCache()->clear(); + } + + /** + * @param string $file + * + * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE + */ + public function getStatus($file) { + return $this->getCache()->getStatus($file); + } + + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); + } + + /** + * update the folder size and the size of all parent folders + * + * @param array|ICacheEntry|null $data (optional) meta data of the folder + */ + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + $cache->correctFolderSize($path, $data, $isBackgroundScan); + } + } + + /** + * get the size of a folder and set it in the cache + * + * @param string $path + * @param array|null|ICacheEntry $entry (optional) meta data of the folder + * @return int|float + */ + public function calculateFolderSize($path, $entry = null) { + $cache = $this->getCache(); + if ($cache instanceof Cache) { + return $cache->calculateFolderSize($path, $entry); + } else { + return 0; + } + } + + /** + * get all file ids on the files on the storage + * + * @return int[] + */ + public function getAll() { + return $this->getCache()->getAll(); + } + + /** + * find a folder in the cache which has not been fully scanned + * + * If multiple incomplete folders are in the cache, the one with the highest id will be returned, + * use the one with the highest id gives the best result with the background scanner, since that is most + * likely the folder where we stopped scanning previously + * + * @return string|false the path of the folder or false when no folder matched + */ + public function getIncomplete() { + return $this->getCache()->getIncomplete(); + } + + /** + * get the path of a file on this storage by it's id + * + * @param int $id + * @return string|null + */ + public function getPathById($id) { + return $this->getCache()->getPathById($id); + } + + /** + * Returns the numeric storage id + * + * @return int + */ + public function getNumericStorageId() { + return $this->getCache()->getNumericStorageId(); + } + + /** + * get the storage id of the storage for a file and the internal path of the file + * unlike getPathById this does not limit the search to files on this storage and + * instead does a global search in the cache table + * + * @param int $id + * @return array first element holding the storage id, second the path + */ + public static function getById($id) { + return parent::getById($id); + } + + public function getQueryFilterForStorage(): ISearchOperator { + return $this->getCache()->getQueryFilterForStorage(); + } + + public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry { + $rawEntry = $this->getCache()->getCacheEntryFromSearchResult($rawEntry); + if ($rawEntry) { + $entry = $this->formatCacheEntry(clone $rawEntry); + return $entry ?: null; + } + + return null; + } +} diff --git a/lib/private/Files/Cache/Wrapper/JailPropagator.php b/lib/private/Files/Cache/Wrapper/JailPropagator.php new file mode 100644 index 00000000000..d6409b7875e --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailPropagator.php @@ -0,0 +1,28 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Propagator; +use OC\Files\Storage\Wrapper\Jail; + +class JailPropagator extends Propagator { + /** + * @var Jail + */ + protected $storage; + + /** + * @param string $internalPath + * @param int $time + * @param int $sizeDifference + */ + public function propagateChange($internalPath, $time, $sizeDifference = 0) { + /** @var \OC\Files\Storage\Storage $storage */ + [$storage, $sourceInternalPath] = $this->storage->resolvePath($internalPath); + $storage->getPropagator()->propagateChange($sourceInternalPath, $time, $sizeDifference); + } +} diff --git a/lib/private/Files/Cache/Wrapper/JailWatcher.php b/lib/private/Files/Cache/Wrapper/JailWatcher.php new file mode 100644 index 00000000000..b1ae516654a --- /dev/null +++ b/lib/private/Files/Cache/Wrapper/JailWatcher.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Cache\Wrapper; + +use OC\Files\Cache\Watcher; + +class JailWatcher extends Watcher { + private string $root; + private Watcher $watcher; + + public function __construct(Watcher $watcher, string $root) { + $this->watcher = $watcher; + $this->root = $root; + } + + protected function getRoot(): string { + return $this->root; + } + + protected function getSourcePath($path): string { + if ($path === '') { + return $this->getRoot(); + } else { + return $this->getRoot() . '/' . ltrim($path, '/'); + } + } + + public function setPolicy($policy) { + $this->watcher->setPolicy($policy); + } + + public function getPolicy() { + return $this->watcher->getPolicy(); + } + + + public function checkUpdate($path, $cachedEntry = null) { + return $this->watcher->checkUpdate($this->getSourcePath($path), $cachedEntry); + } + + public function update($path, $cachedData) { + $this->watcher->update($this->getSourcePath($path), $cachedData); + } + + public function needsUpdate($path, $cachedData) { + return $this->watcher->needsUpdate($this->getSourcePath($path), $cachedData); + } + + public function cleanFolder($path) { + $this->watcher->cleanFolder($this->getSourcePath($path)); + } + + public function onUpdate(callable $callback): void { + $this->watcher->onUpdate($callback); + } +} diff --git a/lib/private/Files/Config/CachedMountFileInfo.php b/lib/private/Files/Config/CachedMountFileInfo.php new file mode 100644 index 00000000000..69bd4e9301e --- /dev/null +++ b/lib/private/Files/Config/CachedMountFileInfo.php @@ -0,0 +1,40 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Config; + +use OCP\Files\Config\ICachedMountFileInfo; +use OCP\IUser; + +class CachedMountFileInfo extends CachedMountInfo implements ICachedMountFileInfo { + private string $internalPath; + + public function __construct( + IUser $user, + int $storageId, + int $rootId, + string $mountPoint, + ?int $mountId, + string $mountProvider, + string $rootInternalPath, + string $internalPath, + ) { + parent::__construct($user, $storageId, $rootId, $mountPoint, $mountProvider, $mountId, $rootInternalPath); + $this->internalPath = $internalPath; + } + + public function getInternalPath(): string { + if ($this->getRootInternalPath()) { + return substr($this->internalPath, strlen($this->getRootInternalPath()) + 1); + } else { + return $this->internalPath; + } + } + + public function getPath(): string { + return $this->getMountPoint() . $this->getInternalPath(); + } +} diff --git a/lib/private/Files/Config/CachedMountInfo.php b/lib/private/Files/Config/CachedMountInfo.php new file mode 100644 index 00000000000..79dd6c6ea1d --- /dev/null +++ b/lib/private/Files/Config/CachedMountInfo.php @@ -0,0 +1,121 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Config; + +use OC\Files\Filesystem; +use OCP\Files\Config\ICachedMountInfo; +use OCP\Files\Node; +use OCP\IUser; + +class CachedMountInfo implements ICachedMountInfo { + protected IUser $user; + protected int $storageId; + protected int $rootId; + protected string $mountPoint; + protected ?int $mountId; + protected string $rootInternalPath; + protected string $mountProvider; + protected string $key; + + /** + * CachedMountInfo constructor. + * + * @param IUser $user + * @param int $storageId + * @param int $rootId + * @param string $mountPoint + * @param int|null $mountId + * @param string $rootInternalPath + */ + public function __construct( + IUser $user, + int $storageId, + int $rootId, + string $mountPoint, + string $mountProvider, + ?int $mountId = null, + string $rootInternalPath = '', + ) { + $this->user = $user; + $this->storageId = $storageId; + $this->rootId = $rootId; + $this->mountPoint = $mountPoint; + $this->mountId = $mountId; + $this->rootInternalPath = $rootInternalPath; + if (strlen($mountProvider) > 128) { + throw new \Exception("Mount provider $mountProvider name exceeds the limit of 128 characters"); + } + $this->mountProvider = $mountProvider; + $this->key = $rootId . '::' . $mountPoint; + } + + /** + * @return IUser + */ + public function getUser(): IUser { + return $this->user; + } + + /** + * @return int the numeric storage id of the mount + */ + public function getStorageId(): int { + return $this->storageId; + } + + /** + * @return int the fileid of the root of the mount + */ + public function getRootId(): int { + return $this->rootId; + } + + /** + * @return Node|null the root node of the mount + */ + public function getMountPointNode(): ?Node { + // TODO injection etc + Filesystem::initMountPoints($this->getUser()->getUID()); + $userNode = \OC::$server->getUserFolder($this->getUser()->getUID()); + return $userNode->getParent()->getFirstNodeById($this->getRootId()); + } + + /** + * @return string the mount point of the mount for the user + */ + public function getMountPoint(): string { + return $this->mountPoint; + } + + /** + * Get the id of the configured mount + * + * @return int|null mount id or null if not applicable + * @since 9.1.0 + */ + public function getMountId(): ?int { + return $this->mountId; + } + + /** + * Get the internal path (within the storage) of the root of the mount + * + * @return string + */ + public function getRootInternalPath(): string { + return $this->rootInternalPath; + } + + public function getMountProvider(): string { + return $this->mountProvider; + } + + public function getKey(): string { + return $this->key; + } +} diff --git a/lib/private/Files/Config/LazyPathCachedMountInfo.php b/lib/private/Files/Config/LazyPathCachedMountInfo.php new file mode 100644 index 00000000000..d2396109b1a --- /dev/null +++ b/lib/private/Files/Config/LazyPathCachedMountInfo.php @@ -0,0 +1,48 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Config; + +use OCP\IUser; + +class LazyPathCachedMountInfo extends CachedMountInfo { + // we don't allow \ in paths so it makes a great placeholder + private const PATH_PLACEHOLDER = '\\PLACEHOLDER\\'; + + /** @var callable(CachedMountInfo): string */ + protected $rootInternalPathCallback; + + /** + * @param IUser $user + * @param int $storageId + * @param int $rootId + * @param string $mountPoint + * @param string $mountProvider + * @param int|null $mountId + * @param callable(CachedMountInfo): string $rootInternalPathCallback + * @throws \Exception + */ + public function __construct( + IUser $user, + int $storageId, + int $rootId, + string $mountPoint, + string $mountProvider, + ?int $mountId, + callable $rootInternalPathCallback, + ) { + parent::__construct($user, $storageId, $rootId, $mountPoint, $mountProvider, $mountId, self::PATH_PLACEHOLDER); + $this->rootInternalPathCallback = $rootInternalPathCallback; + } + + public function getRootInternalPath(): string { + if ($this->rootInternalPath === self::PATH_PLACEHOLDER) { + $this->rootInternalPath = ($this->rootInternalPathCallback)($this); + } + return $this->rootInternalPath; + } +} diff --git a/lib/private/Files/Config/LazyStorageMountInfo.php b/lib/private/Files/Config/LazyStorageMountInfo.php new file mode 100644 index 00000000000..eb2c60dfa46 --- /dev/null +++ b/lib/private/Files/Config/LazyStorageMountInfo.php @@ -0,0 +1,84 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Config; + +use OCP\Files\Mount\IMountPoint; +use OCP\IUser; + +class LazyStorageMountInfo extends CachedMountInfo { + private IMountPoint $mount; + + /** + * CachedMountInfo constructor. + * + * @param IUser $user + * @param IMountPoint $mount + */ + public function __construct(IUser $user, IMountPoint $mount) { + $this->user = $user; + $this->mount = $mount; + $this->rootId = 0; + $this->storageId = 0; + $this->mountPoint = ''; + $this->key = ''; + } + + /** + * @return int the numeric storage id of the mount + */ + public function getStorageId(): int { + if (!$this->storageId) { + $this->storageId = $this->mount->getNumericStorageId(); + } + return parent::getStorageId(); + } + + /** + * @return int the fileid of the root of the mount + */ + public function getRootId(): int { + if (!$this->rootId) { + $this->rootId = $this->mount->getStorageRootId(); + } + return parent::getRootId(); + } + + /** + * @return string the mount point of the mount for the user + */ + public function getMountPoint(): string { + if (!$this->mountPoint) { + $this->mountPoint = $this->mount->getMountPoint(); + } + return parent::getMountPoint(); + } + + public function getMountId(): ?int { + return $this->mount->getMountId(); + } + + /** + * Get the internal path (within the storage) of the root of the mount + * + * @return string + */ + public function getRootInternalPath(): string { + return $this->mount->getInternalPath($this->mount->getMountPoint()); + } + + public function getMountProvider(): string { + return $this->mount->getMountProvider(); + } + + public function getKey(): string { + if (!$this->key) { + $this->key = $this->getRootId() . '::' . $this->getMountPoint(); + } + return $this->key; + } +} diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php new file mode 100644 index 00000000000..9d63184e05f --- /dev/null +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -0,0 +1,247 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Config; + +use OC\Hooks\Emitter; +use OC\Hooks\EmitterTrait; +use OCP\Diagnostics\IEventLogger; +use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Config\IMountProvider; +use OCP\Files\Config\IMountProviderCollection; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Config\IUserMountCache; +use OCP\Files\Mount\IMountManager; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; + +class MountProviderCollection implements IMountProviderCollection, Emitter { + use EmitterTrait; + + /** + * @var list<IHomeMountProvider> + */ + private array $homeProviders = []; + + /** + * @var list<IMountProvider> + */ + private array $providers = []; + + /** @var list<IRootMountProvider> */ + private array $rootProviders = []; + + /** @var list<callable> */ + private array $mountFilters = []; + + public function __construct( + private IStorageFactory $loader, + private IUserMountCache $mountCache, + private IEventLogger $eventLogger, + ) { + } + + /** + * @return list<IMountPoint> + */ + private function getMountsFromProvider(IMountProvider $provider, IUser $user, IStorageFactory $loader): array { + $class = str_replace('\\', '_', get_class($provider)); + $uid = $user->getUID(); + $this->eventLogger->start('fs:setup:provider:' . $class, "Getting mounts from $class for $uid"); + $mounts = $provider->getMountsForUser($user, $loader) ?? []; + $this->eventLogger->end('fs:setup:provider:' . $class); + return array_values($mounts); + } + + /** + * @param list<IMountProvider> $providers + * @return list<IMountPoint> + */ + private function getUserMountsForProviders(IUser $user, array $providers): array { + $loader = $this->loader; + $mounts = array_map(function (IMountProvider $provider) use ($user, $loader) { + return $this->getMountsFromProvider($provider, $user, $loader); + }, $providers); + $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) { + return array_merge($mounts, $providerMounts); + }, []); + return $this->filterMounts($user, $mounts); + } + + /** + * @return list<IMountPoint> + */ + public function getMountsForUser(IUser $user): array { + return $this->getUserMountsForProviders($user, $this->providers); + } + + /** + * @return list<IMountPoint> + */ + public function getUserMountsForProviderClasses(IUser $user, array $mountProviderClasses): array { + $providers = array_filter( + $this->providers, + fn (IMountProvider $mountProvider) => (in_array(get_class($mountProvider), $mountProviderClasses)) + ); + return $this->getUserMountsForProviders($user, $providers); + } + + /** + * @return list<IMountPoint> + */ + public function addMountForUser(IUser $user, IMountManager $mountManager, ?callable $providerFilter = null): array { + // shared mount provider gets to go last since it needs to know existing files + // to check for name collisions + $firstMounts = []; + if ($providerFilter) { + $providers = array_filter($this->providers, $providerFilter); + } else { + $providers = $this->providers; + } + $firstProviders = array_filter($providers, function (IMountProvider $provider) { + return (get_class($provider) !== 'OCA\Files_Sharing\MountProvider'); + }); + $lastProviders = array_filter($providers, function (IMountProvider $provider) { + return (get_class($provider) === 'OCA\Files_Sharing\MountProvider'); + }); + foreach ($firstProviders as $provider) { + $mounts = $this->getMountsFromProvider($provider, $user, $this->loader); + $firstMounts = array_merge($firstMounts, $mounts); + } + $firstMounts = $this->filterMounts($user, $firstMounts); + array_walk($firstMounts, [$mountManager, 'addMount']); + + $lateMounts = []; + foreach ($lastProviders as $provider) { + $mounts = $this->getMountsFromProvider($provider, $user, $this->loader); + $lateMounts = array_merge($lateMounts, $mounts); + } + + $lateMounts = $this->filterMounts($user, $lateMounts); + $this->eventLogger->start('fs:setup:add-mounts', 'Add mounts to the filesystem'); + array_walk($lateMounts, [$mountManager, 'addMount']); + $this->eventLogger->end('fs:setup:add-mounts'); + + return array_values(array_merge($lateMounts, $firstMounts)); + } + + /** + * Get the configured home mount for this user + * + * @since 9.1.0 + */ + public function getHomeMountForUser(IUser $user): IMountPoint { + $providers = array_reverse($this->homeProviders); // call the latest registered provider first to give apps an opportunity to overwrite builtin + foreach ($providers as $homeProvider) { + if ($mount = $homeProvider->getHomeMountForUser($user, $this->loader)) { + $mount->setMountPoint('/' . $user->getUID()); //make sure the mountpoint is what we expect + return $mount; + } + } + throw new \Exception('No home storage configured for user ' . $user); + } + + /** + * Add a provider for mount points + */ + public function registerProvider(IMountProvider $provider): void { + $this->providers[] = $provider; + + $this->emit('\OC\Files\Config', 'registerMountProvider', [$provider]); + } + + public function registerMountFilter(callable $filter): void { + $this->mountFilters[] = $filter; + } + + /** + * @param list<IMountPoint> $mountPoints + * @return list<IMountPoint> + */ + private function filterMounts(IUser $user, array $mountPoints): array { + return array_values(array_filter($mountPoints, function (IMountPoint $mountPoint) use ($user) { + foreach ($this->mountFilters as $filter) { + if ($filter($mountPoint, $user) === false) { + return false; + } + } + return true; + })); + } + + /** + * Add a provider for home mount points + * + * @param IHomeMountProvider $provider + * @since 9.1.0 + */ + public function registerHomeProvider(IHomeMountProvider $provider) { + $this->homeProviders[] = $provider; + $this->emit('\OC\Files\Config', 'registerHomeMountProvider', [$provider]); + } + + /** + * Get the mount cache which can be used to search for mounts without setting up the filesystem + */ + public function getMountCache(): IUserMountCache { + return $this->mountCache; + } + + public function registerRootProvider(IRootMountProvider $provider): void { + $this->rootProviders[] = $provider; + } + + /** + * Get all root mountpoints + * + * @return list<IMountPoint> + * @since 20.0.0 + */ + public function getRootMounts(): array { + $loader = $this->loader; + $mounts = array_map(function (IRootMountProvider $provider) use ($loader) { + return $provider->getRootMounts($loader); + }, $this->rootProviders); + $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) { + return array_merge($mounts, $providerMounts); + }, []); + + if (count($mounts) === 0) { + throw new \Exception('No root mounts provided by any provider'); + } + + return array_values($mounts); + } + + public function clearProviders(): void { + $this->providers = []; + $this->homeProviders = []; + $this->rootProviders = []; + } + + /** + * @return list<IMountProvider> + */ + public function getProviders(): array { + return $this->providers; + } + + /** + * @return list<IHomeMountProvider> + */ + public function getHomeProviders(): array { + return $this->homeProviders; + } + + /** + * @return list<IRootMountProvider> + */ + public function getRootProviders(): array { + return $this->rootProviders; + } +} diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php new file mode 100644 index 00000000000..3e53a67a044 --- /dev/null +++ b/lib/private/Files/Config/UserMountCache.php @@ -0,0 +1,502 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Config; + +use OC\User\LazyUser; +use OCP\Cache\CappedMemoryCache; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Diagnostics\IEventLogger; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Config\Event\UserMountAddedEvent; +use OCP\Files\Config\Event\UserMountRemovedEvent; +use OCP\Files\Config\Event\UserMountUpdatedEvent; +use OCP\Files\Config\ICachedMountFileInfo; +use OCP\Files\Config\ICachedMountInfo; +use OCP\Files\Config\IUserMountCache; +use OCP\Files\NotFoundException; +use OCP\IDBConnection; +use OCP\IUser; +use OCP\IUserManager; +use Psr\Log\LoggerInterface; + +/** + * Cache mounts points per user in the cache so we can easily look them up + */ +class UserMountCache implements IUserMountCache { + + /** + * Cached mount info. + * @var CappedMemoryCache<ICachedMountInfo[]> + **/ + private CappedMemoryCache $mountsForUsers; + /** + * fileid => internal path mapping for cached mount info. + * @var CappedMemoryCache<string> + **/ + private CappedMemoryCache $internalPathCache; + /** @var CappedMemoryCache<array> */ + private CappedMemoryCache $cacheInfoCache; + + /** + * UserMountCache constructor. + */ + public function __construct( + private IDBConnection $connection, + private IUserManager $userManager, + private LoggerInterface $logger, + private IEventLogger $eventLogger, + private IEventDispatcher $eventDispatcher, + ) { + $this->cacheInfoCache = new CappedMemoryCache(); + $this->internalPathCache = new CappedMemoryCache(); + $this->mountsForUsers = new CappedMemoryCache(); + } + + public function registerMounts(IUser $user, array $mounts, ?array $mountProviderClasses = null) { + $this->eventLogger->start('fs:setup:user:register', 'Registering mounts for user'); + /** @var array<string, ICachedMountInfo> $newMounts */ + $newMounts = []; + foreach ($mounts as $mount) { + // filter out any storages which aren't scanned yet since we aren't interested in files from those storages (yet) + if ($mount->getStorageRootId() !== -1) { + $mountInfo = new LazyStorageMountInfo($user, $mount); + $newMounts[$mountInfo->getKey()] = $mountInfo; + } + } + + $cachedMounts = $this->getMountsForUser($user); + if (is_array($mountProviderClasses)) { + $cachedMounts = array_filter($cachedMounts, function (ICachedMountInfo $mountInfo) use ($mountProviderClasses, $newMounts) { + // for existing mounts that didn't have a mount provider set + // we still want the ones that map to new mounts + if ($mountInfo->getMountProvider() === '' && isset($newMounts[$mountInfo->getKey()])) { + return true; + } + return in_array($mountInfo->getMountProvider(), $mountProviderClasses); + }); + } + + $addedMounts = []; + $removedMounts = []; + + foreach ($newMounts as $mountKey => $newMount) { + if (!isset($cachedMounts[$mountKey])) { + $addedMounts[] = $newMount; + } + } + + foreach ($cachedMounts as $mountKey => $cachedMount) { + if (!isset($newMounts[$mountKey])) { + $removedMounts[] = $cachedMount; + } + } + + $changedMounts = $this->findChangedMounts($newMounts, $cachedMounts); + + if ($addedMounts || $removedMounts || $changedMounts) { + $this->connection->beginTransaction(); + $userUID = $user->getUID(); + try { + foreach ($addedMounts as $mount) { + $this->logger->debug("Adding mount '{$mount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $mount->getMountProvider()]); + $this->addToCache($mount); + /** @psalm-suppress InvalidArgument */ + $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; + } + foreach ($removedMounts as $mount) { + $this->logger->debug("Removing mount '{$mount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $mount->getMountProvider()]); + $this->removeFromCache($mount); + unset($this->mountsForUsers[$userUID][$mount->getKey()]); + } + foreach ($changedMounts as $mountPair) { + $newMount = $mountPair[1]; + $this->logger->debug("Updating mount '{$newMount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $newMount->getMountProvider()]); + $this->updateCachedMount($newMount); + /** @psalm-suppress InvalidArgument */ + $this->mountsForUsers[$userUID][$newMount->getKey()] = $newMount; + } + $this->connection->commit(); + } catch (\Throwable $e) { + $this->connection->rollBack(); + throw $e; + } + + // Only fire events after all mounts have already been adjusted in the database. + foreach ($addedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountAddedEvent($mount)); + } + foreach ($removedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountRemovedEvent($mount)); + } + foreach ($changedMounts as $mountPair) { + $this->eventDispatcher->dispatchTyped(new UserMountUpdatedEvent($mountPair[0], $mountPair[1])); + } + } + $this->eventLogger->end('fs:setup:user:register'); + } + + /** + * @param array<string, ICachedMountInfo> $newMounts + * @param array<string, ICachedMountInfo> $cachedMounts + * @return list<list{0: ICachedMountInfo, 1: ICachedMountInfo}> Pairs of old and new mounts + */ + private function findChangedMounts(array $newMounts, array $cachedMounts): array { + $changed = []; + foreach ($cachedMounts as $key => $cachedMount) { + if (isset($newMounts[$key])) { + $newMount = $newMounts[$key]; + if ( + $newMount->getStorageId() !== $cachedMount->getStorageId() + || $newMount->getMountId() !== $cachedMount->getMountId() + || $newMount->getMountProvider() !== $cachedMount->getMountProvider() + ) { + $changed[] = [$cachedMount, $newMount]; + } + } + } + return $changed; + } + + private function addToCache(ICachedMountInfo $mount) { + if ($mount->getStorageId() !== -1) { + $this->connection->insertIfNotExist('*PREFIX*mounts', [ + 'storage_id' => $mount->getStorageId(), + 'root_id' => $mount->getRootId(), + 'user_id' => $mount->getUser()->getUID(), + 'mount_point' => $mount->getMountPoint(), + 'mount_id' => $mount->getMountId(), + 'mount_provider_class' => $mount->getMountProvider(), + ], ['root_id', 'user_id', 'mount_point']); + } else { + // in some cases this is legitimate, like orphaned shares + $this->logger->debug('Could not get storage info for mount at ' . $mount->getMountPoint()); + } + } + + private function updateCachedMount(ICachedMountInfo $mount) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->update('mounts') + ->set('storage_id', $builder->createNamedParameter($mount->getStorageId())) + ->set('mount_point', $builder->createNamedParameter($mount->getMountPoint())) + ->set('mount_id', $builder->createNamedParameter($mount->getMountId(), IQueryBuilder::PARAM_INT)) + ->set('mount_provider_class', $builder->createNamedParameter($mount->getMountProvider())) + ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($mount->getUser()->getUID()))) + ->andWhere($builder->expr()->eq('root_id', $builder->createNamedParameter($mount->getRootId(), IQueryBuilder::PARAM_INT))); + + $query->executeStatement(); + } + + private function removeFromCache(ICachedMountInfo $mount) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('mounts') + ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($mount->getUser()->getUID()))) + ->andWhere($builder->expr()->eq('root_id', $builder->createNamedParameter($mount->getRootId(), IQueryBuilder::PARAM_INT))) + ->andWhere($builder->expr()->eq('mount_point', $builder->createNamedParameter($mount->getMountPoint()))); + $query->executeStatement(); + } + + /** + * @param array $row + * @param (callable(CachedMountInfo): string)|null $pathCallback + * @return CachedMountInfo + */ + private function dbRowToMountInfo(array $row, ?callable $pathCallback = null): ICachedMountInfo { + $user = new LazyUser($row['user_id'], $this->userManager); + $mount_id = $row['mount_id']; + if (!is_null($mount_id)) { + $mount_id = (int)$mount_id; + } + if ($pathCallback) { + return new LazyPathCachedMountInfo( + $user, + (int)$row['storage_id'], + (int)$row['root_id'], + $row['mount_point'], + $row['mount_provider_class'] ?? '', + $mount_id, + $pathCallback, + ); + } else { + return new CachedMountInfo( + $user, + (int)$row['storage_id'], + (int)$row['root_id'], + $row['mount_point'], + $row['mount_provider_class'] ?? '', + $mount_id, + $row['path'] ?? '', + ); + } + } + + /** + * @param IUser $user + * @return ICachedMountInfo[] + */ + public function getMountsForUser(IUser $user) { + $userUID = $user->getUID(); + if (!$this->userManager->userExists($userUID)) { + return []; + } + if (!isset($this->mountsForUsers[$userUID])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class') + ->from('mounts', 'm') + ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userUID))); + + $result = $query->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + + /** @var array<string, ICachedMountInfo> $mounts */ + $mounts = []; + foreach ($rows as $row) { + $mount = $this->dbRowToMountInfo($row, [$this, 'getInternalPathForMountInfo']); + if ($mount !== null) { + $mounts[$mount->getKey()] = $mount; + } + } + $this->mountsForUsers[$userUID] = $mounts; + } + return $this->mountsForUsers[$userUID]; + } + + public function getInternalPathForMountInfo(CachedMountInfo $info): string { + $cached = $this->internalPathCache->get($info->getRootId()); + if ($cached !== null) { + return $cached; + } + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('path') + ->from('filecache') + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($info->getRootId()))); + return $query->executeQuery()->fetchOne() ?: ''; + } + + /** + * @param int $numericStorageId + * @param string|null $user limit the results to a single user + * @return CachedMountInfo[] + */ + public function getMountsForStorageId($numericStorageId, $user = null) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') + ->from('mounts', 'm') + ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) + ->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($numericStorageId, IQueryBuilder::PARAM_INT))); + + if ($user) { + $query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user))); + } + + $result = $query->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + + return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows)); + } + + /** + * @param int $rootFileId + * @return CachedMountInfo[] + */ + public function getMountsForRootId($rootFileId) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class') + ->from('mounts', 'm') + ->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid')) + ->where($builder->expr()->eq('root_id', $builder->createNamedParameter($rootFileId, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + + return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows)); + } + + /** + * @param $fileId + * @return array{int, string, int} + * @throws \OCP\Files\NotFoundException + */ + private function getCacheInfoFromFileId($fileId): array { + if (!isset($this->cacheInfoCache[$fileId])) { + $builder = $this->connection->getQueryBuilder(); + $query = $builder->select('storage', 'path', 'mimetype') + ->from('filecache') + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if (is_array($row)) { + $this->cacheInfoCache[$fileId] = [ + (int)$row['storage'], + (string)$row['path'], + (int)$row['mimetype'] + ]; + } else { + throw new NotFoundException('File with id "' . $fileId . '" not found'); + } + } + return $this->cacheInfoCache[$fileId]; + } + + /** + * @param int $fileId + * @param string|null $user optionally restrict the results to a single user + * @return ICachedMountFileInfo[] + * @since 9.0.0 + */ + public function getMountsForFileId($fileId, $user = null) { + try { + [$storageId, $internalPath] = $this->getCacheInfoFromFileId($fileId); + } catch (NotFoundException $e) { + return []; + } + $mountsForStorage = $this->getMountsForStorageId($storageId, $user); + + // filter mounts that are from the same storage but not a parent of the file we care about + $filteredMounts = array_filter($mountsForStorage, function (ICachedMountInfo $mount) use ($internalPath, $fileId) { + if ($fileId === $mount->getRootId()) { + return true; + } + $internalMountPath = $mount->getRootInternalPath(); + + return $internalMountPath === '' || str_starts_with($internalPath, $internalMountPath . '/'); + }); + + $filteredMounts = array_values(array_filter($filteredMounts, function (ICachedMountInfo $mount) { + return $this->userManager->userExists($mount->getUser()->getUID()); + })); + + return array_map(function (ICachedMountInfo $mount) use ($internalPath) { + return new CachedMountFileInfo( + $mount->getUser(), + $mount->getStorageId(), + $mount->getRootId(), + $mount->getMountPoint(), + $mount->getMountId(), + $mount->getMountProvider(), + $mount->getRootInternalPath(), + $internalPath + ); + }, $filteredMounts); + } + + /** + * Remove all cached mounts for a user + * + * @param IUser $user + */ + public function removeUserMounts(IUser $user) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('mounts') + ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($user->getUID()))); + $query->executeStatement(); + } + + public function removeUserStorageMount($storageId, $userId) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('mounts') + ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userId))) + ->andWhere($builder->expr()->eq('storage_id', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + $query->executeStatement(); + } + + public function remoteStorageMounts($storageId) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('mounts') + ->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); + $query->executeStatement(); + } + + /** + * @param array $users + * @return array + */ + public function getUsedSpaceForUsers(array $users) { + $builder = $this->connection->getQueryBuilder(); + + $slash = $builder->createNamedParameter('/'); + + $mountPoint = $builder->func()->concat( + $builder->func()->concat($slash, 'user_id'), + $slash + ); + + $userIds = array_map(function (IUser $user) { + return $user->getUID(); + }, $users); + + $query = $builder->select('m.user_id', 'f.size') + ->from('mounts', 'm') + ->innerJoin('m', 'filecache', 'f', + $builder->expr()->andX( + $builder->expr()->eq('m.storage_id', 'f.storage'), + $builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files'))) + )) + ->where($builder->expr()->eq('m.mount_point', $mountPoint)) + ->andWhere($builder->expr()->in('m.user_id', $builder->createNamedParameter($userIds, IQueryBuilder::PARAM_STR_ARRAY))); + + $result = $query->executeQuery(); + + $results = []; + while ($row = $result->fetch()) { + $results[$row['user_id']] = $row['size']; + } + $result->closeCursor(); + return $results; + } + + public function clear(): void { + $this->cacheInfoCache = new CappedMemoryCache(); + $this->mountsForUsers = new CappedMemoryCache(); + } + + public function getMountForPath(IUser $user, string $path): ICachedMountInfo { + $mounts = $this->getMountsForUser($user); + $mountPoints = array_map(function (ICachedMountInfo $mount) { + return $mount->getMountPoint(); + }, $mounts); + $mounts = array_combine($mountPoints, $mounts); + + $current = rtrim($path, '/'); + // walk up the directory tree until we find a path that has a mountpoint set + // the loop will return if a mountpoint is found or break if none are found + while (true) { + $mountPoint = $current . '/'; + if (isset($mounts[$mountPoint])) { + return $mounts[$mountPoint]; + } elseif ($current === '') { + break; + } + + $current = dirname($current); + if ($current === '.' || $current === '/') { + $current = ''; + } + } + + throw new NotFoundException('No cached mount for path ' . $path); + } + + public function getMountsInPath(IUser $user, string $path): array { + $path = rtrim($path, '/') . '/'; + $mounts = $this->getMountsForUser($user); + return array_filter($mounts, function (ICachedMountInfo $mount) use ($path) { + return $mount->getMountPoint() !== $path && str_starts_with($mount->getMountPoint(), $path); + }); + } +} diff --git a/lib/private/Files/Config/UserMountCacheListener.php b/lib/private/Files/Config/UserMountCacheListener.php new file mode 100644 index 00000000000..40995de8986 --- /dev/null +++ b/lib/private/Files/Config/UserMountCacheListener.php @@ -0,0 +1,34 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Config; + +use OC\User\Manager; +use OCP\Files\Config\IUserMountCache; + +/** + * Listen to hooks and update the mount cache as needed + */ +class UserMountCacheListener { + /** + * @var IUserMountCache + */ + private $userMountCache; + + /** + * UserMountCacheListener constructor. + * + * @param IUserMountCache $userMountCache + */ + public function __construct(IUserMountCache $userMountCache) { + $this->userMountCache = $userMountCache; + } + + public function listen(Manager $manager) { + $manager->listen('\OC\User', 'postDelete', [$this->userMountCache, 'removeUserMounts']); + } +} diff --git a/lib/private/Files/Conversion/ConversionManager.php b/lib/private/Files/Conversion/ConversionManager.php new file mode 100644 index 00000000000..2c98a4c6404 --- /dev/null +++ b/lib/private/Files/Conversion/ConversionManager.php @@ -0,0 +1,181 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Conversion; + +use OC\AppFramework\Bootstrap\Coordinator; +use OC\ForbiddenException; +use OC\SystemConfig; +use OCP\Files\Conversion\IConversionManager; +use OCP\Files\Conversion\IConversionProvider; +use OCP\Files\File; +use OCP\Files\GenericFileException; +use OCP\Files\IRootFolder; +use OCP\IL10N; +use OCP\ITempManager; +use OCP\L10N\IFactory; +use OCP\PreConditionNotMetException; +use Psr\Container\ContainerExceptionInterface; +use Psr\Container\ContainerInterface; +use Psr\Container\NotFoundExceptionInterface; +use Psr\Log\LoggerInterface; +use RuntimeException; +use Throwable; + +class ConversionManager implements IConversionManager { + /** @var string[] */ + private array $preferredApps = [ + 'richdocuments', + ]; + + /** @var list<IConversionProvider> */ + private array $preferredProviders = []; + + /** @var list<IConversionProvider> */ + private array $providers = []; + + private IL10N $l10n; + public function __construct( + private Coordinator $coordinator, + private ContainerInterface $serverContainer, + private IRootFolder $rootFolder, + private ITempManager $tempManager, + private LoggerInterface $logger, + private SystemConfig $config, + IFactory $l10nFactory, + ) { + $this->l10n = $l10nFactory->get('files'); + } + + public function hasProviders(): bool { + $context = $this->coordinator->getRegistrationContext(); + return !empty($context->getFileConversionProviders()); + } + + public function getProviders(): array { + $providers = []; + foreach ($this->getRegisteredProviders() as $provider) { + $providers = array_merge($providers, $provider->getSupportedMimeTypes()); + } + return $providers; + } + + public function convert(File $file, string $targetMimeType, ?string $destination = null): string { + if (!$this->hasProviders()) { + throw new PreConditionNotMetException($this->l10n->t('No file conversion providers available')); + } + + // Operate in mebibytes + $fileSize = $file->getSize() / (1024 * 1024); + $threshold = $this->config->getValue('max_file_conversion_filesize', 100); + if ($fileSize > $threshold) { + throw new GenericFileException($this->l10n->t('File is too large to convert')); + } + + $fileMimeType = $file->getMimetype(); + $validProvider = $this->getValidProvider($fileMimeType, $targetMimeType); + + if ($validProvider !== null) { + // Get the target extension given by the provider + $targetExtension = ''; + foreach ($validProvider->getSupportedMimeTypes() as $mimeProvider) { + if ($mimeProvider->getTo() === $targetMimeType) { + $targetExtension = $mimeProvider->getExtension(); + break; + } + } + // If destination not provided, we use the same path + // as the original file, but with the new extension + if ($destination === null) { + $basename = pathinfo($file->getPath(), PATHINFO_FILENAME); + $parent = $file->getParent(); + $destination = $parent->getFullPath($basename . '.' . $targetExtension); + } + + // If destination doesn't match the target extension, we throw an error + if (pathinfo($destination, PATHINFO_EXTENSION) !== $targetExtension) { + throw new GenericFileException($this->l10n->t('Destination does not match conversion extension')); + } + + // Check destination before converting + $this->checkDestination($destination); + + // Convert the file and write it to the destination + $convertedFile = $validProvider->convertFile($file, $targetMimeType); + $convertedFile = $this->writeToDestination($destination, $convertedFile); + return $convertedFile->getPath(); + } + + throw new RuntimeException($this->l10n->t('Could not convert file')); + } + + /** + * @return list<IConversionProvider> + */ + private function getRegisteredProviders(): array { + $context = $this->coordinator->getRegistrationContext(); + foreach ($context->getFileConversionProviders() as $providerRegistration) { + $class = $providerRegistration->getService(); + $appId = $providerRegistration->getAppId(); + + try { + if (in_array($appId, $this->preferredApps)) { + $this->preferredProviders[$class] = $this->serverContainer->get($class); + continue; + } + + $this->providers[$class] = $this->serverContainer->get($class); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface|Throwable $e) { + $this->logger->error('Failed to load file conversion provider ' . $class, [ + 'exception' => $e, + ]); + } + } + + return array_values(array_merge([], $this->preferredProviders, $this->providers)); + } + + private function checkDestination(string $destination): void { + if (!$this->rootFolder->nodeExists(dirname($destination))) { + throw new ForbiddenException($this->l10n->t('Destination does not exist')); + } + + $folder = $this->rootFolder->get(dirname($destination)); + if (!$folder->isCreatable()) { + throw new ForbiddenException($this->l10n->t('Destination is not creatable')); + } + } + + private function writeToDestination(string $destination, mixed $content): File { + $this->checkDestination($destination); + + if ($this->rootFolder->nodeExists($destination)) { + $file = $this->rootFolder->get($destination); + $parent = $file->getParent(); + + // Folder permissions is already checked in checkDestination method + $newName = $parent->getNonExistingName(basename($destination)); + $destination = $parent->getFullPath($newName); + } + + return $this->rootFolder->newFile($destination, $content); + } + + private function getValidProvider(string $fileMimeType, string $targetMimeType): ?IConversionProvider { + foreach ($this->getRegisteredProviders() as $provider) { + foreach ($provider->getSupportedMimeTypes() as $mimeProvider) { + if ($mimeProvider->getFrom() === $fileMimeType && $mimeProvider->getTo() === $targetMimeType) { + return $provider; + } + } + } + + return null; + } +} diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php new file mode 100644 index 00000000000..0679dc1ae72 --- /dev/null +++ b/lib/private/Files/FileInfo.php @@ -0,0 +1,396 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files; + +use OC\Files\Mount\HomeMountPoint; +use OCA\Files_Sharing\External\Mount; +use OCA\Files_Sharing\ISharedMountPoint; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Mount\IMountPoint; +use OCP\IUser; + +/** + * @template-implements \ArrayAccess<string,mixed> + */ +class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { + private array|ICacheEntry $data; + /** + * @var string + */ + private $path; + + /** + * @var \OC\Files\Storage\Storage $storage + */ + private $storage; + + /** + * @var string + */ + private $internalPath; + + /** + * @var \OCP\Files\Mount\IMountPoint + */ + private $mount; + + private ?IUser $owner; + + /** + * @var string[] + */ + private array $childEtags = []; + + /** + * @var IMountPoint[] + */ + private array $subMounts = []; + + private bool $subMountsUsed = false; + + /** + * The size of the file/folder without any sub mount + */ + private int|float $rawSize = 0; + + /** + * @param string|boolean $path + * @param Storage\Storage $storage + * @param string $internalPath + * @param array|ICacheEntry $data + * @param IMountPoint $mount + * @param ?IUser $owner + */ + public function __construct($path, $storage, $internalPath, $data, $mount, $owner = null) { + $this->path = $path; + $this->storage = $storage; + $this->internalPath = $internalPath; + $this->data = $data; + $this->mount = $mount; + $this->owner = $owner; + if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] !== 0) { + $this->rawSize = $this->data['unencrypted_size']; + } else { + $this->rawSize = $this->data['size'] ?? 0; + } + } + + public function offsetSet($offset, $value): void { + if (is_null($offset)) { + throw new \TypeError('Null offset not supported'); + } + $this->data[$offset] = $value; + } + + public function offsetExists($offset): bool { + return isset($this->data[$offset]); + } + + public function offsetUnset($offset): void { + unset($this->data[$offset]); + } + + public function offsetGet(mixed $offset): mixed { + return match ($offset) { + 'type' => $this->getType(), + 'etag' => $this->getEtag(), + 'size' => $this->getSize(), + 'mtime' => $this->getMTime(), + 'permissions' => $this->getPermissions(), + default => $this->data[$offset] ?? null, + }; + } + + /** + * @return string + */ + public function getPath() { + return $this->path; + } + + public function getStorage() { + return $this->storage; + } + + /** + * @return string + */ + public function getInternalPath() { + return $this->internalPath; + } + + /** + * Get FileInfo ID or null in case of part file + * + * @return int|null + */ + public function getId() { + return isset($this->data['fileid']) ? (int)$this->data['fileid'] : null; + } + + /** + * @return string + */ + public function getMimetype() { + return $this->data['mimetype']; + } + + /** + * @return string + */ + public function getMimePart() { + return $this->data['mimepart']; + } + + /** + * @return string + */ + public function getName() { + return empty($this->data['name']) + ? basename($this->getPath()) + : $this->data['name']; + } + + /** + * @return string + */ + public function getEtag() { + $this->updateEntryFromSubMounts(); + if (count($this->childEtags) > 0) { + $combinedEtag = $this->data['etag'] . '::' . implode('::', $this->childEtags); + return md5($combinedEtag); + } else { + return $this->data['etag']; + } + } + + /** + * @param bool $includeMounts + * @return int|float + */ + public function getSize($includeMounts = true) { + if ($includeMounts) { + $this->updateEntryFromSubMounts(); + + if ($this->isEncrypted() && isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return isset($this->data['size']) ? 0 + $this->data['size'] : 0; + } + } else { + return $this->rawSize; + } + } + + /** + * @return int + */ + public function getMTime() { + $this->updateEntryFromSubMounts(); + return (int)$this->data['mtime']; + } + + /** + * @return bool + */ + public function isEncrypted() { + return $this->data['encrypted'] ?? false; + } + + /** + * Return the current version used for the HMAC in the encryption app + */ + public function getEncryptedVersion(): int { + return isset($this->data['encryptedVersion']) ? (int)$this->data['encryptedVersion'] : 1; + } + + /** + * @return int + */ + public function getPermissions() { + return (int)$this->data['permissions']; + } + + /** + * @return string \OCP\Files\FileInfo::TYPE_FILE|\OCP\Files\FileInfo::TYPE_FOLDER + */ + public function getType() { + if (!isset($this->data['type'])) { + $this->data['type'] = ($this->getMimetype() === self::MIMETYPE_FOLDER) ? self::TYPE_FOLDER : self::TYPE_FILE; + } + return $this->data['type']; + } + + public function getData() { + return $this->data; + } + + /** + * @param int $permissions + * @return bool + */ + protected function checkPermissions($permissions) { + return ($this->getPermissions() & $permissions) === $permissions; + } + + /** + * @return bool + */ + public function isReadable() { + return $this->checkPermissions(\OCP\Constants::PERMISSION_READ); + } + + /** + * @return bool + */ + public function isUpdateable() { + return $this->checkPermissions(\OCP\Constants::PERMISSION_UPDATE); + } + + /** + * Check whether new files or folders can be created inside this folder + * + * @return bool + */ + public function isCreatable() { + return $this->checkPermissions(\OCP\Constants::PERMISSION_CREATE); + } + + /** + * @return bool + */ + public function isDeletable() { + return $this->checkPermissions(\OCP\Constants::PERMISSION_DELETE); + } + + /** + * @return bool + */ + public function isShareable() { + return $this->checkPermissions(\OCP\Constants::PERMISSION_SHARE); + } + + /** + * Check if a file or folder is shared + * + * @return bool + */ + public function isShared() { + return $this->mount instanceof ISharedMountPoint; + } + + public function isMounted() { + $isHome = $this->mount instanceof HomeMountPoint; + return !$isHome && !$this->isShared(); + } + + /** + * Get the mountpoint the file belongs to + * + * @return \OCP\Files\Mount\IMountPoint + */ + public function getMountPoint() { + return $this->mount; + } + + /** + * Get the owner of the file + * + * @return ?IUser + */ + public function getOwner() { + return $this->owner; + } + + /** + * @param IMountPoint[] $mounts + */ + public function setSubMounts(array $mounts) { + $this->subMounts = $mounts; + } + + private function updateEntryFromSubMounts(): void { + if ($this->subMountsUsed) { + return; + } + $this->subMountsUsed = true; + foreach ($this->subMounts as $mount) { + $subStorage = $mount->getStorage(); + if ($subStorage) { + $subCache = $subStorage->getCache(''); + $rootEntry = $subCache->get(''); + $this->addSubEntry($rootEntry, $mount->getMountPoint()); + } + } + } + + /** + * Add a cache entry which is the child of this folder + * + * Sets the size, etag and size to for cross-storage childs + * + * @param array|ICacheEntry $data cache entry for the child + * @param string $entryPath full path of the child entry + */ + public function addSubEntry($data, $entryPath) { + if (!$data) { + return; + } + $hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0; + if ($hasUnencryptedSize) { + $subSize = $data['unencrypted_size']; + } else { + $subSize = $data['size'] ?: 0; + } + $this->data['size'] += $subSize; + if ($hasUnencryptedSize) { + $this->data['unencrypted_size'] += $subSize; + } + if (isset($data['mtime'])) { + $this->data['mtime'] = max($this->data['mtime'], $data['mtime']); + } + if (isset($data['etag'])) { + // prefix the etag with the relative path of the subentry to propagate etag on mount moves + $relativeEntryPath = substr($entryPath, strlen($this->getPath())); + // attach the permissions to propagate etag on permission changes of submounts + $permissions = isset($data['permissions']) ? $data['permissions'] : 0; + $this->childEtags[] = $relativeEntryPath . '/' . $data['etag'] . $permissions; + } + } + + /** + * @inheritdoc + */ + public function getChecksum() { + return $this->data['checksum']; + } + + public function getExtension(): string { + return pathinfo($this->getName(), PATHINFO_EXTENSION); + } + + public function getCreationTime(): int { + return (int)$this->data['creation_time']; + } + + public function getUploadTime(): int { + return (int)$this->data['upload_time']; + } + + public function getParentId(): int { + return $this->data['parent'] ?? -1; + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? []; + } +} diff --git a/lib/private/Files/FilenameValidator.php b/lib/private/Files/FilenameValidator.php new file mode 100644 index 00000000000..a78c6d3cc3c --- /dev/null +++ b/lib/private/Files/FilenameValidator.php @@ -0,0 +1,335 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files; + +use OCP\Files\EmptyFileNameException; +use OCP\Files\FileNameTooLongException; +use OCP\Files\IFilenameValidator; +use OCP\Files\InvalidCharacterInPathException; +use OCP\Files\InvalidDirectoryException; +use OCP\Files\InvalidPathException; +use OCP\Files\ReservedWordException; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\IL10N; +use OCP\L10N\IFactory; +use Psr\Log\LoggerInterface; + +/** + * @since 30.0.0 + */ +class FilenameValidator implements IFilenameValidator { + + public const INVALID_FILE_TYPE = 100; + + private IL10N $l10n; + + /** + * @var list<string> + */ + private array $forbiddenNames = []; + + /** + * @var list<string> + */ + private array $forbiddenBasenames = []; + /** + * @var list<string> + */ + private array $forbiddenCharacters = []; + + /** + * @var list<string> + */ + private array $forbiddenExtensions = []; + + public function __construct( + IFactory $l10nFactory, + private IDBConnection $database, + private IConfig $config, + private LoggerInterface $logger, + ) { + $this->l10n = $l10nFactory->get('core'); + } + + /** + * Get a list of reserved filenames that must not be used + * This list should be checked case-insensitive, all names are returned lowercase. + * @return list<string> + * @since 30.0.0 + */ + public function getForbiddenExtensions(): array { + if (empty($this->forbiddenExtensions)) { + $forbiddenExtensions = $this->getConfigValue('forbidden_filename_extensions', ['.filepart']); + + // Always forbid .part files as they are used internally + $forbiddenExtensions[] = '.part'; + + $this->forbiddenExtensions = array_values($forbiddenExtensions); + } + return $this->forbiddenExtensions; + } + + /** + * Get a list of forbidden filename extensions that must not be used + * This list should be checked case-insensitive, all names are returned lowercase. + * @return list<string> + * @since 30.0.0 + */ + public function getForbiddenFilenames(): array { + if (empty($this->forbiddenNames)) { + $forbiddenNames = $this->getConfigValue('forbidden_filenames', ['.htaccess']); + + // Handle legacy config option + // TODO: Drop with Nextcloud 34 + $legacyForbiddenNames = $this->getConfigValue('blacklisted_files', []); + if (!empty($legacyForbiddenNames)) { + $this->logger->warning('System config option "blacklisted_files" is deprecated and will be removed in Nextcloud 34, use "forbidden_filenames" instead.'); + } + $forbiddenNames = array_merge($legacyForbiddenNames, $forbiddenNames); + + // Ensure we are having a proper string list + $this->forbiddenNames = array_values($forbiddenNames); + } + return $this->forbiddenNames; + } + + /** + * Get a list of forbidden file basenames that must not be used + * This list should be checked case-insensitive, all names are returned lowercase. + * @return list<string> + * @since 30.0.0 + */ + public function getForbiddenBasenames(): array { + if (empty($this->forbiddenBasenames)) { + $forbiddenBasenames = $this->getConfigValue('forbidden_filename_basenames', []); + // Ensure we are having a proper string list + $this->forbiddenBasenames = array_values($forbiddenBasenames); + } + return $this->forbiddenBasenames; + } + + /** + * Get a list of characters forbidden in filenames + * + * Note: Characters in the range [0-31] are always forbidden, + * even if not inside this list (see OCP\Files\Storage\IStorage::verifyPath). + * + * @return list<string> + * @since 30.0.0 + */ + public function getForbiddenCharacters(): array { + if (empty($this->forbiddenCharacters)) { + // Get always forbidden characters + $forbiddenCharacters = str_split(\OCP\Constants::FILENAME_INVALID_CHARS); + + // Get admin defined invalid characters + $additionalChars = $this->config->getSystemValue('forbidden_filename_characters', []); + if (!is_array($additionalChars)) { + $this->logger->error('Invalid system config value for "forbidden_filename_characters" is ignored.'); + $additionalChars = []; + } + $forbiddenCharacters = array_merge($forbiddenCharacters, $additionalChars); + + // Handle legacy config option + // TODO: Drop with Nextcloud 34 + $legacyForbiddenCharacters = $this->config->getSystemValue('forbidden_chars', []); + if (!is_array($legacyForbiddenCharacters)) { + $this->logger->error('Invalid system config value for "forbidden_chars" is ignored.'); + $legacyForbiddenCharacters = []; + } + if (!empty($legacyForbiddenCharacters)) { + $this->logger->warning('System config option "forbidden_chars" is deprecated and will be removed in Nextcloud 34, use "forbidden_filename_characters" instead.'); + } + $forbiddenCharacters = array_merge($legacyForbiddenCharacters, $forbiddenCharacters); + + $this->forbiddenCharacters = array_values($forbiddenCharacters); + } + return $this->forbiddenCharacters; + } + + /** + * @inheritdoc + */ + public function isFilenameValid(string $filename): bool { + try { + $this->validateFilename($filename); + } catch (\OCP\Files\InvalidPathException) { + return false; + } + return true; + } + + /** + * @inheritdoc + */ + public function validateFilename(string $filename): void { + $trimmed = trim($filename); + if ($trimmed === '') { + throw new EmptyFileNameException(); + } + + // the special directories . and .. would cause never ending recursion + // we check the trimmed name here to ensure unexpected trimming will not cause severe issues + if ($trimmed === '.' || $trimmed === '..') { + throw new InvalidDirectoryException($this->l10n->t('Dot files are not allowed')); + } + + // 255 characters is the limit on common file systems (ext/xfs) + // oc_filecache has a 250 char length limit for the filename + if (isset($filename[250])) { + throw new FileNameTooLongException(); + } + + if (!$this->database->supports4ByteText()) { + // verify database - e.g. mysql only 3-byte chars + if (preg_match('%(?: + \xF0[\x90-\xBF][\x80-\xBF]{2} # planes 1-3 + | [\xF1-\xF3][\x80-\xBF]{3} # planes 4-15 + | \xF4[\x80-\x8F][\x80-\xBF]{2} # plane 16 +)%xs', $filename)) { + throw new InvalidCharacterInPathException(); + } + } + + $this->checkForbiddenName($filename); + + $this->checkForbiddenExtension($filename); + + $this->checkForbiddenCharacters($filename); + } + + /** + * Check if the filename is forbidden + * @param string $path Path to check the filename + * @return bool True if invalid name, False otherwise + */ + public function isForbidden(string $path): bool { + // We support paths here as this function is also used in some storage internals + $filename = basename($path); + $filename = mb_strtolower($filename); + + if ($filename === '') { + return false; + } + + // Check for forbidden filenames + $forbiddenNames = $this->getForbiddenFilenames(); + if (in_array($filename, $forbiddenNames)) { + return true; + } + + // Filename is not forbidden + return false; + } + + public function sanitizeFilename(string $name, ?string $charReplacement = null): string { + $forbiddenCharacters = $this->getForbiddenCharacters(); + + if ($charReplacement === null) { + $charReplacement = array_diff(['_', '-', ' '], $forbiddenCharacters); + $charReplacement = reset($charReplacement) ?: ''; + } + if (mb_strlen($charReplacement) !== 1) { + throw new \InvalidArgumentException('No or invalid character replacement given'); + } + + $nameLowercase = mb_strtolower($name); + foreach ($this->getForbiddenExtensions() as $extension) { + if (str_ends_with($nameLowercase, $extension)) { + $name = substr($name, 0, strlen($name) - strlen($extension)); + } + } + + $basename = strlen($name) > 1 + ? substr($name, 0, strpos($name, '.', 1) ?: null) + : $name; + if (in_array(mb_strtolower($basename), $this->getForbiddenBasenames())) { + $name = str_replace($basename, $this->l10n->t('%1$s (renamed)', [$basename]), $name); + } + + if ($name === '') { + $name = $this->l10n->t('renamed file'); + } + + if (in_array(mb_strtolower($name), $this->getForbiddenFilenames())) { + $name = $this->l10n->t('%1$s (renamed)', [$name]); + } + + $name = str_replace($forbiddenCharacters, $charReplacement, $name); + return $name; + } + + protected function checkForbiddenName(string $filename): void { + $filename = mb_strtolower($filename); + if ($this->isForbidden($filename)) { + throw new ReservedWordException($this->l10n->t('"%1$s" is a forbidden file or folder name.', [$filename])); + } + + // Check for forbidden basenames - basenames are the part of the file until the first dot + // (except if the dot is the first character as this is then part of the basename "hidden files") + $basename = substr($filename, 0, strpos($filename, '.', 1) ?: null); + $forbiddenNames = $this->getForbiddenBasenames(); + if (in_array($basename, $forbiddenNames)) { + throw new ReservedWordException($this->l10n->t('"%1$s" is a forbidden prefix for file or folder names.', [$filename])); + } + } + + + /** + * Check if a filename contains any of the forbidden characters + * @param string $filename + * @throws InvalidCharacterInPathException + */ + protected function checkForbiddenCharacters(string $filename): void { + $sanitizedFileName = filter_var($filename, FILTER_UNSAFE_RAW, FILTER_FLAG_STRIP_LOW); + if ($sanitizedFileName !== $filename) { + throw new InvalidCharacterInPathException(); + } + + foreach ($this->getForbiddenCharacters() as $char) { + if (str_contains($filename, $char)) { + throw new InvalidCharacterInPathException($this->l10n->t('"%1$s" is not allowed inside a file or folder name.', [$char])); + } + } + } + + /** + * Check if a filename has a forbidden filename extension + * @param string $filename The filename to validate + * @throws InvalidPathException + */ + protected function checkForbiddenExtension(string $filename): void { + $filename = mb_strtolower($filename); + // Check for forbidden filename extensions + $forbiddenExtensions = $this->getForbiddenExtensions(); + foreach ($forbiddenExtensions as $extension) { + if (str_ends_with($filename, $extension)) { + if (str_starts_with($extension, '.')) { + throw new InvalidPathException($this->l10n->t('"%1$s" is a forbidden file type.', [$extension]), self::INVALID_FILE_TYPE); + } else { + throw new InvalidPathException($this->l10n->t('Filenames must not end with "%1$s".', [$extension])); + } + } + } + } + + /** + * Helper to get lower case list from config with validation + * @return string[] + */ + private function getConfigValue(string $key, array $fallback): array { + $values = $this->config->getSystemValue($key, $fallback); + if (!is_array($values)) { + $this->logger->error('Invalid system config value for "' . $key . '" is ignored.'); + $values = $fallback; + } + + return array_map(mb_strtolower(...), $values); + } +}; diff --git a/lib/private/Files/Filesystem.php b/lib/private/Files/Filesystem.php new file mode 100644 index 00000000000..8fe56cf060c --- /dev/null +++ b/lib/private/Files/Filesystem.php @@ -0,0 +1,740 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files; + +use OC\Files\Mount\MountPoint; +use OC\Files\Storage\StorageFactory; +use OC\User\NoUserException; +use OCP\Cache\CappedMemoryCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\Node\FilesystemTornDownEvent; +use OCP\Files\Mount\IMountManager; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; +use OCP\IUserManager; +use OCP\IUserSession; +use Psr\Log\LoggerInterface; + +class Filesystem { + private static ?Mount\Manager $mounts = null; + + public static bool $loaded = false; + + private static ?View $defaultInstance = null; + + private static ?CappedMemoryCache $normalizedPathCache = null; + + /** @var string[]|null */ + private static ?array $blacklist = null; + + /** + * classname which used for hooks handling + * used as signalclass in OC_Hooks::emit() + */ + public const CLASSNAME = 'OC_Filesystem'; + + /** + * signalname emitted before file renaming + * + * @param string $oldpath + * @param string $newpath + */ + public const signal_rename = 'rename'; + + /** + * signal emitted after file renaming + * + * @param string $oldpath + * @param string $newpath + */ + public const signal_post_rename = 'post_rename'; + + /** + * signal emitted before file/dir creation + * + * @param string $path + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_create = 'create'; + + /** + * signal emitted after file/dir creation + * + * @param string $path + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_post_create = 'post_create'; + + /** + * signal emits before file/dir copy + * + * @param string $oldpath + * @param string $newpath + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_copy = 'copy'; + + /** + * signal emits after file/dir copy + * + * @param string $oldpath + * @param string $newpath + */ + public const signal_post_copy = 'post_copy'; + + /** + * signal emits before file/dir save + * + * @param string $path + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_write = 'write'; + + /** + * signal emits after file/dir save + * + * @param string $path + */ + public const signal_post_write = 'post_write'; + + /** + * signal emitted before file/dir update + * + * @param string $path + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_update = 'update'; + + /** + * signal emitted after file/dir update + * + * @param string $path + * @param bool $run changing this flag to false in hook handler will cancel event + */ + public const signal_post_update = 'post_update'; + + /** + * signal emits when reading file/dir + * + * @param string $path + */ + public const signal_read = 'read'; + + /** + * signal emits when removing file/dir + * + * @param string $path + */ + public const signal_delete = 'delete'; + + /** + * parameters definitions for signals + */ + public const signal_param_path = 'path'; + public const signal_param_oldpath = 'oldpath'; + public const signal_param_newpath = 'newpath'; + + /** + * run - changing this flag to false in hook handler will cancel event + */ + public const signal_param_run = 'run'; + + public const signal_create_mount = 'create_mount'; + public const signal_delete_mount = 'delete_mount'; + public const signal_param_mount_type = 'mounttype'; + public const signal_param_users = 'users'; + + private static ?\OC\Files\Storage\StorageFactory $loader = null; + + private static bool $logWarningWhenAddingStorageWrapper = true; + + /** + * @param bool $shouldLog + * @return bool previous value + * @internal + */ + public static function logWarningWhenAddingStorageWrapper(bool $shouldLog): bool { + $previousValue = self::$logWarningWhenAddingStorageWrapper; + self::$logWarningWhenAddingStorageWrapper = $shouldLog; + return $previousValue; + } + + /** + * @param string $wrapperName + * @param callable $wrapper + * @param int $priority + */ + public static function addStorageWrapper($wrapperName, $wrapper, $priority = 50) { + if (self::$logWarningWhenAddingStorageWrapper) { + \OCP\Server::get(LoggerInterface::class)->warning("Storage wrapper '{wrapper}' was not registered via the 'OC_Filesystem - preSetup' hook which could cause potential problems.", [ + 'wrapper' => $wrapperName, + 'app' => 'filesystem', + ]); + } + + $mounts = self::getMountManager()->getAll(); + /** @var StorageFactory $loader */ + $loader = self::getLoader(); + if (!$loader->addStorageWrapper($wrapperName, $wrapper, $priority, $mounts)) { + // do not re-wrap if storage with this name already existed + return; + } + } + + /** + * Returns the storage factory + * + * @return IStorageFactory + */ + public static function getLoader() { + if (!self::$loader) { + self::$loader = \OC::$server->get(IStorageFactory::class); + } + return self::$loader; + } + + /** + * Returns the mount manager + */ + public static function getMountManager(): Mount\Manager { + self::initMountManager(); + assert(self::$mounts !== null); + return self::$mounts; + } + + /** + * get the mountpoint of the storage object for a path + * ( note: because a storage is not always mounted inside the fakeroot, the + * returned mountpoint is relative to the absolute root of the filesystem + * and doesn't take the chroot into account ) + * + * @param string $path + * @return string + */ + public static function getMountPoint($path) { + if (!self::$mounts) { + \OC_Util::setupFS(); + } + $mount = self::$mounts->find($path); + return $mount->getMountPoint(); + } + + /** + * get a list of all mount points in a directory + * + * @param string $path + * @return string[] + */ + public static function getMountPoints($path) { + if (!self::$mounts) { + \OC_Util::setupFS(); + } + $result = []; + $mounts = self::$mounts->findIn($path); + foreach ($mounts as $mount) { + $result[] = $mount->getMountPoint(); + } + return $result; + } + + /** + * get the storage mounted at $mountPoint + * + * @param string $mountPoint + * @return \OC\Files\Storage\Storage|null + */ + public static function getStorage($mountPoint) { + $mount = self::getMountManager()->find($mountPoint); + return $mount->getStorage(); + } + + /** + * @param string $id + * @return Mount\MountPoint[] + */ + public static function getMountByStorageId($id) { + return self::getMountManager()->findByStorageId($id); + } + + /** + * @param int $id + * @return Mount\MountPoint[] + */ + public static function getMountByNumericId($id) { + return self::getMountManager()->findByNumericId($id); + } + + /** + * resolve a path to a storage and internal path + * + * @param string $path + * @return array{?\OCP\Files\Storage\IStorage, string} an array consisting of the storage and the internal path + */ + public static function resolvePath($path): array { + $mount = self::getMountManager()->find($path); + return [$mount->getStorage(), rtrim($mount->getInternalPath($path), '/')]; + } + + public static function init(string|IUser|null $user, string $root): bool { + if (self::$defaultInstance) { + return false; + } + self::initInternal($root); + + //load custom mount config + self::initMountPoints($user); + + return true; + } + + public static function initInternal(string $root): bool { + if (self::$defaultInstance) { + return false; + } + self::getLoader(); + self::$defaultInstance = new View($root); + /** @var IEventDispatcher $eventDispatcher */ + $eventDispatcher = \OC::$server->get(IEventDispatcher::class); + $eventDispatcher->addListener(FilesystemTornDownEvent::class, function () { + self::$defaultInstance = null; + self::$loaded = false; + }); + + self::initMountManager(); + + self::$loaded = true; + + return true; + } + + public static function initMountManager(): void { + if (!self::$mounts) { + self::$mounts = \OC::$server->get(IMountManager::class); + } + } + + /** + * Initialize system and personal mount points for a user + * + * @throws \OC\User\NoUserException if the user is not available + */ + public static function initMountPoints(string|IUser|null $user = ''): void { + /** @var IUserManager $userManager */ + $userManager = \OC::$server->get(IUserManager::class); + + $userObject = ($user instanceof IUser) ? $user : $userManager->get($user); + if ($userObject) { + /** @var SetupManager $setupManager */ + $setupManager = \OC::$server->get(SetupManager::class); + $setupManager->setupForUser($userObject); + } else { + throw new NoUserException(); + } + } + + /** + * Get the default filesystem view + */ + public static function getView(): ?View { + if (!self::$defaultInstance) { + /** @var IUserSession $session */ + $session = \OC::$server->get(IUserSession::class); + $user = $session->getUser(); + if ($user) { + $userDir = '/' . $user->getUID() . '/files'; + self::initInternal($userDir); + } + } + return self::$defaultInstance; + } + + /** + * tear down the filesystem, removing all storage providers + */ + public static function tearDown() { + \OC_Util::tearDownFS(); + } + + /** + * get the relative path of the root data directory for the current user + * + * @return ?string + * + * Returns path like /admin/files + */ + public static function getRoot() { + if (!self::$defaultInstance) { + return null; + } + return self::$defaultInstance->getRoot(); + } + + /** + * mount an \OC\Files\Storage\Storage in our virtual filesystem + * + * @param \OC\Files\Storage\Storage|string $class + * @param array $arguments + * @param string $mountpoint + */ + public static function mount($class, $arguments, $mountpoint) { + if (!self::$mounts) { + \OC_Util::setupFS(); + } + $mount = new Mount\MountPoint($class, $mountpoint, $arguments, self::getLoader()); + self::$mounts->addMount($mount); + } + + /** + * return the path to a local version of the file + * we need this because we can't know if a file is stored local or not from + * outside the filestorage and for some purposes a local file is needed + */ + public static function getLocalFile(string $path): string|false { + return self::$defaultInstance->getLocalFile($path); + } + + /** + * return path to file which reflects one visible in browser + * + * @param string $path + * @return string + */ + public static function getLocalPath($path) { + $datadir = \OC_User::getHome(\OC_User::getUser()) . '/files'; + $newpath = $path; + if (strncmp($newpath, $datadir, strlen($datadir)) == 0) { + $newpath = substr($path, strlen($datadir)); + } + return $newpath; + } + + /** + * check if the requested path is valid + * + * @param string $path + * @return bool + */ + public static function isValidPath($path) { + $path = self::normalizePath($path); + if (!$path || $path[0] !== '/') { + $path = '/' . $path; + } + if (str_contains($path, '/../') || strrchr($path, '/') === '/..') { + return false; + } + return true; + } + + /** + * @param string $filename + * @return bool + */ + public static function isFileBlacklisted($filename) { + $filename = self::normalizePath($filename); + + if (self::$blacklist === null) { + self::$blacklist = \OC::$server->getConfig()->getSystemValue('blacklisted_files', ['.htaccess']); + } + + $filename = strtolower(basename($filename)); + return in_array($filename, self::$blacklist); + } + + /** + * check if the directory should be ignored when scanning + * NOTE: the special directories . and .. would cause never ending recursion + * + * @param string $dir + * @return boolean + */ + public static function isIgnoredDir($dir) { + if ($dir === '.' || $dir === '..') { + return true; + } + return false; + } + + /** + * following functions are equivalent to their php builtin equivalents for arguments/return values. + */ + public static function mkdir($path) { + return self::$defaultInstance->mkdir($path); + } + + public static function rmdir($path) { + return self::$defaultInstance->rmdir($path); + } + + public static function is_dir($path) { + return self::$defaultInstance->is_dir($path); + } + + public static function is_file($path) { + return self::$defaultInstance->is_file($path); + } + + public static function stat($path) { + return self::$defaultInstance->stat($path); + } + + public static function filetype($path) { + return self::$defaultInstance->filetype($path); + } + + public static function filesize($path) { + return self::$defaultInstance->filesize($path); + } + + public static function readfile($path) { + return self::$defaultInstance->readfile($path); + } + + public static function isCreatable($path) { + return self::$defaultInstance->isCreatable($path); + } + + public static function isReadable($path) { + return self::$defaultInstance->isReadable($path); + } + + public static function isUpdatable($path) { + return self::$defaultInstance->isUpdatable($path); + } + + public static function isDeletable($path) { + return self::$defaultInstance->isDeletable($path); + } + + public static function isSharable($path) { + return self::$defaultInstance->isSharable($path); + } + + public static function file_exists($path) { + return self::$defaultInstance->file_exists($path); + } + + public static function filemtime($path) { + return self::$defaultInstance->filemtime($path); + } + + public static function touch($path, $mtime = null) { + return self::$defaultInstance->touch($path, $mtime); + } + + /** + * @return string|false + */ + public static function file_get_contents($path) { + return self::$defaultInstance->file_get_contents($path); + } + + public static function file_put_contents($path, $data) { + return self::$defaultInstance->file_put_contents($path, $data); + } + + public static function unlink($path) { + return self::$defaultInstance->unlink($path); + } + + public static function rename($source, $target) { + return self::$defaultInstance->rename($source, $target); + } + + public static function copy($source, $target) { + return self::$defaultInstance->copy($source, $target); + } + + public static function fopen($path, $mode) { + return self::$defaultInstance->fopen($path, $mode); + } + + /** + * @param string $path + * @throws \OCP\Files\InvalidPathException + */ + public static function toTmpFile($path): string|false { + return self::$defaultInstance->toTmpFile($path); + } + + public static function fromTmpFile($tmpFile, $path) { + return self::$defaultInstance->fromTmpFile($tmpFile, $path); + } + + public static function getMimeType($path) { + return self::$defaultInstance->getMimeType($path); + } + + public static function hash($type, $path, $raw = false) { + return self::$defaultInstance->hash($type, $path, $raw); + } + + public static function free_space($path = '/') { + return self::$defaultInstance->free_space($path); + } + + public static function search($query) { + return self::$defaultInstance->search($query); + } + + /** + * @param string $query + */ + public static function searchByMime($query) { + return self::$defaultInstance->searchByMime($query); + } + + /** + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return FileInfo[] array or file info + */ + public static function searchByTag($tag, $userId) { + return self::$defaultInstance->searchByTag($tag, $userId); + } + + /** + * check if a file or folder has been updated since $time + * + * @param string $path + * @param int $time + * @return bool + */ + public static function hasUpdated($path, $time) { + return self::$defaultInstance->hasUpdated($path, $time); + } + + /** + * Fix common problems with a file path + * + * @param string $path + * @param bool $stripTrailingSlash whether to strip the trailing slash + * @param bool $isAbsolutePath whether the given path is absolute + * @param bool $keepUnicode true to disable unicode normalization + * @psalm-taint-escape file + * @return string + */ + public static function normalizePath($path, $stripTrailingSlash = true, $isAbsolutePath = false, $keepUnicode = false) { + /** + * FIXME: This is a workaround for existing classes and files which call + * this function with another type than a valid string. This + * conversion should get removed as soon as all existing + * function calls have been fixed. + */ + $path = (string)$path; + + if ($path === '') { + return '/'; + } + + if (is_null(self::$normalizedPathCache)) { + self::$normalizedPathCache = new CappedMemoryCache(2048); + } + + $cacheKey = json_encode([$path, $stripTrailingSlash, $isAbsolutePath, $keepUnicode]); + + if ($cacheKey && isset(self::$normalizedPathCache[$cacheKey])) { + return self::$normalizedPathCache[$cacheKey]; + } + + //normalize unicode if possible + if (!$keepUnicode) { + $path = \OC_Util::normalizeUnicode($path); + } + + //add leading slash, if it is already there we strip it anyway + $path = '/' . $path; + + $patterns = [ + '#\\\\#s', // no windows style '\\' slashes + '#/\.(/\.)*/#s', // remove '/./' + '#\//+#s', // remove sequence of slashes + '#/\.$#s', // remove trailing '/.' + ]; + + do { + $count = 0; + $path = preg_replace($patterns, '/', $path, -1, $count); + } while ($count > 0); + + //remove trailing slash + if ($stripTrailingSlash && strlen($path) > 1) { + $path = rtrim($path, '/'); + } + + self::$normalizedPathCache[$cacheKey] = $path; + + return $path; + } + + /** + * get the filesystem info + * + * @param string $path + * @param bool|string $includeMountPoints whether to add mountpoint sizes, + * defaults to true + * @return \OC\Files\FileInfo|false False if file does not exist + */ + public static function getFileInfo($path, $includeMountPoints = true) { + return self::getView()->getFileInfo($path, $includeMountPoints); + } + + /** + * change file metadata + * + * @param string $path + * @param array $data + * @return int + * + * returns the fileid of the updated file + */ + public static function putFileInfo($path, $data) { + return self::$defaultInstance->putFileInfo($path, $data); + } + + /** + * get the content of a directory + * + * @param string $directory path under datadirectory + * @param string $mimetype_filter limit returned content to this mimetype or mimepart + * @return \OC\Files\FileInfo[] + */ + public static function getDirectoryContent($directory, $mimetype_filter = '') { + return self::$defaultInstance->getDirectoryContent($directory, $mimetype_filter); + } + + /** + * Get the path of a file by id + * + * Note that the resulting path is not guaranteed to be unique for the id, multiple paths can point to the same file + * + * @param int $id + * @throws NotFoundException + * @return string + */ + public static function getPath($id) { + return self::$defaultInstance->getPath($id); + } + + /** + * Get the owner for a file or folder + * + * @param string $path + * @return string + */ + public static function getOwner($path) { + return self::$defaultInstance->getOwner($path); + } + + /** + * get the ETag for a file or folder + */ + public static function getETag(string $path): string|false { + return self::$defaultInstance->getETag($path); + } +} diff --git a/lib/private/Files/Lock/LockManager.php b/lib/private/Files/Lock/LockManager.php new file mode 100644 index 00000000000..978c378e506 --- /dev/null +++ b/lib/private/Files/Lock/LockManager.php @@ -0,0 +1,101 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Lock; + +use OCP\Files\Lock\ILock; +use OCP\Files\Lock\ILockManager; +use OCP\Files\Lock\ILockProvider; +use OCP\Files\Lock\LockContext; +use OCP\PreConditionNotMetException; +use Psr\Container\ContainerExceptionInterface; +use Psr\Container\NotFoundExceptionInterface; + +class LockManager implements ILockManager { + private ?string $lockProviderClass = null; + private ?ILockProvider $lockProvider = null; + private ?LockContext $lockInScope = null; + + public function registerLockProvider(ILockProvider $lockProvider): void { + if ($this->lockProvider) { + throw new PreConditionNotMetException('There is already a registered lock provider'); + } + + $this->lockProvider = $lockProvider; + } + + public function registerLazyLockProvider(string $lockProviderClass): void { + if ($this->lockProviderClass || $this->lockProvider) { + throw new PreConditionNotMetException('There is already a registered lock provider'); + } + + $this->lockProviderClass = $lockProviderClass; + } + + private function getLockProvider(): ?ILockProvider { + if ($this->lockProvider) { + return $this->lockProvider; + } + if ($this->lockProviderClass) { + try { + $this->lockProvider = \OCP\Server::get($this->lockProviderClass); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface $e) { + } + } + + return $this->lockProvider; + } + + public function isLockProviderAvailable(): bool { + return $this->getLockProvider() !== null; + } + + public function runInScope(LockContext $lock, callable $callback): void { + if (!$this->getLockProvider()) { + $callback(); + return; + } + + if ($this->lockInScope) { + throw new PreConditionNotMetException('Could not obtain lock scope as already in use by ' . $this->lockInScope); + } + + try { + $this->lockInScope = $lock; + $callback(); + } finally { + $this->lockInScope = null; + } + } + + public function getLockInScope(): ?LockContext { + return $this->lockInScope; + } + + public function getLocks(int $fileId): array { + if (!$this->getLockProvider()) { + throw new PreConditionNotMetException('No lock provider available'); + } + + return $this->getLockProvider()->getLocks($fileId); + } + + public function lock(LockContext $lockInfo): ILock { + if (!$this->getLockProvider()) { + throw new PreConditionNotMetException('No lock provider available'); + } + + return $this->getLockProvider()->lock($lockInfo); + } + + public function unlock(LockContext $lockInfo): void { + if (!$this->getLockProvider()) { + throw new PreConditionNotMetException('No lock provider available'); + } + + $this->getLockProvider()->unlock($lockInfo); + } +} diff --git a/lib/private/Files/Mount/CacheMountProvider.php b/lib/private/Files/Mount/CacheMountProvider.php new file mode 100644 index 00000000000..27c7eec9da3 --- /dev/null +++ b/lib/private/Files/Mount/CacheMountProvider.php @@ -0,0 +1,57 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OCP\Files\Config\IMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; +use OCP\IUser; + +/** + * Mount provider for custom cache storages + */ +class CacheMountProvider implements IMountProvider { + /** + * @var IConfig + */ + private $config; + + /** + * ObjectStoreHomeMountProvider constructor. + * + * @param IConfig $config + */ + public function __construct(IConfig $config) { + $this->config = $config; + } + + /** + * Get the cache mount for a user + * + * @param IUser $user + * @param IStorageFactory $loader + * @return \OCP\Files\Mount\IMountPoint[] + */ + public function getMountsForUser(IUser $user, IStorageFactory $loader) { + $cacheBaseDir = $this->config->getSystemValueString('cache_path', ''); + if ($cacheBaseDir !== '') { + $cacheDir = rtrim($cacheBaseDir, '/') . '/' . $user->getUID(); + if (!file_exists($cacheDir)) { + mkdir($cacheDir, 0770, true); + mkdir($cacheDir . '/uploads', 0770, true); + } + + return [ + new MountPoint('\OC\Files\Storage\Local', '/' . $user->getUID() . '/cache', ['datadir' => $cacheDir], $loader, null, null, self::class), + new MountPoint('\OC\Files\Storage\Local', '/' . $user->getUID() . '/uploads', ['datadir' => $cacheDir . '/uploads'], $loader, null, null, self::class) + ]; + } else { + return []; + } + } +} diff --git a/lib/private/Files/Mount/HomeMountPoint.php b/lib/private/Files/Mount/HomeMountPoint.php new file mode 100644 index 00000000000..5a648f08c89 --- /dev/null +++ b/lib/private/Files/Mount/HomeMountPoint.php @@ -0,0 +1,34 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Mount; + +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; + +class HomeMountPoint extends MountPoint { + private IUser $user; + + public function __construct( + IUser $user, + $storage, + string $mountpoint, + ?array $arguments = null, + ?IStorageFactory $loader = null, + ?array $mountOptions = null, + ?int $mountId = null, + ?string $mountProvider = null, + ) { + parent::__construct($storage, $mountpoint, $arguments, $loader, $mountOptions, $mountId, $mountProvider); + $this->user = $user; + } + + public function getUser(): IUser { + return $this->user; + } +} diff --git a/lib/private/Files/Mount/LocalHomeMountProvider.php b/lib/private/Files/Mount/LocalHomeMountProvider.php new file mode 100644 index 00000000000..a2b3d3b2a99 --- /dev/null +++ b/lib/private/Files/Mount/LocalHomeMountProvider.php @@ -0,0 +1,29 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; + +/** + * Mount provider for regular posix home folders + */ +class LocalHomeMountProvider implements IHomeMountProvider { + /** + * Get the cache mount for a user + * + * @param IUser $user + * @param IStorageFactory $loader + * @return \OCP\Files\Mount\IMountPoint|null + */ + public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { + $arguments = ['user' => $user]; + return new HomeMountPoint($user, '\OC\Files\Storage\Home', '/' . $user->getUID(), $arguments, $loader, null, null, self::class); + } +} diff --git a/lib/private/Files/Mount/Manager.php b/lib/private/Files/Mount/Manager.php new file mode 100644 index 00000000000..55de488c726 --- /dev/null +++ b/lib/private/Files/Mount/Manager.php @@ -0,0 +1,235 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OC\Files\Filesystem; +use OC\Files\SetupManager; +use OC\Files\SetupManagerFactory; +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Config\ICachedMountInfo; +use OCP\Files\Mount\IMountManager; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotFoundException; + +class Manager implements IMountManager { + /** @var MountPoint[] */ + private array $mounts = []; + /** @var CappedMemoryCache<IMountPoint> */ + private CappedMemoryCache $pathCache; + /** @var CappedMemoryCache<IMountPoint[]> */ + private CappedMemoryCache $inPathCache; + private SetupManager $setupManager; + + public function __construct(SetupManagerFactory $setupManagerFactory) { + $this->pathCache = new CappedMemoryCache(); + $this->inPathCache = new CappedMemoryCache(); + $this->setupManager = $setupManagerFactory->create($this); + } + + /** + * @param IMountPoint $mount + */ + public function addMount(IMountPoint $mount) { + $this->mounts[$mount->getMountPoint()] = $mount; + $this->pathCache->clear(); + $this->inPathCache->clear(); + } + + /** + * @param string $mountPoint + */ + public function removeMount(string $mountPoint) { + $mountPoint = Filesystem::normalizePath($mountPoint); + if (\strlen($mountPoint) > 1) { + $mountPoint .= '/'; + } + unset($this->mounts[$mountPoint]); + $this->pathCache->clear(); + $this->inPathCache->clear(); + } + + /** + * @param string $mountPoint + * @param string $target + */ + public function moveMount(string $mountPoint, string $target) { + $this->mounts[$target] = $this->mounts[$mountPoint]; + unset($this->mounts[$mountPoint]); + $this->pathCache->clear(); + $this->inPathCache->clear(); + } + + /** + * Find the mount for $path + * + * @param string $path + * @return IMountPoint + */ + public function find(string $path): IMountPoint { + $this->setupManager->setupForPath($path); + $path = Filesystem::normalizePath($path); + + if (isset($this->pathCache[$path])) { + return $this->pathCache[$path]; + } + + + + if (count($this->mounts) === 0) { + $this->setupManager->setupRoot(); + if (count($this->mounts) === 0) { + throw new \Exception('No mounts even after explicitly setting up the root mounts'); + } + } + + $current = $path; + while (true) { + $mountPoint = $current . '/'; + if (isset($this->mounts[$mountPoint])) { + $this->pathCache[$path] = $this->mounts[$mountPoint]; + return $this->mounts[$mountPoint]; + } elseif ($current === '') { + break; + } + + $current = dirname($current); + if ($current === '.' || $current === '/') { + $current = ''; + } + } + + throw new NotFoundException('No mount for path ' . $path . ' existing mounts (' . count($this->mounts) . '): ' . implode(',', array_keys($this->mounts))); + } + + /** + * Find all mounts in $path + * + * @param string $path + * @return IMountPoint[] + */ + public function findIn(string $path): array { + $this->setupManager->setupForPath($path, true); + $path = $this->formatPath($path); + + if (isset($this->inPathCache[$path])) { + return $this->inPathCache[$path]; + } + + $result = []; + $pathLength = \strlen($path); + $mountPoints = array_keys($this->mounts); + foreach ($mountPoints as $mountPoint) { + if (substr($mountPoint, 0, $pathLength) === $path && \strlen($mountPoint) > $pathLength) { + $result[] = $this->mounts[$mountPoint]; + } + } + + $this->inPathCache[$path] = $result; + return $result; + } + + public function clear() { + $this->mounts = []; + $this->pathCache->clear(); + $this->inPathCache->clear(); + } + + /** + * Find mounts by storage id + * + * @param string $id + * @return IMountPoint[] + */ + public function findByStorageId(string $id): array { + if (\strlen($id) > 64) { + $id = md5($id); + } + $result = []; + foreach ($this->mounts as $mount) { + if ($mount->getStorageId() === $id) { + $result[] = $mount; + } + } + return $result; + } + + /** + * @return IMountPoint[] + */ + public function getAll(): array { + return $this->mounts; + } + + /** + * Find mounts by numeric storage id + * + * @param int $id + * @return IMountPoint[] + */ + public function findByNumericId(int $id): array { + $result = []; + foreach ($this->mounts as $mount) { + if ($mount->getNumericStorageId() === $id) { + $result[] = $mount; + } + } + return $result; + } + + /** + * @param string $path + * @return string + */ + private function formatPath(string $path): string { + $path = Filesystem::normalizePath($path); + if (\strlen($path) > 1) { + $path .= '/'; + } + return $path; + } + + public function getSetupManager(): SetupManager { + return $this->setupManager; + } + + /** + * Return all mounts in a path from a specific mount provider + * + * @param string $path + * @param string[] $mountProviders + * @return MountPoint[] + */ + public function getMountsByMountProvider(string $path, array $mountProviders) { + $this->getSetupManager()->setupForProvider($path, $mountProviders); + if (in_array('', $mountProviders)) { + return $this->mounts; + } else { + return array_filter($this->mounts, function ($mount) use ($mountProviders) { + return in_array($mount->getMountProvider(), $mountProviders); + }); + } + } + + /** + * Return the mount matching a cached mount info (or mount file info) + * + * @param ICachedMountInfo $info + * + * @return IMountPoint|null + */ + public function getMountFromMountInfo(ICachedMountInfo $info): ?IMountPoint { + $this->setupManager->setupForPath($info->getMountPoint()); + foreach ($this->mounts as $mount) { + if ($mount->getMountPoint() === $info->getMountPoint()) { + return $mount; + } + } + return null; + } +} diff --git a/lib/private/Files/Mount/MountPoint.php b/lib/private/Files/Mount/MountPoint.php new file mode 100644 index 00000000000..bab2dc8e4bd --- /dev/null +++ b/lib/private/Files/Mount/MountPoint.php @@ -0,0 +1,293 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OC\Files\Filesystem; +use OC\Files\Storage\Storage; +use OC\Files\Storage\StorageFactory; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Storage\IStorageFactory; +use Psr\Log\LoggerInterface; + +class MountPoint implements IMountPoint { + /** + * @var \OC\Files\Storage\Storage|null $storage + */ + protected $storage = null; + protected $class; + protected $storageId; + protected $numericStorageId = null; + protected $rootId = null; + + /** + * Configuration options for the storage backend + * + * @var array + */ + protected $arguments = []; + protected $mountPoint; + + /** + * Mount specific options + * + * @var array + */ + protected $mountOptions = []; + + /** + * @var \OC\Files\Storage\StorageFactory $loader + */ + private $loader; + + /** + * Specified whether the storage is invalid after failing to + * instantiate it. + * + * @var bool + */ + private $invalidStorage = false; + + /** @var int|null */ + protected $mountId; + + /** @var string */ + protected $mountProvider; + + /** + * @param string|\OC\Files\Storage\Storage $storage + * @param string $mountpoint + * @param array $arguments (optional) configuration for the storage backend + * @param \OCP\Files\Storage\IStorageFactory $loader + * @param array $mountOptions mount specific options + * @param int|null $mountId + * @param string|null $mountProvider + * @throws \Exception + */ + public function __construct( + $storage, + string $mountpoint, + ?array $arguments = null, + ?IStorageFactory $loader = null, + ?array $mountOptions = null, + ?int $mountId = null, + ?string $mountProvider = null, + ) { + if (is_null($arguments)) { + $arguments = []; + } + if (is_null($loader)) { + $this->loader = new StorageFactory(); + } else { + $this->loader = $loader; + } + + if (!is_null($mountOptions)) { + $this->mountOptions = $mountOptions; + } + + $mountpoint = $this->formatPath($mountpoint); + $this->mountPoint = $mountpoint; + $this->mountId = $mountId; + if ($storage instanceof Storage) { + $this->class = get_class($storage); + $this->storage = $this->loader->wrap($this, $storage); + } else { + // Update old classes to new namespace + if (str_contains($storage, 'OC_Filestorage_')) { + $storage = '\OC\Files\Storage\\' . substr($storage, 15); + } + $this->class = $storage; + $this->arguments = $arguments; + } + if ($mountProvider) { + if (strlen($mountProvider) > 128) { + throw new \Exception("Mount provider $mountProvider name exceeds the limit of 128 characters"); + } + } + $this->mountProvider = $mountProvider ?? ''; + } + + /** + * get complete path to the mount point, relative to data/ + * + * @return string + */ + public function getMountPoint() { + return $this->mountPoint; + } + + /** + * Sets the mount point path, relative to data/ + * + * @param string $mountPoint new mount point + */ + public function setMountPoint($mountPoint) { + $this->mountPoint = $this->formatPath($mountPoint); + } + + /** + * create the storage that is mounted + */ + private function createStorage() { + if ($this->invalidStorage) { + return; + } + + if (class_exists($this->class)) { + try { + $class = $this->class; + // prevent recursion by setting the storage before applying wrappers + $this->storage = new $class($this->arguments); + $this->storage = $this->loader->wrap($this, $this->storage); + } catch (\Exception $exception) { + $this->storage = null; + $this->invalidStorage = true; + if ($this->mountPoint === '/') { + // the root storage could not be initialized, show the user! + throw new \Exception('The root storage could not be initialized. Please contact your local administrator.', $exception->getCode(), $exception); + } else { + \OC::$server->get(LoggerInterface::class)->error($exception->getMessage(), ['exception' => $exception]); + } + return; + } + } else { + \OC::$server->get(LoggerInterface::class)->error('Storage backend ' . $this->class . ' not found', ['app' => 'core']); + $this->invalidStorage = true; + return; + } + } + + /** + * @return \OC\Files\Storage\Storage|null + */ + public function getStorage() { + if (is_null($this->storage)) { + $this->createStorage(); + } + return $this->storage; + } + + /** + * @return string|null + */ + public function getStorageId() { + if (!$this->storageId) { + $storage = $this->getStorage(); + if (is_null($storage)) { + return null; + } + $this->storageId = $storage->getId(); + if (strlen($this->storageId) > 64) { + $this->storageId = md5($this->storageId); + } + } + return $this->storageId; + } + + /** + * @return int + */ + public function getNumericStorageId() { + if (is_null($this->numericStorageId)) { + $storage = $this->getStorage(); + if (is_null($storage)) { + return -1; + } + $this->numericStorageId = $storage->getCache()->getNumericStorageId(); + } + return $this->numericStorageId; + } + + /** + * @param string $path + * @return string + */ + public function getInternalPath($path) { + $path = Filesystem::normalizePath($path, true, false, true); + if ($this->mountPoint === $path or $this->mountPoint . '/' === $path) { + $internalPath = ''; + } else { + $internalPath = substr($path, strlen($this->mountPoint)); + } + // substr returns false instead of an empty string, we always want a string + return (string)$internalPath; + } + + /** + * @param string $path + * @return string + */ + private function formatPath($path) { + $path = Filesystem::normalizePath($path); + if (strlen($path) > 1) { + $path .= '/'; + } + return $path; + } + + /** + * @param callable $wrapper + */ + public function wrapStorage($wrapper) { + $storage = $this->getStorage(); + // storage can be null if it couldn't be initialized + if ($storage != null) { + $this->storage = $wrapper($this->mountPoint, $storage, $this); + } + } + + /** + * Get a mount option + * + * @param string $name Name of the mount option to get + * @param mixed $default Default value for the mount option + * @return mixed + */ + public function getOption($name, $default) { + return $this->mountOptions[$name] ?? $default; + } + + /** + * Get all options for the mount + * + * @return array + */ + public function getOptions() { + return $this->mountOptions; + } + + /** + * Get the file id of the root of the storage + * + * @return int + */ + public function getStorageRootId() { + if (is_null($this->rootId) || $this->rootId === -1) { + $storage = $this->getStorage(); + // if we can't create the storage return -1 as root id, this is then handled the same as if the root isn't scanned yet + if ($storage === null) { + $this->rootId = -1; + } else { + $this->rootId = (int)$storage->getCache()->getId(''); + } + } + return $this->rootId; + } + + public function getMountId() { + return $this->mountId; + } + + public function getMountType() { + return ''; + } + + public function getMountProvider(): string { + return $this->mountProvider; + } +} diff --git a/lib/private/Files/Mount/MoveableMount.php b/lib/private/Files/Mount/MoveableMount.php new file mode 100644 index 00000000000..755733bf651 --- /dev/null +++ b/lib/private/Files/Mount/MoveableMount.php @@ -0,0 +1,30 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OCP\Files\Mount\IMovableMount; + +/** + * Defines the mount point to be (re)moved by the user + */ +interface MoveableMount extends IMovableMount { + /** + * Move the mount point to $target + * + * @param string $target the target mount point + * @return bool + */ + public function moveMount($target); + + /** + * Remove the mount points + * + * @return bool + */ + public function removeMount(); +} diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php new file mode 100644 index 00000000000..4b088f2c808 --- /dev/null +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -0,0 +1,45 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Mount; + +use OC\Files\ObjectStore\HomeObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; +use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Storage\IStorageFactory; +use OCP\IUser; + +/** + * Mount provider for object store home storages + */ +class ObjectHomeMountProvider implements IHomeMountProvider { + public function __construct( + private PrimaryObjectStoreConfig $objectStoreConfig, + ) { + } + + /** + * Get the home mount for a user + * + * @param IUser $user + * @param IStorageFactory $loader + * @return ?IMountPoint + */ + public function getHomeMountForUser(IUser $user, IStorageFactory $loader): ?IMountPoint { + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForUser($user); + if ($objectStoreConfig === null) { + return null; + } + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + 'user' => $user, + ]); + + return new HomeMountPoint($user, HomeObjectStoreStorage::class, '/' . $user->getUID(), $arguments, $loader, null, null, self::class); + } +} diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php new file mode 100644 index 00000000000..1546ef98f50 --- /dev/null +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -0,0 +1,138 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Mount; + +use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\Storage\Wrapper\Jail; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; +use Psr\Log\LoggerInterface; + +/** + * Mount provider for object store app data folder for previews + */ +class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { + private LoggerInterface $logger; + /** @var IConfig */ + private $config; + + public function __construct(LoggerInterface $logger, IConfig $config) { + $this->logger = $logger; + $this->config = $config; + } + + /** + * @return MountPoint[] + * @throws \Exception + */ + public function getRootMounts(IStorageFactory $loader): array { + if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { + return []; + } + if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { + return []; + } + + $instanceId = $this->config->getSystemValueString('instanceid', ''); + $mountPoints = []; + $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + $i = 0; + foreach ($directoryRange as $parent) { + foreach ($directoryRange as $child) { + $mountPoints[] = new MountPoint( + AppdataPreviewObjectStoreStorage::class, + '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, + $this->getMultiBucketObjectStore($i), + $loader, + null, + null, + self::class + ); + $i++; + } + } + + $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); + $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); + $fakeRootStorageJail = new Jail([ + 'storage' => $fakeRootStorage, + 'root' => '/appdata_' . $instanceId . '/preview', + ]); + + // add a fallback location to be able to fetch existing previews from the old bucket + $mountPoints[] = new MountPoint( + $fakeRootStorageJail, + '/appdata_' . $instanceId . '/preview/old-multibucket', + null, + $loader, + null, + null, + self::class + ); + + return $mountPoints; + } + + protected function getMultiBucketObjectStore(int $number): array { + $config = $this->config->getSystemValue('objectstore_multibucket'); + + // sanity checks + if (empty($config['class'])) { + $this->logger->error('No class given for objectstore', ['app' => 'files']); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + $config['arguments']['bucket'] .= "-preview-$number"; + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + $config['arguments']['internal-id'] = $number; + + return $config['arguments']; + } + + protected function getMultiBucketObjectStoreForRoot(): array { + $config = $this->config->getSystemValue('objectstore_multibucket'); + + // sanity checks + if (empty($config['class'])) { + $this->logger->error('No class given for objectstore', ['app' => 'files']); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $config['arguments']['bucket'] .= '0'; + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + return $config['arguments']; + } +} diff --git a/lib/private/Files/Mount/RootMountProvider.php b/lib/private/Files/Mount/RootMountProvider.php new file mode 100644 index 00000000000..5e0c924ad38 --- /dev/null +++ b/lib/private/Files/Mount/RootMountProvider.php @@ -0,0 +1,50 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Mount; + +use OC; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; +use OC\Files\Storage\LocalRootStorage; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; + +class RootMountProvider implements IRootMountProvider { + private PrimaryObjectStoreConfig $objectStoreConfig; + private IConfig $config; + + public function __construct(PrimaryObjectStoreConfig $objectStoreConfig, IConfig $config) { + $this->objectStoreConfig = $objectStoreConfig; + $this->config = $config; + } + + public function getRootMounts(IStorageFactory $loader): array { + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + + if ($objectStoreConfig) { + return [$this->getObjectStoreRootMount($loader, $objectStoreConfig)]; + } else { + return [$this->getLocalRootMount($loader)]; + } + } + + private function getLocalRootMount(IStorageFactory $loader): MountPoint { + $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); + return new MountPoint(LocalRootStorage::class, '/', ['datadir' => $configDataDirectory], $loader, null, null, self::class); + } + + private function getObjectStoreRootMount(IStorageFactory $loader, array $objectStoreConfig): MountPoint { + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + ]); + + return new MountPoint(ObjectStoreStorage::class, '/', $arguments, $loader, null, null, self::class); + } +} diff --git a/lib/private/Files/Node/File.php b/lib/private/Files/Node/File.php new file mode 100644 index 00000000000..eb6411d7d13 --- /dev/null +++ b/lib/private/Files/Node/File.php @@ -0,0 +1,140 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OCP\Files\GenericFileException; +use OCP\Files\NotPermittedException; +use OCP\Lock\LockedException; + +class File extends Node implements \OCP\Files\File { + /** + * Creates a Folder that represents a non-existing path + * + * @param string $path path + * @return NonExistingFile non-existing node + */ + protected function createNonExistingNode($path) { + return new NonExistingFile($this->root, $this->view, $path); + } + + /** + * @return string + * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException + */ + public function getContent() { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_READ)) { + $content = $this->view->file_get_contents($this->path); + if ($content === false) { + throw new GenericFileException(); + } + return $content; + } else { + throw new NotPermittedException(); + } + } + + /** + * @param string|resource $data + * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException + */ + public function putContent($data) { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_UPDATE)) { + $this->sendHooks(['preWrite']); + if ($this->view->file_put_contents($this->path, $data) === false) { + throw new GenericFileException('file_put_contents failed'); + } + $this->fileInfo = null; + $this->sendHooks(['postWrite']); + } else { + throw new NotPermittedException(); + } + } + + /** + * @param string $mode + * @return resource|false + * @throws NotPermittedException + * @throws LockedException + */ + public function fopen($mode) { + $preHooks = []; + $postHooks = []; + $requiredPermissions = \OCP\Constants::PERMISSION_READ; + switch ($mode) { + case 'r+': + case 'rb+': + case 'w+': + case 'wb+': + case 'x+': + case 'xb+': + case 'a+': + case 'ab+': + case 'w': + case 'wb': + case 'x': + case 'xb': + case 'a': + case 'ab': + $preHooks[] = 'preWrite'; + $postHooks[] = 'postWrite'; + $requiredPermissions |= \OCP\Constants::PERMISSION_UPDATE; + break; + } + + if ($this->checkPermissions($requiredPermissions)) { + $this->sendHooks($preHooks); + $result = $this->view->fopen($this->path, $mode); + $this->sendHooks($postHooks); + return $result; + } else { + throw new NotPermittedException(); + } + } + + /** + * @throws NotPermittedException + * @throws \OCP\Files\InvalidPathException + * @throws \OCP\Files\NotFoundException + */ + public function delete() { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_DELETE)) { + $this->sendHooks(['preDelete']); + $fileInfo = $this->getFileInfo(); + $this->view->unlink($this->path); + $nonExisting = new NonExistingFile($this->root, $this->view, $this->path, $fileInfo); + $this->sendHooks(['postDelete'], [$nonExisting]); + $this->fileInfo = null; + } else { + throw new NotPermittedException(); + } + } + + /** + * @param string $type + * @param bool $raw + * @return string + */ + public function hash($type, $raw = false) { + return $this->view->hash($type, $this->path, $raw); + } + + /** + * @inheritdoc + */ + public function getChecksum() { + return $this->getFileInfo()->getChecksum(); + } + + public function getExtension(): string { + return $this->getFileInfo()->getExtension(); + } +} diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php new file mode 100644 index 00000000000..7453b553119 --- /dev/null +++ b/lib/private/Files/Node/Folder.php @@ -0,0 +1,472 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OC\Files\Cache\QuerySearchHelper; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchOrder; +use OC\Files\Search\SearchQuery; +use OC\Files\Utils\PathHelper; +use OC\User\LazyUser; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\FileInfo; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Node as INode; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; +use OCP\Files\Search\ISearchQuery; +use OCP\IUserManager; + +class Folder extends Node implements \OCP\Files\Folder { + + private ?IUserManager $userManager = null; + + /** + * Creates a Folder that represents a non-existing path + * + * @param string $path path + * @return NonExistingFolder non-existing node + */ + protected function createNonExistingNode($path) { + return new NonExistingFolder($this->root, $this->view, $path); + } + + /** + * @param string $path path relative to the folder + * @return string + * @throws \OCP\Files\NotPermittedException + */ + public function getFullPath($path) { + $path = $this->normalizePath($path); + if (!$this->isValidPath($path)) { + throw new NotPermittedException('Invalid path "' . $path . '"'); + } + return $this->path . $path; + } + + /** + * @param string $path + * @return string|null + */ + public function getRelativePath($path) { + return PathHelper::getRelativePath($this->getPath(), $path); + } + + /** + * check if a node is a (grand-)child of the folder + * + * @param \OC\Files\Node\Node $node + * @return bool + */ + public function isSubNode($node) { + return str_starts_with($node->getPath(), $this->path . '/'); + } + + /** + * get the content of this directory + * + * @return Node[] + * @throws \OCP\Files\NotFoundException + */ + public function getDirectoryListing() { + $folderContent = $this->view->getDirectoryContent($this->path, '', $this->getFileInfo(false)); + + return array_map(function (FileInfo $info) { + if ($info->getMimetype() === FileInfo::MIMETYPE_FOLDER) { + return new Folder($this->root, $this->view, $info->getPath(), $info, $this); + } else { + return new File($this->root, $this->view, $info->getPath(), $info, $this); + } + }, $folderContent); + } + + protected function createNode(string $path, ?FileInfo $info = null, bool $infoHasSubMountsIncluded = true): INode { + if (is_null($info)) { + $isDir = $this->view->is_dir($path); + } else { + $isDir = $info->getType() === FileInfo::TYPE_FOLDER; + } + $parent = dirname($path) === $this->getPath() ? $this : null; + if ($isDir) { + return new Folder($this->root, $this->view, $path, $info, $parent, $infoHasSubMountsIncluded); + } else { + return new File($this->root, $this->view, $path, $info, $parent); + } + } + + public function get($path) { + return $this->root->get($this->getFullPath($path)); + } + + public function nodeExists($path) { + try { + $this->get($path); + return true; + } catch (NotFoundException|NotPermittedException) { + return false; + } + } + + /** + * @param string $path + * @return \OC\Files\Node\Folder + * @throws \OCP\Files\NotPermittedException + */ + public function newFolder($path) { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_CREATE)) { + $fullPath = $this->getFullPath($path); + $nonExisting = new NonExistingFolder($this->root, $this->view, $fullPath); + $this->sendHooks(['preWrite', 'preCreate'], [$nonExisting]); + if (!$this->view->mkdir($fullPath)) { + // maybe another concurrent process created the folder already + if (!$this->view->is_dir($fullPath)) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '"'); + } else { + // we need to ensure we don't return before the concurrent request has finished updating the cache + $tries = 5; + while (!$this->view->getFileInfo($fullPath)) { + if ($tries < 1) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '", folder exists but unable to get cache entry'); + } + usleep(5 * 1000); + $tries--; + } + } + } + $parent = dirname($fullPath) === $this->getPath() ? $this : null; + $node = new Folder($this->root, $this->view, $fullPath, null, $parent); + $this->sendHooks(['postWrite', 'postCreate'], [$node]); + return $node; + } else { + throw new NotPermittedException('No create permission for folder "' . $path . '"'); + } + } + + /** + * @param string $path + * @param string | resource | null $content + * @return \OC\Files\Node\File + * @throws \OCP\Files\NotPermittedException + */ + public function newFile($path, $content = null) { + if ($path === '') { + throw new NotPermittedException('Could not create as provided path is empty'); + } + if ($this->checkPermissions(\OCP\Constants::PERMISSION_CREATE)) { + $fullPath = $this->getFullPath($path); + $nonExisting = new NonExistingFile($this->root, $this->view, $fullPath); + $this->sendHooks(['preWrite', 'preCreate'], [$nonExisting]); + if ($content !== null) { + $result = $this->view->file_put_contents($fullPath, $content); + } else { + $result = $this->view->touch($fullPath); + } + if ($result === false) { + throw new NotPermittedException('Could not create path "' . $fullPath . '"'); + } + $node = new File($this->root, $this->view, $fullPath, null, $this); + $this->sendHooks(['postWrite', 'postCreate'], [$node]); + return $node; + } + throw new NotPermittedException('No create permission for path "' . $path . '"'); + } + + private function queryFromOperator(ISearchOperator $operator, ?string $uid = null, int $limit = 0, int $offset = 0): ISearchQuery { + if ($uid === null) { + $user = null; + } else { + /** @var IUserManager $userManager */ + $userManager = \OCP\Server::get(IUserManager::class); + $user = $userManager->get($uid); + } + return new SearchQuery($operator, $limit, $offset, [], $user); + } + + /** + * search for files with the name matching $query + * + * @param string|ISearchQuery $query + * @return \OC\Files\Node\Node[] + */ + public function search($query) { + if (is_string($query)) { + $query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%' . $query . '%')); + } + + // search is handled by a single query covering all caches that this folder contains + // this is done by collect + + $limitToHome = $query->limitToHome(); + if ($limitToHome && count(explode('/', $this->path)) !== 3) { + throw new \InvalidArgumentException('searching by owner is only allowed in the users home folder'); + } + + /** @var QuerySearchHelper $searchHelper */ + $searchHelper = \OC::$server->get(QuerySearchHelper::class); + [$caches, $mountByMountPoint] = $searchHelper->getCachesAndMountPointsForSearch($this->root, $this->path, $limitToHome); + $resultsPerCache = $searchHelper->searchInCaches($query, $caches); + + // loop through all results per-cache, constructing the FileInfo object from the CacheEntry and merge them all + $files = array_merge(...array_map(function (array $results, string $relativeMountPoint) use ($mountByMountPoint) { + $mount = $mountByMountPoint[$relativeMountPoint]; + return array_map(function (ICacheEntry $result) use ($relativeMountPoint, $mount) { + return $this->cacheEntryToFileInfo($mount, $relativeMountPoint, $result); + }, $results); + }, array_values($resultsPerCache), array_keys($resultsPerCache))); + + // don't include this folder in the results + $files = array_values(array_filter($files, function (FileInfo $file) { + return $file->getPath() !== $this->getPath(); + })); + + // since results were returned per-cache, they are no longer fully sorted + $order = $query->getOrder(); + if ($order) { + usort($files, function (FileInfo $a, FileInfo $b) use ($order) { + foreach ($order as $orderField) { + $cmp = $orderField->sortFileInfo($a, $b); + if ($cmp !== 0) { + return $cmp; + } + } + return 0; + }); + } + + return array_map(function (FileInfo $file) { + return $this->createNode($file->getPath(), $file); + }, $files); + } + + private function cacheEntryToFileInfo(IMountPoint $mount, string $appendRoot, ICacheEntry $cacheEntry): FileInfo { + $cacheEntry['internalPath'] = $cacheEntry['path']; + $cacheEntry['path'] = rtrim($appendRoot . $cacheEntry->getPath(), '/'); + $subPath = $cacheEntry['path'] !== '' ? '/' . $cacheEntry['path'] : ''; + $storage = $mount->getStorage(); + + $owner = null; + $ownerId = $storage->getOwner($cacheEntry['internalPath']); + if ($ownerId !== false) { + // Cache the user manager (for performance) + if ($this->userManager === null) { + $this->userManager = \OCP\Server::get(IUserManager::class); + } + $owner = new LazyUser($ownerId, $this->userManager); + } + + return new \OC\Files\FileInfo( + $this->path . $subPath, + $storage, + $cacheEntry['internalPath'], + $cacheEntry, + $mount, + $owner, + ); + } + + /** + * search for files by mimetype + * + * @param string $mimetype + * @return Node[] + */ + public function searchByMime($mimetype) { + if (!str_contains($mimetype, '/')) { + $query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%')); + } else { + $query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype)); + } + return $this->search($query); + } + + /** + * search for files by tag + * + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return Node[] + */ + public function searchByTag($tag, $userId) { + $query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', $tag), $userId); + return $this->search($query); + } + + public function searchBySystemTag(string $tagName, string $userId, int $limit = 0, int $offset = 0): array { + $query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'systemtag', $tagName), $userId, $limit, $offset); + return $this->search($query); + } + + /** + * @param int $id + * @return \OCP\Files\Node[] + */ + public function getById($id) { + return $this->root->getByIdInPath((int)$id, $this->getPath()); + } + + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + return $this->root->getFirstNodeByIdInPath($id, $this->getPath()); + } + + public function getAppDataDirectoryName(): string { + $instanceId = \OC::$server->getConfig()->getSystemValueString('instanceid'); + return 'appdata_' . $instanceId; + } + + /** + * In case the path we are currently in is inside the appdata_* folder, + * the original getById method does not work, because it can only look inside + * the user's mount points. But the user has no mount point for the root storage. + * + * So in that case we directly check the mount of the root if it contains + * the id. If it does we check if the path is inside the path we are working + * in. + * + * @param int $id + * @return array + */ + protected function getByIdInRootMount(int $id): array { + if (!method_exists($this->root, 'createNode')) { + // Always expected to be false. Being a method of Folder, this is + // always implemented. For it is an internal method and should not + // be exposed and made public, it is not part of an interface. + return []; + } + $mount = $this->root->getMount(''); + $storage = $mount->getStorage(); + $cacheEntry = $storage?->getCache($this->path)->get($id); + if (!$cacheEntry) { + return []; + } + + $absolutePath = '/' . ltrim($cacheEntry->getPath(), '/'); + $currentPath = rtrim($this->path, '/') . '/'; + + if (!str_starts_with($absolutePath, $currentPath)) { + return []; + } + + return [$this->root->createNode( + $absolutePath, new \OC\Files\FileInfo( + $absolutePath, + $storage, + $cacheEntry->getPath(), + $cacheEntry, + $mount + ))]; + } + + public function getFreeSpace() { + return $this->view->free_space($this->path); + } + + public function delete() { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_DELETE)) { + $this->sendHooks(['preDelete']); + $fileInfo = $this->getFileInfo(); + $this->view->rmdir($this->path); + $nonExisting = new NonExistingFolder($this->root, $this->view, $this->path, $fileInfo); + $this->sendHooks(['postDelete'], [$nonExisting]); + } else { + throw new NotPermittedException('No delete permission for path "' . $this->path . '"'); + } + } + + /** + * Add a suffix to the name in case the file exists + * + * @param string $name + * @return string + * @throws NotPermittedException + */ + public function getNonExistingName($name) { + $uniqueName = \OC_Helper::buildNotExistingFileNameForView($this->getPath(), $name, $this->view); + return trim($this->getRelativePath($uniqueName), '/'); + } + + /** + * @param int $limit + * @param int $offset + * @return INode[] + */ + public function getRecent($limit, $offset = 0) { + $filterOutNonEmptyFolder = new SearchBinaryOperator( + // filter out non empty folders + ISearchBinaryOperator::OPERATOR_OR, + [ + new SearchBinaryOperator( + ISearchBinaryOperator::OPERATOR_NOT, + [ + new SearchComparison( + ISearchComparison::COMPARE_EQUAL, + 'mimetype', + FileInfo::MIMETYPE_FOLDER + ), + ] + ), + new SearchComparison( + ISearchComparison::COMPARE_EQUAL, + 'size', + 0 + ), + ] + ); + + $filterNonRecentFiles = new SearchComparison( + ISearchComparison::COMPARE_GREATER_THAN, + 'mtime', + strtotime('-2 week') + ); + if ($offset === 0 && $limit <= 100) { + $query = new SearchQuery( + new SearchBinaryOperator( + ISearchBinaryOperator::OPERATOR_AND, + [ + $filterOutNonEmptyFolder, + $filterNonRecentFiles, + ], + ), + $limit, + $offset, + [ + new SearchOrder( + ISearchOrder::DIRECTION_DESCENDING, + 'mtime' + ), + ] + ); + } else { + $query = new SearchQuery( + $filterOutNonEmptyFolder, + $limit, + $offset, + [ + new SearchOrder( + ISearchOrder::DIRECTION_DESCENDING, + 'mtime' + ), + ] + ); + } + + return $this->search($query); + } + + public function verifyPath($fileName, $readonly = false): void { + $this->view->verifyPath( + $this->getPath(), + $fileName, + $readonly, + ); + } +} diff --git a/lib/private/Files/Node/HookConnector.php b/lib/private/Files/Node/HookConnector.php new file mode 100644 index 00000000000..1149951174c --- /dev/null +++ b/lib/private/Files/Node/HookConnector.php @@ -0,0 +1,227 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OC\Files\Filesystem; +use OC\Files\View; +use OCP\EventDispatcher\GenericEvent; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Exceptions\AbortedEventException; +use OCP\Files\Events\Node\BeforeNodeCopiedEvent; +use OCP\Files\Events\Node\BeforeNodeCreatedEvent; +use OCP\Files\Events\Node\BeforeNodeDeletedEvent; +use OCP\Files\Events\Node\BeforeNodeReadEvent; +use OCP\Files\Events\Node\BeforeNodeRenamedEvent; +use OCP\Files\Events\Node\BeforeNodeTouchedEvent; +use OCP\Files\Events\Node\BeforeNodeWrittenEvent; +use OCP\Files\Events\Node\NodeCopiedEvent; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeRenamedEvent; +use OCP\Files\Events\Node\NodeTouchedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\Files\FileInfo; +use OCP\Files\IRootFolder; +use OCP\Util; +use Psr\Log\LoggerInterface; + +class HookConnector { + /** @var FileInfo[] */ + private array $deleteMetaCache = []; + + public function __construct( + private IRootFolder $root, + private View $view, + private IEventDispatcher $dispatcher, + private LoggerInterface $logger, + ) { + } + + public function viewToNode() { + Util::connectHook('OC_Filesystem', 'write', $this, 'write'); + Util::connectHook('OC_Filesystem', 'post_write', $this, 'postWrite'); + + Util::connectHook('OC_Filesystem', 'create', $this, 'create'); + Util::connectHook('OC_Filesystem', 'post_create', $this, 'postCreate'); + + Util::connectHook('OC_Filesystem', 'delete', $this, 'delete'); + Util::connectHook('OC_Filesystem', 'post_delete', $this, 'postDelete'); + + Util::connectHook('OC_Filesystem', 'rename', $this, 'rename'); + Util::connectHook('OC_Filesystem', 'post_rename', $this, 'postRename'); + + Util::connectHook('OC_Filesystem', 'copy', $this, 'copy'); + Util::connectHook('OC_Filesystem', 'post_copy', $this, 'postCopy'); + + Util::connectHook('OC_Filesystem', 'touch', $this, 'touch'); + Util::connectHook('OC_Filesystem', 'post_touch', $this, 'postTouch'); + + Util::connectHook('OC_Filesystem', 'read', $this, 'read'); + } + + public function write($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'preWrite', [$node]); + $this->dispatcher->dispatch('\OCP\Files::preWrite', new GenericEvent($node)); + + $event = new BeforeNodeWrittenEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function postWrite($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'postWrite', [$node]); + $this->dispatcher->dispatch('\OCP\Files::postWrite', new GenericEvent($node)); + + $event = new NodeWrittenEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function create($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'preCreate', [$node]); + $this->dispatcher->dispatch('\OCP\Files::preCreate', new GenericEvent($node)); + + $event = new BeforeNodeCreatedEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function postCreate($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'postCreate', [$node]); + $this->dispatcher->dispatch('\OCP\Files::postCreate', new GenericEvent($node)); + + $event = new NodeCreatedEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function delete($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->deleteMetaCache[$node->getPath()] = $node->getFileInfo(); + $this->root->emit('\OC\Files', 'preDelete', [$node]); + $this->dispatcher->dispatch('\OCP\Files::preDelete', new GenericEvent($node)); + + $event = new BeforeNodeDeletedEvent($node); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('delete process aborted', ['exception' => $e]); + } + } + + public function postDelete($arguments) { + $node = $this->getNodeForPath($arguments['path']); + unset($this->deleteMetaCache[$node->getPath()]); + $this->root->emit('\OC\Files', 'postDelete', [$node]); + $this->dispatcher->dispatch('\OCP\Files::postDelete', new GenericEvent($node)); + + $event = new NodeDeletedEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function touch($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'preTouch', [$node]); + $this->dispatcher->dispatch('\OCP\Files::preTouch', new GenericEvent($node)); + + $event = new BeforeNodeTouchedEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function postTouch($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'postTouch', [$node]); + $this->dispatcher->dispatch('\OCP\Files::postTouch', new GenericEvent($node)); + + $event = new NodeTouchedEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + public function rename($arguments) { + $source = $this->getNodeForPath($arguments['oldpath']); + $target = $this->getNodeForPath($arguments['newpath']); + $this->root->emit('\OC\Files', 'preRename', [$source, $target]); + $this->dispatcher->dispatch('\OCP\Files::preRename', new GenericEvent([$source, $target])); + + $event = new BeforeNodeRenamedEvent($source, $target); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('rename process aborted', ['exception' => $e]); + } + } + + public function postRename($arguments) { + $source = $this->getNodeForPath($arguments['oldpath']); + $target = $this->getNodeForPath($arguments['newpath']); + $this->root->emit('\OC\Files', 'postRename', [$source, $target]); + $this->dispatcher->dispatch('\OCP\Files::postRename', new GenericEvent([$source, $target])); + + $event = new NodeRenamedEvent($source, $target); + $this->dispatcher->dispatchTyped($event); + } + + public function copy($arguments) { + $source = $this->getNodeForPath($arguments['oldpath']); + $target = $this->getNodeForPath($arguments['newpath'], $source instanceof Folder); + $this->root->emit('\OC\Files', 'preCopy', [$source, $target]); + $this->dispatcher->dispatch('\OCP\Files::preCopy', new GenericEvent([$source, $target])); + + $event = new BeforeNodeCopiedEvent($source, $target); + try { + $this->dispatcher->dispatchTyped($event); + } catch (AbortedEventException $e) { + $arguments['run'] = false; + $this->logger->warning('copy process aborted', ['exception' => $e]); + } + } + + public function postCopy($arguments) { + $source = $this->getNodeForPath($arguments['oldpath']); + $target = $this->getNodeForPath($arguments['newpath']); + $this->root->emit('\OC\Files', 'postCopy', [$source, $target]); + $this->dispatcher->dispatch('\OCP\Files::postCopy', new GenericEvent([$source, $target])); + + $event = new NodeCopiedEvent($source, $target); + $this->dispatcher->dispatchTyped($event); + } + + public function read($arguments) { + $node = $this->getNodeForPath($arguments['path']); + $this->root->emit('\OC\Files', 'read', [$node]); + $this->dispatcher->dispatch('\OCP\Files::read', new GenericEvent([$node])); + + $event = new BeforeNodeReadEvent($node); + $this->dispatcher->dispatchTyped($event); + } + + private function getNodeForPath(string $path, bool $isDir = false): Node { + $info = Filesystem::getView()->getFileInfo($path); + if (!$info) { + $fullPath = Filesystem::getView()->getAbsolutePath($path); + if (isset($this->deleteMetaCache[$fullPath])) { + $info = $this->deleteMetaCache[$fullPath]; + } else { + $info = null; + } + if ($isDir || Filesystem::is_dir($path)) { + return new NonExistingFolder($this->root, $this->view, $fullPath, $info); + } else { + return new NonExistingFile($this->root, $this->view, $fullPath, $info); + } + } + if ($info->getType() === FileInfo::TYPE_FILE) { + return new File($this->root, $this->view, $info->getPath(), $info); + } else { + return new Folder($this->root, $this->view, $info->getPath(), $info); + } + } +} diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php new file mode 100644 index 00000000000..37b1efa0fad --- /dev/null +++ b/lib/private/Files/Node/LazyFolder.php @@ -0,0 +1,568 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Node; + +use OC\Files\Filesystem; +use OC\Files\Utils\PathHelper; +use OCP\Constants; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotPermittedException; + +/** + * Class LazyFolder + * + * This is a lazy wrapper around a folder. So only + * once it is needed this will get initialized. + * + * @package OC\Files\Node + */ +class LazyFolder implements Folder { + /** @var \Closure(): Folder */ + private \Closure $folderClosure; + protected ?Folder $folder = null; + protected IRootFolder $rootFolder; + protected array $data; + + /** + * @param IRootFolder $rootFolder + * @param \Closure(): Folder $folderClosure + * @param array $data + */ + public function __construct(IRootFolder $rootFolder, \Closure $folderClosure, array $data = []) { + $this->rootFolder = $rootFolder; + $this->folderClosure = $folderClosure; + $this->data = $data; + } + + protected function getRootFolder(): IRootFolder { + return $this->rootFolder; + } + + protected function getRealFolder(): Folder { + if ($this->folder === null) { + $this->folder = call_user_func($this->folderClosure); + } + return $this->folder; + } + + /** + * Magic method to first get the real rootFolder and then + * call $method with $args on it + * + * @param $method + * @param $args + * @return mixed + */ + public function __call($method, $args) { + return call_user_func_array([$this->getRealFolder(), $method], $args); + } + + /** + * @inheritDoc + */ + public function getUser() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function listen($scope, $method, callable $callback) { + $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function removeListener($scope = null, $method = null, ?callable $callback = null) { + $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function emit($scope, $method, $arguments = []) { + $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function mount($storage, $mountPoint, $arguments = []) { + $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMount(string $mountPoint): IMountPoint { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @return IMountPoint[] + */ + public function getMountsIn(string $mountPoint): array { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMountByStorageId($storageId) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMountByNumericStorageId($numericId) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function unMount($mount) { + $this->__call(__FUNCTION__, func_get_args()); + } + + public function get($path) { + return $this->getRootFolder()->get($this->getFullPath($path)); + } + + /** + * @inheritDoc + */ + public function rename($targetPath) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function delete() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function copy($targetPath) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function touch($mtime = null) { + $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getStorage() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getPath() { + if (isset($this->data['path'])) { + return $this->data['path']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getInternalPath() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getId() { + if (isset($this->data['fileid'])) { + return $this->data['fileid']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function stat() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMTime() { + if (isset($this->data['mtime'])) { + return $this->data['mtime']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getSize($includeMounts = true): int|float { + if (isset($this->data['size'])) { + return $this->data['size']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getEtag() { + if (isset($this->data['etag'])) { + return $this->data['etag']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getPermissions() { + if (isset($this->data['permissions'])) { + return $this->data['permissions']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isReadable() { + if (isset($this->data['permissions'])) { + return ($this->data['permissions'] & Constants::PERMISSION_READ) == Constants::PERMISSION_READ; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isUpdateable() { + if (isset($this->data['permissions'])) { + return ($this->data['permissions'] & Constants::PERMISSION_UPDATE) == Constants::PERMISSION_UPDATE; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isDeletable() { + if (isset($this->data['permissions'])) { + return ($this->data['permissions'] & Constants::PERMISSION_DELETE) == Constants::PERMISSION_DELETE; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isShareable() { + if (isset($this->data['permissions'])) { + return ($this->data['permissions'] & Constants::PERMISSION_SHARE) == Constants::PERMISSION_SHARE; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getParent() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getName() { + if (isset($this->data['path'])) { + return basename($this->data['path']); + } + if (isset($this->data['name'])) { + return $this->data['name']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getUserFolder($userId) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMimetype() { + if (isset($this->data['mimetype'])) { + return $this->data['mimetype']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMimePart() { + if (isset($this->data['mimetype'])) { + [$part,] = explode('/', $this->data['mimetype']); + return $part; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isEncrypted() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getType() { + if (isset($this->data['type'])) { + return $this->data['type']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isShared() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isMounted() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getMountPoint() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getOwner() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getChecksum() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getExtension(): string { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getFullPath($path) { + if (isset($this->data['path'])) { + $path = PathHelper::normalizePath($path); + if (!Filesystem::isValidPath($path)) { + throw new NotPermittedException('Invalid path "' . $path . '"'); + } + return $this->data['path'] . $path; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isSubNode($node) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getDirectoryListing() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function nodeExists($path) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function newFolder($path) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function newFile($path, $content = null) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function search($query) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function searchByMime($mimetype) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function searchByTag($tag, $userId) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function searchBySystemTag(string $tagName, string $userId, int $limit = 0, int $offset = 0) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getById($id) { + return $this->getRootFolder()->getByIdInPath((int)$id, $this->getPath()); + } + + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + return $this->getRootFolder()->getFirstNodeByIdInPath($id, $this->getPath()); + } + + /** + * @inheritDoc + */ + public function getFreeSpace() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function isCreatable() { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getNonExistingName($name) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function move($targetPath) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function lock($type) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function changeLock($targetType) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function unlock($type) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getRecent($limit, $offset = 0) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getCreationTime(): int { + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + */ + public function getUploadTime(): int { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getRelativePath($path) { + return PathHelper::getRelativePath($this->getPath(), $path); + } + + public function getParentId(): int { + if (isset($this->data['parent'])) { + return $this->data['parent']; + } + return $this->__call(__FUNCTION__, func_get_args()); + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); + } + + public function verifyPath($fileName, $readonly = false): void { + $this->__call(__FUNCTION__, func_get_args()); + } +} diff --git a/lib/private/Files/Node/LazyRoot.php b/lib/private/Files/Node/LazyRoot.php new file mode 100644 index 00000000000..bc3f3a2e80f --- /dev/null +++ b/lib/private/Files/Node/LazyRoot.php @@ -0,0 +1,56 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Node; +use OCP\Files\Node as INode; + +/** + * Class LazyRoot + * + * This is a lazy wrapper around the root. So only + * once it is needed this will get initialized. + * + * @package OC\Files\Node + */ +class LazyRoot extends LazyFolder implements IRootFolder { + public function __construct(\Closure $folderClosure, array $data = []) { + parent::__construct($this, $folderClosure, $data); + } + + protected function getRootFolder(): IRootFolder { + $folder = $this->getRealFolder(); + if (!$folder instanceof IRootFolder) { + throw new \Exception('Lazy root folder closure didn\'t return a root folder'); + } + return $folder; + } + + public function getUserFolder($userId) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getByIdInPath(int $id, string $path) { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getFirstNodeByIdInPath(int $id, string $path): ?Node { + return $this->__call(__FUNCTION__, func_get_args()); + } + + public function getNodeFromCacheEntryAndMount(ICacheEntry $cacheEntry, IMountPoint $mountPoint): INode { + return $this->getRootFolder()->getNodeFromCacheEntryAndMount($cacheEntry, $mountPoint); + } + + public function getAppDataDirectoryName(): string { + return $this->__call(__FUNCTION__, func_get_args()); + } +} diff --git a/lib/private/Files/Node/LazyUserFolder.php b/lib/private/Files/Node/LazyUserFolder.php new file mode 100644 index 00000000000..77479c2fa5e --- /dev/null +++ b/lib/private/Files/Node/LazyUserFolder.php @@ -0,0 +1,66 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Node; + +use OCP\Constants; +use OCP\Files\File; +use OCP\Files\FileInfo; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountManager; +use OCP\Files\NotFoundException; +use OCP\IUser; +use Psr\Log\LoggerInterface; + +class LazyUserFolder extends LazyFolder { + private IUser $user; + private string $path; + private IMountManager $mountManager; + + public function __construct(IRootFolder $rootFolder, IUser $user, IMountManager $mountManager) { + $this->user = $user; + $this->mountManager = $mountManager; + $this->path = '/' . $user->getUID() . '/files'; + parent::__construct($rootFolder, function () use ($user): Folder { + try { + $node = $this->getRootFolder()->get($this->path); + if ($node instanceof File) { + $e = new \RuntimeException(); + \OCP\Server::get(LoggerInterface::class)->error('User root storage is not a folder: ' . $this->path, [ + 'exception' => $e, + ]); + throw $e; + } + return $node; + } catch (NotFoundException $e) { + if (!$this->getRootFolder()->nodeExists('/' . $user->getUID())) { + $this->getRootFolder()->newFolder('/' . $user->getUID()); + } + return $this->getRootFolder()->newFolder($this->path); + } + }, [ + 'path' => $this->path, + // Sharing user root folder is not allowed + 'permissions' => Constants::PERMISSION_ALL ^ Constants::PERMISSION_SHARE, + 'type' => FileInfo::TYPE_FOLDER, + 'mimetype' => FileInfo::MIMETYPE_FOLDER, + ]); + } + + public function getMountPoint() { + if ($this->folder !== null) { + return $this->folder->getMountPoint(); + } + $mountPoint = $this->mountManager->find('/' . $this->user->getUID()); + if (is_null($mountPoint)) { + throw new \Exception('No mountpoint for user folder'); + } + return $mountPoint; + } +} diff --git a/lib/private/Files/Node/Node.php b/lib/private/Files/Node/Node.php new file mode 100644 index 00000000000..5dbdc4054bf --- /dev/null +++ b/lib/private/Files/Node/Node.php @@ -0,0 +1,489 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OC\Files\Filesystem; +use OC\Files\Mount\MoveableMount; +use OC\Files\Utils\PathHelper; +use OCP\EventDispatcher\GenericEvent; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\FileInfo; +use OCP\Files\InvalidPathException; +use OCP\Files\IRootFolder; +use OCP\Files\Node as INode; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Lock\LockedException; +use OCP\PreConditionNotMetException; + +// FIXME: this class really should be abstract (+1) +class Node implements INode { + /** + * @var \OC\Files\View $view + */ + protected $view; + + protected IRootFolder $root; + + /** + * @var string $path Absolute path to the node (e.g. /admin/files/folder/file) + */ + protected $path; + + protected ?FileInfo $fileInfo; + + protected ?INode $parent; + + private bool $infoHasSubMountsIncluded; + + /** + * @param \OC\Files\View $view + * @param \OCP\Files\IRootFolder $root + * @param string $path + * @param FileInfo $fileInfo + */ + public function __construct(IRootFolder $root, $view, $path, $fileInfo = null, ?INode $parent = null, bool $infoHasSubMountsIncluded = true) { + if (Filesystem::normalizePath($view->getRoot()) !== '/') { + throw new PreConditionNotMetException('The view passed to the node should not have any fake root set'); + } + $this->view = $view; + $this->root = $root; + $this->path = $path; + $this->fileInfo = $fileInfo; + $this->parent = $parent; + $this->infoHasSubMountsIncluded = $infoHasSubMountsIncluded; + } + + /** + * Creates a Node of the same type that represents a non-existing path + * + * @param string $path path + * @return Node non-existing node + * @throws \Exception + */ + protected function createNonExistingNode($path) { + throw new \Exception('Must be implemented by subclasses'); + } + + /** + * Returns the matching file info + * + * @return FileInfo + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getFileInfo(bool $includeMountPoint = true) { + if (!$this->fileInfo) { + if (!Filesystem::isValidPath($this->path)) { + throw new InvalidPathException(); + } + $fileInfo = $this->view->getFileInfo($this->path, $includeMountPoint); + $this->infoHasSubMountsIncluded = $includeMountPoint; + if ($fileInfo instanceof FileInfo) { + $this->fileInfo = $fileInfo; + } else { + throw new NotFoundException(); + } + } elseif ($includeMountPoint && !$this->infoHasSubMountsIncluded && $this instanceof Folder) { + if ($this->fileInfo instanceof \OC\Files\FileInfo) { + $this->view->addSubMounts($this->fileInfo); + } + $this->infoHasSubMountsIncluded = true; + } + return $this->fileInfo; + } + + /** + * @param string[] $hooks + */ + protected function sendHooks($hooks, ?array $args = null) { + $args = !empty($args) ? $args : [$this]; + /** @var IEventDispatcher $dispatcher */ + $dispatcher = \OC::$server->get(IEventDispatcher::class); + foreach ($hooks as $hook) { + if (method_exists($this->root, 'emit')) { + $this->root->emit('\OC\Files', $hook, $args); + } + + if (in_array($hook, ['preWrite', 'postWrite', 'preCreate', 'postCreate', 'preTouch', 'postTouch', 'preDelete', 'postDelete'], true)) { + $event = new GenericEvent($args[0]); + } else { + $event = new GenericEvent($args); + } + + $dispatcher->dispatch('\OCP\Files::' . $hook, $event); + } + } + + /** + * @param int $permissions + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + protected function checkPermissions($permissions) { + return ($this->getPermissions() & $permissions) === $permissions; + } + + public function delete() { + } + + /** + * @param int $mtime + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException + */ + public function touch($mtime = null) { + if ($this->checkPermissions(\OCP\Constants::PERMISSION_UPDATE)) { + $this->sendHooks(['preTouch']); + $this->view->touch($this->path, $mtime); + $this->sendHooks(['postTouch']); + if ($this->fileInfo) { + if (is_null($mtime)) { + $mtime = time(); + } + $this->fileInfo['mtime'] = $mtime; + } + } else { + throw new NotPermittedException(); + } + } + + public function getStorage() { + $storage = $this->getMountPoint()->getStorage(); + if (!$storage) { + throw new \Exception('No storage for node'); + } + return $storage; + } + + /** + * @return string + */ + public function getPath() { + return $this->path; + } + + /** + * @return string + */ + public function getInternalPath() { + return $this->getFileInfo(false)->getInternalPath(); + } + + /** + * @return int + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getId() { + return $this->getFileInfo(false)->getId() ?? -1; + } + + /** + * @return array + */ + public function stat() { + return $this->view->stat($this->path); + } + + /** + * @return int + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getMTime() { + return $this->getFileInfo()->getMTime(); + } + + /** + * @param bool $includeMounts + * @return int|float + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getSize($includeMounts = true): int|float { + return $this->getFileInfo()->getSize($includeMounts); + } + + /** + * @return string + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getEtag() { + return $this->getFileInfo()->getEtag(); + } + + /** + * @return int + * @throws InvalidPathException + * @throws NotFoundException + */ + public function getPermissions() { + return $this->getFileInfo(false)->getPermissions(); + } + + /** + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + public function isReadable() { + return $this->getFileInfo(false)->isReadable(); + } + + /** + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + public function isUpdateable() { + return $this->getFileInfo(false)->isUpdateable(); + } + + /** + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + public function isDeletable() { + return $this->getFileInfo(false)->isDeletable(); + } + + /** + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + public function isShareable() { + return $this->getFileInfo(false)->isShareable(); + } + + /** + * @return bool + * @throws InvalidPathException + * @throws NotFoundException + */ + public function isCreatable() { + return $this->getFileInfo(false)->isCreatable(); + } + + public function getParent(): INode|IRootFolder { + if ($this->parent === null) { + $newPath = dirname($this->path); + if ($newPath === '' || $newPath === '.' || $newPath === '/') { + return $this->root; + } + + // Manually fetch the parent if the current node doesn't have a file info yet + try { + $fileInfo = $this->getFileInfo(); + } catch (NotFoundException) { + $this->parent = $this->root->get($newPath); + /** @var \OCP\Files\Folder $this->parent */ + return $this->parent; + } + + // gather the metadata we already know about our parent + $parentData = [ + 'path' => $newPath, + 'fileid' => $fileInfo->getParentId(), + ]; + + // and create lazy folder with it instead of always querying + $this->parent = new LazyFolder($this->root, function () use ($newPath) { + return $this->root->get($newPath); + }, $parentData); + } + + return $this->parent; + } + + /** + * @return string + */ + public function getName() { + return basename($this->path); + } + + /** + * @param string $path + * @return string + */ + protected function normalizePath($path) { + return PathHelper::normalizePath($path); + } + + /** + * check if the requested path is valid + * + * @param string $path + * @return bool + */ + public function isValidPath($path) { + return Filesystem::isValidPath($path); + } + + public function isMounted() { + return $this->getFileInfo(false)->isMounted(); + } + + public function isShared() { + return $this->getFileInfo(false)->isShared(); + } + + public function getMimeType() { + return $this->getFileInfo(false)->getMimetype(); + } + + public function getMimePart() { + return $this->getFileInfo(false)->getMimePart(); + } + + public function getType() { + return $this->getFileInfo(false)->getType(); + } + + public function isEncrypted() { + return $this->getFileInfo(false)->isEncrypted(); + } + + public function getMountPoint() { + return $this->getFileInfo(false)->getMountPoint(); + } + + public function getOwner() { + return $this->getFileInfo(false)->getOwner(); + } + + public function getChecksum() { + } + + public function getExtension(): string { + return $this->getFileInfo(false)->getExtension(); + } + + /** + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @throws LockedException + */ + public function lock($type) { + $this->view->lockFile($this->path, $type); + } + + /** + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @throws LockedException + */ + public function changeLock($type) { + $this->view->changeLock($this->path, $type); + } + + /** + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @throws LockedException + */ + public function unlock($type) { + $this->view->unlockFile($this->path, $type); + } + + /** + * @param string $targetPath + * @return INode + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException if copy not allowed or failed + */ + public function copy($targetPath) { + $targetPath = $this->normalizePath($targetPath); + $parent = $this->root->get(dirname($targetPath)); + if ($parent instanceof Folder and $this->isValidPath($targetPath) and $parent->isCreatable()) { + $nonExisting = $this->createNonExistingNode($targetPath); + $this->sendHooks(['preCopy'], [$this, $nonExisting]); + $this->sendHooks(['preWrite'], [$nonExisting]); + if (!$this->view->copy($this->path, $targetPath)) { + throw new NotPermittedException('Could not copy ' . $this->path . ' to ' . $targetPath); + } + $targetNode = $this->root->get($targetPath); + $this->sendHooks(['postCopy'], [$this, $targetNode]); + $this->sendHooks(['postWrite'], [$targetNode]); + return $targetNode; + } else { + throw new NotPermittedException('No permission to copy to path ' . $targetPath); + } + } + + /** + * @param string $targetPath + * @return INode + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException if move not allowed or failed + * @throws LockedException + */ + public function move($targetPath) { + $targetPath = $this->normalizePath($targetPath); + $parent = $this->root->get(dirname($targetPath)); + if ( + ($parent instanceof Folder) + && $this->isValidPath($targetPath) + && ( + $parent->isCreatable() + || ( + $parent->getInternalPath() === '' + && ($parent->getMountPoint() instanceof MoveableMount) + ) + ) + ) { + $nonExisting = $this->createNonExistingNode($targetPath); + $this->sendHooks(['preRename'], [$this, $nonExisting]); + $this->sendHooks(['preWrite'], [$nonExisting]); + if (!$this->view->rename($this->path, $targetPath)) { + throw new NotPermittedException('Could not move ' . $this->path . ' to ' . $targetPath); + } + + $mountPoint = $this->getMountPoint(); + if ($mountPoint) { + // update the cached fileinfo with the new (internal) path + /** @var \OC\Files\FileInfo $oldFileInfo */ + $oldFileInfo = $this->getFileInfo(); + $this->fileInfo = new \OC\Files\FileInfo($targetPath, $oldFileInfo->getStorage(), $mountPoint->getInternalPath($targetPath), $oldFileInfo->getData(), $mountPoint, $oldFileInfo->getOwner()); + } + + $targetNode = $this->root->get($targetPath); + $this->sendHooks(['postRename'], [$this, $targetNode]); + $this->sendHooks(['postWrite'], [$targetNode]); + $this->path = $targetPath; + return $targetNode; + } else { + throw new NotPermittedException('No permission to move to path ' . $targetPath); + } + } + + public function getCreationTime(): int { + return $this->getFileInfo()->getCreationTime(); + } + + public function getUploadTime(): int { + return $this->getFileInfo()->getUploadTime(); + } + + public function getParentId(): int { + return $this->fileInfo->getParentId(); + } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } +} diff --git a/lib/private/Files/Node/NonExistingFile.php b/lib/private/Files/Node/NonExistingFile.php new file mode 100644 index 00000000000..66ec2e6c040 --- /dev/null +++ b/lib/private/Files/Node/NonExistingFile.php @@ -0,0 +1,136 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OCP\Files\NotFoundException; + +class NonExistingFile extends File { + /** + * @param string $newPath + * @throws \OCP\Files\NotFoundException + */ + public function rename($newPath) { + throw new NotFoundException(); + } + + public function delete() { + throw new NotFoundException(); + } + + public function copy($targetPath) { + throw new NotFoundException(); + } + + public function touch($mtime = null) { + throw new NotFoundException(); + } + + public function getId() { + if ($this->fileInfo) { + return parent::getId(); + } else { + throw new NotFoundException(); + } + } + + public function getInternalPath() { + if ($this->fileInfo) { + return parent::getInternalPath(); + } else { + return $this->getParent()->getMountPoint()->getInternalPath($this->getPath()); + } + } + + public function stat() { + throw new NotFoundException(); + } + + public function getMTime() { + if ($this->fileInfo) { + return parent::getMTime(); + } else { + throw new NotFoundException(); + } + } + + public function getSize($includeMounts = true): int|float { + if ($this->fileInfo) { + return parent::getSize($includeMounts); + } else { + throw new NotFoundException(); + } + } + + public function getEtag() { + if ($this->fileInfo) { + return parent::getEtag(); + } else { + throw new NotFoundException(); + } + } + + public function getPermissions() { + if ($this->fileInfo) { + return parent::getPermissions(); + } else { + throw new NotFoundException(); + } + } + + public function isReadable() { + if ($this->fileInfo) { + return parent::isReadable(); + } else { + throw new NotFoundException(); + } + } + + public function isUpdateable() { + if ($this->fileInfo) { + return parent::isUpdateable(); + } else { + throw new NotFoundException(); + } + } + + public function isDeletable() { + if ($this->fileInfo) { + return parent::isDeletable(); + } else { + throw new NotFoundException(); + } + } + + public function isShareable() { + if ($this->fileInfo) { + return parent::isShareable(); + } else { + throw new NotFoundException(); + } + } + + public function getContent() { + throw new NotFoundException(); + } + + public function putContent($data) { + throw new NotFoundException(); + } + + public function getMimeType() { + if ($this->fileInfo) { + return parent::getMimeType(); + } else { + throw new NotFoundException(); + } + } + + public function fopen($mode) { + throw new NotFoundException(); + } +} diff --git a/lib/private/Files/Node/NonExistingFolder.php b/lib/private/Files/Node/NonExistingFolder.php new file mode 100644 index 00000000000..4489fdaf010 --- /dev/null +++ b/lib/private/Files/Node/NonExistingFolder.php @@ -0,0 +1,172 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OCP\Files\NotFoundException; + +class NonExistingFolder extends Folder { + /** + * @param string $newPath + * @throws \OCP\Files\NotFoundException + */ + public function rename($newPath) { + throw new NotFoundException(); + } + + public function delete() { + throw new NotFoundException(); + } + + public function copy($targetPath) { + throw new NotFoundException(); + } + + public function touch($mtime = null) { + throw new NotFoundException(); + } + + public function getId() { + if ($this->fileInfo) { + return parent::getId(); + } else { + throw new NotFoundException(); + } + } + + public function getInternalPath() { + if ($this->fileInfo) { + return parent::getInternalPath(); + } else { + return $this->getParent()->getMountPoint()->getInternalPath($this->getPath()); + } + } + + public function stat() { + throw new NotFoundException(); + } + + public function getMTime() { + if ($this->fileInfo) { + return parent::getMTime(); + } else { + throw new NotFoundException(); + } + } + + public function getSize($includeMounts = true): int|float { + if ($this->fileInfo) { + return parent::getSize($includeMounts); + } else { + throw new NotFoundException(); + } + } + + public function getEtag() { + if ($this->fileInfo) { + return parent::getEtag(); + } else { + throw new NotFoundException(); + } + } + + public function getPermissions() { + if ($this->fileInfo) { + return parent::getPermissions(); + } else { + throw new NotFoundException(); + } + } + + public function isReadable() { + if ($this->fileInfo) { + return parent::isReadable(); + } else { + throw new NotFoundException(); + } + } + + public function isUpdateable() { + if ($this->fileInfo) { + return parent::isUpdateable(); + } else { + throw new NotFoundException(); + } + } + + public function isDeletable() { + if ($this->fileInfo) { + return parent::isDeletable(); + } else { + throw new NotFoundException(); + } + } + + public function isShareable() { + if ($this->fileInfo) { + return parent::isShareable(); + } else { + throw new NotFoundException(); + } + } + + public function get($path) { + throw new NotFoundException(); + } + + public function getDirectoryListing() { + throw new NotFoundException(); + } + + public function nodeExists($path) { + return false; + } + + public function newFolder($path) { + throw new NotFoundException(); + } + + public function newFile($path, $content = null) { + throw new NotFoundException(); + } + + public function search($query) { + throw new NotFoundException(); + } + + public function searchByMime($mimetype) { + throw new NotFoundException(); + } + + public function searchByTag($tag, $userId) { + throw new NotFoundException(); + } + + public function searchBySystemTag(string $tagName, string $userId, int $limit = 0, int $offset = 0): array { + throw new NotFoundException(); + } + + public function getById($id) { + throw new NotFoundException(); + } + + public function getFirstNodeById(int $id): ?\OCP\Files\Node { + throw new NotFoundException(); + } + + public function getFreeSpace() { + throw new NotFoundException(); + } + + public function isCreatable() { + if ($this->fileInfo) { + return parent::isCreatable(); + } else { + throw new NotFoundException(); + } + } +} diff --git a/lib/private/Files/Node/Root.php b/lib/private/Files/Node/Root.php new file mode 100644 index 00000000000..76afca9dee8 --- /dev/null +++ b/lib/private/Files/Node/Root.php @@ -0,0 +1,534 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Node; + +use OC\Files\FileInfo; +use OC\Files\Mount\Manager; +use OC\Files\Mount\MountPoint; +use OC\Files\Utils\PathHelper; +use OC\Files\View; +use OC\Hooks\PublicEmitter; +use OC\User\NoUserException; +use OCP\Cache\CappedMemoryCache; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Config\IUserMountCache; +use OCP\Files\Events\Node\FilesystemTornDownEvent; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Node as INode; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IUser; +use OCP\IUserManager; +use OCP\Server; +use Psr\Log\LoggerInterface; + +/** + * Class Root + * + * Hooks available in scope \OC\Files + * - preWrite(\OCP\Files\Node $node) + * - postWrite(\OCP\Files\Node $node) + * - preCreate(\OCP\Files\Node $node) + * - postCreate(\OCP\Files\Node $node) + * - preDelete(\OCP\Files\Node $node) + * - postDelete(\OCP\Files\Node $node) + * - preTouch(\OC\FilesP\Node $node, int $mtime) + * - postTouch(\OCP\Files\Node $node) + * - preCopy(\OCP\Files\Node $source, \OCP\Files\Node $target) + * - postCopy(\OCP\Files\Node $source, \OCP\Files\Node $target) + * - preRename(\OCP\Files\Node $source, \OCP\Files\Node $target) + * - postRename(\OCP\Files\Node $source, \OCP\Files\Node $target) + * + * @package OC\Files\Node + */ +class Root extends Folder implements IRootFolder { + private Manager $mountManager; + private PublicEmitter $emitter; + private ?IUser $user; + private CappedMemoryCache $userFolderCache; + private IUserMountCache $userMountCache; + private LoggerInterface $logger; + private IUserManager $userManager; + private IEventDispatcher $eventDispatcher; + private ICache $pathByIdCache; + + /** + * @param Manager $manager + * @param View $view + * @param IUser|null $user + */ + public function __construct( + $manager, + $view, + $user, + IUserMountCache $userMountCache, + LoggerInterface $logger, + IUserManager $userManager, + IEventDispatcher $eventDispatcher, + ICacheFactory $cacheFactory, + ) { + parent::__construct($this, $view, ''); + $this->mountManager = $manager; + $this->user = $user; + $this->emitter = new PublicEmitter(); + $this->userFolderCache = new CappedMemoryCache(); + $this->userMountCache = $userMountCache; + $this->logger = $logger; + $this->userManager = $userManager; + $eventDispatcher->addListener(FilesystemTornDownEvent::class, function () { + $this->userFolderCache = new CappedMemoryCache(); + }); + $this->pathByIdCache = $cacheFactory->createLocal('path-by-id'); + } + + /** + * Get the user for which the filesystem is setup + * + * @return \OC\User\User + */ + public function getUser() { + return $this->user; + } + + /** + * @param string $scope + * @param string $method + * @param callable $callback + */ + public function listen($scope, $method, callable $callback) { + $this->emitter->listen($scope, $method, $callback); + } + + /** + * @param string $scope optional + * @param string $method optional + * @param callable $callback optional + */ + public function removeListener($scope = null, $method = null, ?callable $callback = null) { + $this->emitter->removeListener($scope, $method, $callback); + } + + /** + * @param string $scope + * @param string $method + * @param Node[] $arguments + */ + public function emit($scope, $method, $arguments = []) { + $this->emitter->emit($scope, $method, $arguments); + } + + /** + * @param \OC\Files\Storage\Storage $storage + * @param string $mountPoint + * @param array $arguments + */ + public function mount($storage, $mountPoint, $arguments = []) { + $mount = new MountPoint($storage, $mountPoint, $arguments); + $this->mountManager->addMount($mount); + } + + public function getMount(string $mountPoint): IMountPoint { + return $this->mountManager->find($mountPoint); + } + + /** + * @param string $mountPoint + * @return \OC\Files\Mount\MountPoint[] + */ + public function getMountsIn(string $mountPoint): array { + return $this->mountManager->findIn($mountPoint); + } + + /** + * @param string $storageId + * @return \OC\Files\Mount\MountPoint[] + */ + public function getMountByStorageId($storageId) { + return $this->mountManager->findByStorageId($storageId); + } + + /** + * @param int $numericId + * @return MountPoint[] + */ + public function getMountByNumericStorageId($numericId) { + return $this->mountManager->findByNumericId($numericId); + } + + /** + * @param \OC\Files\Mount\MountPoint $mount + */ + public function unMount($mount) { + $this->mountManager->remove($mount); + } + + public function get($path) { + $path = $this->normalizePath($path); + if ($this->isValidPath($path)) { + $fullPath = $this->getFullPath($path); + $fileInfo = $this->view->getFileInfo($fullPath, false); + if ($fileInfo) { + return $this->createNode($fullPath, $fileInfo, false); + } else { + throw new NotFoundException($path); + } + } else { + throw new NotPermittedException(); + } + } + + //most operations can't be done on the root + + /** + * @param string $targetPath + * @return Node + * @throws \OCP\Files\NotPermittedException + */ + public function rename($targetPath) { + throw new NotPermittedException(); + } + + public function delete() { + throw new NotPermittedException(); + } + + /** + * @param string $targetPath + * @return Node + * @throws \OCP\Files\NotPermittedException + */ + public function copy($targetPath) { + throw new NotPermittedException(); + } + + /** + * @param int $mtime + * @throws \OCP\Files\NotPermittedException + */ + public function touch($mtime = null) { + throw new NotPermittedException(); + } + + /** + * @return \OC\Files\Storage\Storage + * @throws \OCP\Files\NotFoundException + */ + public function getStorage() { + throw new NotFoundException(); + } + + /** + * @return string + */ + public function getPath() { + return '/'; + } + + /** + * @return string + */ + public function getInternalPath() { + return ''; + } + + /** + * @return int + */ + public function getId() { + return 0; + } + + /** + * @return array + */ + public function stat() { + return []; + } + + /** + * @return int + */ + public function getMTime() { + return 0; + } + + /** + * @param bool $includeMounts + * @return int|float + */ + public function getSize($includeMounts = true): int|float { + return 0; + } + + /** + * @return string + */ + public function getEtag() { + return ''; + } + + /** + * @return int + */ + public function getPermissions() { + return \OCP\Constants::PERMISSION_CREATE; + } + + /** + * @return bool + */ + public function isReadable() { + return false; + } + + /** + * @return bool + */ + public function isUpdateable() { + return false; + } + + /** + * @return bool + */ + public function isDeletable() { + return false; + } + + /** + * @return bool + */ + public function isShareable() { + return false; + } + + /** + * @throws \OCP\Files\NotFoundException + */ + public function getParent(): INode|IRootFolder { + throw new NotFoundException(); + } + + /** + * @return string + */ + public function getName() { + return ''; + } + + /** + * Returns a view to user's files folder + * + * @param string $userId user ID + * @return \OCP\Files\Folder + * @throws NoUserException + * @throws NotPermittedException + */ + public function getUserFolder($userId) { + $userObject = $this->userManager->get($userId); + + if (is_null($userObject)) { + $e = new NoUserException('Backends provided no user object'); + $this->logger->error( + sprintf( + 'Backends provided no user object for %s', + $userId + ), + [ + 'app' => 'files', + 'exception' => $e, + ] + ); + throw $e; + } + + $userId = $userObject->getUID(); + + if (!$this->userFolderCache->hasKey($userId)) { + if ($this->mountManager->getSetupManager()->isSetupComplete($userObject)) { + try { + $folder = $this->get('/' . $userId . '/files'); + if (!$folder instanceof \OCP\Files\Folder) { + throw new \Exception("Account folder for \"$userId\" exists as a file"); + } + } catch (NotFoundException $e) { + if (!$this->nodeExists('/' . $userId)) { + $this->newFolder('/' . $userId); + } + $folder = $this->newFolder('/' . $userId . '/files'); + } + } else { + $folder = new LazyUserFolder($this, $userObject, $this->mountManager); + } + + $this->userFolderCache->set($userId, $folder); + } + + return $this->userFolderCache->get($userId); + } + + public function getUserMountCache() { + return $this->userMountCache; + } + + public function getFirstNodeByIdInPath(int $id, string $path): ?INode { + // scope the cache by user, so we don't return nodes for different users + if ($this->user) { + $cachedPath = $this->pathByIdCache->get($this->user->getUID() . '::' . $id); + if ($cachedPath && str_starts_with($cachedPath, $path)) { + // getting the node by path is significantly cheaper than finding it by id + try { + $node = $this->get($cachedPath); + // by validating that the cached path still has the requested fileid we can work around the need to invalidate the cached path + // if the cached path is invalid or a different file now we fall back to the uncached logic + if ($node && $node->getId() === $id) { + return $node; + } + } catch (NotFoundException|NotPermittedException) { + // The file may be moved but the old path still in cache + } + } + } + $node = current($this->getByIdInPath($id, $path)); + if (!$node) { + return null; + } + + if ($this->user) { + $this->pathByIdCache->set($this->user->getUID() . '::' . $id, $node->getPath()); + } + return $node; + } + + /** + * @param int $id + * @return Node[] + */ + public function getByIdInPath(int $id, string $path): array { + $mountCache = $this->getUserMountCache(); + if ($path !== '' && strpos($path, '/', 1) > 0) { + [, $user] = explode('/', $path); + } else { + $user = null; + } + $mountsContainingFile = $mountCache->getMountsForFileId($id, $user); + + // if the mount isn't in the cache yet, perform a setup first, then try again + if (count($mountsContainingFile) === 0) { + $this->mountManager->getSetupManager()->setupForPath($path, true); + $mountsContainingFile = $mountCache->getMountsForFileId($id, $user); + } + + // when a user has access through the same storage through multiple paths + // (such as an external storage that is both mounted for a user and shared to the user) + // the mount cache will only hold a single entry for the storage + // this can lead to issues as the different ways the user has access to a storage can have different permissions + // + // so instead of using the cached entries directly, we instead filter the current mounts by the rootid of the cache entry + + $mountRootIds = array_map(function ($mount) { + return $mount->getRootId(); + }, $mountsContainingFile); + $mountRootPaths = array_map(function ($mount) { + return $mount->getRootInternalPath(); + }, $mountsContainingFile); + $mountProviders = array_unique(array_map(function ($mount) { + return $mount->getMountProvider(); + }, $mountsContainingFile)); + $mountRoots = array_combine($mountRootIds, $mountRootPaths); + + $mounts = $this->mountManager->getMountsByMountProvider($path, $mountProviders); + + $mountsContainingFile = array_filter($mounts, function ($mount) use ($mountRoots) { + return isset($mountRoots[$mount->getStorageRootId()]); + }); + + if (count($mountsContainingFile) === 0) { + if ($user === $this->getAppDataDirectoryName()) { + $folder = $this->get($path); + if ($folder instanceof Folder) { + return $folder->getByIdInRootMount($id); + } else { + throw new \Exception('getByIdInPath with non folder'); + } + } + return []; + } + + $nodes = array_map(function (IMountPoint $mount) use ($id, $mountRoots) { + $rootInternalPath = $mountRoots[$mount->getStorageRootId()]; + $cacheEntry = $mount->getStorage()->getCache()->get($id); + if (!$cacheEntry) { + return null; + } + + // cache jails will hide the "true" internal path + $internalPath = ltrim($rootInternalPath . '/' . $cacheEntry->getPath(), '/'); + $pathRelativeToMount = substr($internalPath, strlen($rootInternalPath)); + $pathRelativeToMount = ltrim($pathRelativeToMount, '/'); + $absolutePath = rtrim($mount->getMountPoint() . $pathRelativeToMount, '/'); + $storage = $mount->getStorage(); + if ($storage === null) { + return null; + } + $ownerId = $storage->getOwner($pathRelativeToMount); + if ($ownerId !== false) { + $owner = Server::get(IUserManager::class)->get($ownerId); + } else { + $owner = null; + } + return $this->createNode($absolutePath, new FileInfo( + $absolutePath, + $storage, + $cacheEntry->getPath(), + $cacheEntry, + $mount, + $owner, + )); + }, $mountsContainingFile); + + $nodes = array_filter($nodes); + + $folders = array_filter($nodes, function (Node $node) use ($path) { + return PathHelper::getRelativePath($path, $node->getPath()) !== null; + }); + usort($folders, function ($a, $b) { + return $b->getPath() <=> $a->getPath(); + }); + return $folders; + } + + public function getNodeFromCacheEntryAndMount(ICacheEntry $cacheEntry, IMountPoint $mountPoint): INode { + $path = $cacheEntry->getPath(); + $fullPath = $mountPoint->getMountPoint() . $path; + // todo: LazyNode? + $info = new FileInfo($fullPath, $mountPoint->getStorage(), $path, $cacheEntry, $mountPoint); + $parentPath = dirname($fullPath); + $parent = new LazyFolder($this, function () use ($parentPath) { + $parent = $this->get($parentPath); + if ($parent instanceof \OCP\Files\Folder) { + return $parent; + } else { + throw new \Exception("parent $parentPath is not a folder"); + } + }, [ + 'path' => $parentPath, + ]); + $isDir = $info->getType() === FileInfo::TYPE_FOLDER; + $view = new View(''); + if ($isDir) { + return new Folder($this, $view, $fullPath, $info, $parent); + } else { + return new File($this, $view, $fullPath, $info, $parent); + } + } +} diff --git a/lib/private/Files/Notify/Change.php b/lib/private/Files/Notify/Change.php new file mode 100644 index 00000000000..c8eccd11ae2 --- /dev/null +++ b/lib/private/Files/Notify/Change.php @@ -0,0 +1,48 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Notify; + +use OCP\Files\Notify\IChange; + +class Change implements IChange { + /** @var int */ + private $type; + + /** @var string */ + private $path; + + /** + * Change constructor. + * + * @param int $type + * @param string $path + */ + public function __construct($type, $path) { + $this->type = $type; + $this->path = $path; + } + + /** + * Get the type of the change + * + * @return int IChange::ADDED, IChange::REMOVED, IChange::MODIFIED or IChange::RENAMED + */ + public function getType() { + return $this->type; + } + + /** + * Get the path of the file that was changed relative to the root of the storage + * + * Note, for rename changes this path is the old path for the file + * + * @return mixed + */ + public function getPath() { + return $this->path; + } +} diff --git a/lib/private/Files/Notify/RenameChange.php b/lib/private/Files/Notify/RenameChange.php new file mode 100644 index 00000000000..28554ceaa26 --- /dev/null +++ b/lib/private/Files/Notify/RenameChange.php @@ -0,0 +1,35 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Notify; + +use OCP\Files\Notify\IRenameChange; + +class RenameChange extends Change implements IRenameChange { + /** @var string */ + private $targetPath; + + /** + * Change constructor. + * + * @param int $type + * @param string $path + * @param string $targetPath + */ + public function __construct($type, $path, $targetPath) { + parent::__construct($type, $path); + $this->targetPath = $targetPath; + } + + /** + * Get the new path of the renamed file relative to the storage root + * + * @return string + */ + public function getTargetPath() { + return $this->targetPath; + } +} diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php new file mode 100644 index 00000000000..aaaee044bac --- /dev/null +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -0,0 +1,29 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { + private string $internalId; + + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (!isset($parameters['internal-id'])) { + throw new \Exception('missing id in parameters'); + } + $this->internalId = (string)$parameters['internal-id']; + parent::__construct($parameters); + } + + public function getId(): string { + return 'object::appdata::preview:' . $this->internalId; + } +} diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php new file mode 100644 index 00000000000..2729bb3c037 --- /dev/null +++ b/lib/private/Files/ObjectStore/Azure.php @@ -0,0 +1,120 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use MicrosoftAzure\Storage\Blob\BlobRestProxy; +use MicrosoftAzure\Storage\Blob\Models\CreateBlockBlobOptions; +use MicrosoftAzure\Storage\Common\Exceptions\ServiceException; +use OCP\Files\ObjectStore\IObjectStore; + +class Azure implements IObjectStore { + /** @var string */ + private $containerName; + /** @var string */ + private $accountName; + /** @var string */ + private $accountKey; + /** @var BlobRestProxy|null */ + private $blobClient = null; + /** @var string|null */ + private $endpoint = null; + /** @var bool */ + private $autoCreate = false; + + /** + * @param array $parameters + */ + public function __construct(array $parameters) { + $this->containerName = $parameters['container']; + $this->accountName = $parameters['account_name']; + $this->accountKey = $parameters['account_key']; + if (isset($parameters['endpoint'])) { + $this->endpoint = $parameters['endpoint']; + } + if (isset($parameters['autocreate'])) { + $this->autoCreate = $parameters['autocreate']; + } + } + + /** + * @return BlobRestProxy + */ + private function getBlobClient() { + if (!$this->blobClient) { + $protocol = $this->endpoint ? substr($this->endpoint, 0, strpos($this->endpoint, ':')) : 'https'; + $connectionString = 'DefaultEndpointsProtocol=' . $protocol . ';AccountName=' . $this->accountName . ';AccountKey=' . $this->accountKey; + if ($this->endpoint) { + $connectionString .= ';BlobEndpoint=' . $this->endpoint; + } + $this->blobClient = BlobRestProxy::createBlobService($connectionString); + + if ($this->autoCreate) { + try { + $this->blobClient->createContainer($this->containerName); + } catch (ServiceException $e) { + if ($e->getCode() === 409) { + // already exists + } else { + throw $e; + } + } + } + } + return $this->blobClient; + } + + /** + * @return string the container or bucket name where objects are stored + */ + public function getStorageId() { + return 'azure::blob::' . $this->containerName; + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + */ + public function readObject($urn) { + $blob = $this->getBlobClient()->getBlob($this->containerName, $urn); + return $blob->getContentStream(); + } + + public function writeObject($urn, $stream, ?string $mimetype = null) { + $options = new CreateBlockBlobOptions(); + if ($mimetype) { + $options->setContentType($mimetype); + } + $this->getBlobClient()->createBlockBlob($this->containerName, $urn, $stream, $options); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + */ + public function deleteObject($urn) { + $this->getBlobClient()->deleteBlob($this->containerName, $urn); + } + + public function objectExists($urn) { + try { + $this->getBlobClient()->getBlobMetadata($this->containerName, $urn); + return true; + } catch (ServiceException $e) { + if ($e->getCode() === 404) { + return false; + } else { + throw $e; + } + } + } + + public function copyObject($from, $to) { + $this->getBlobClient()->copyBlob($this->containerName, $to, $this->containerName, $from); + } +} diff --git a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php new file mode 100644 index 00000000000..4e2d10705fe --- /dev/null +++ b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php @@ -0,0 +1,42 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use Exception; +use OCP\Files\IHomeStorage; +use OCP\IUser; + +class HomeObjectStoreStorage extends ObjectStoreStorage implements IHomeStorage { + protected IUser $user; + + /** + * The home user storage requires a user object to create a unique storage id + * + * @param array $parameters + * @throws Exception + */ + public function __construct(array $parameters) { + if (! isset($parameters['user']) || ! $parameters['user'] instanceof IUser) { + throw new Exception('missing user object in parameters'); + } + $this->user = $parameters['user']; + parent::__construct($parameters); + } + + public function getId(): string { + return 'object::user:' . $this->user->getUID(); + } + + public function getOwner(string $path): string|false { + return $this->user->getUID(); + } + + public function getUser(): IUser { + return $this->user; + } +} diff --git a/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php new file mode 100644 index 00000000000..369182b069d --- /dev/null +++ b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php @@ -0,0 +1,13 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\ObjectStore; + +class InvalidObjectStoreConfigurationException extends \Exception { + +} diff --git a/lib/private/Files/ObjectStore/Mapper.php b/lib/private/Files/ObjectStore/Mapper.php new file mode 100644 index 00000000000..e1174a285a6 --- /dev/null +++ b/lib/private/Files/ObjectStore/Mapper.php @@ -0,0 +1,54 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use OCP\IConfig; +use OCP\IUser; + +/** + * Class Mapper + * + * @package OC\Files\ObjectStore + * + * Map a user to a bucket. + */ +class Mapper { + /** @var IUser */ + private $user; + + /** @var IConfig */ + private $config; + + /** + * Mapper constructor. + * + * @param IUser $user + * @param IConfig $config + */ + public function __construct(IUser $user, IConfig $config) { + $this->user = $user; + $this->config = $config; + } + + /** + * @param int $numBuckets + * @return string + */ + public function getBucket($numBuckets = 64) { + // Get the bucket config and shift if provided. + // Allow us to prevent writing in old filled buckets + $config = $this->config->getSystemValue('objectstore_multibucket'); + $minBucket = is_array($config) && isset($config['arguments']['min_bucket']) + ? (int)$config['arguments']['min_bucket'] + : 0; + + $hash = md5($this->user->getUID()); + $num = hexdec(substr($hash, 0, 4)); + return (string)(($num % ($numBuckets - $minBucket)) + $minBucket); + } +} diff --git a/lib/private/Files/ObjectStore/ObjectStoreScanner.php b/lib/private/Files/ObjectStore/ObjectStoreScanner.php new file mode 100644 index 00000000000..5c3992b8458 --- /dev/null +++ b/lib/private/Files/ObjectStore/ObjectStoreScanner.php @@ -0,0 +1,79 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use OC\Files\Cache\Scanner; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\FileInfo; + +class ObjectStoreScanner extends Scanner { + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + return null; + } + + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + return null; + } + + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + return 0; + } + + public function backgroundScan() { + $lastPath = null; + // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck) + // we sort by path DESC to ensure that contents of a folder are handled before the parent folder + while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function () use ($path) { + $item = $this->cache->get($path); + if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) { + $fh = $this->storage->fopen($path, 'r'); + if ($fh) { + $stat = fstat($fh); + if ($stat['size']) { + $this->cache->update($item->getId(), ['size' => $stat['size']]); + } + } + } + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } + } + + /** + * Unlike the default Cache::getIncomplete this one sorts by path. + * + * This is needed since self::backgroundScan doesn't fix child entries when running on a parent folder. + * By sorting by path we ensure that we encounter the child entries first. + * + * @return false|string + * @throws \OCP\DB\Exception + */ + private function getIncomplete() { + $query = $this->connection->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->orderBy('path', 'DESC') + ->setMaxResults(1); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + if ($path === false) { + return false; + } + + // Make sure Oracle does not continue with null for empty strings + return (string)$path; + } +} diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php new file mode 100644 index 00000000000..9ab11f8a3df --- /dev/null +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -0,0 +1,815 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use Aws\S3\Exception\S3Exception; +use Aws\S3\Exception\S3MultipartUploadException; +use Icewind\Streams\CallbackWrapper; +use Icewind\Streams\CountWrapper; +use Icewind\Streams\IteratorDirectory; +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheEntry; +use OC\Files\Storage\PolyFill\CopyDirectory; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; +use OCP\Files\FileInfo; +use OCP\Files\GenericFileException; +use OCP\Files\NotFoundException; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; +use OCP\Files\Storage\IChunkedFileWrite; +use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; + +class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { + use CopyDirectory; + + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; + + private LoggerInterface $logger; + + private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + private bool $preserveCacheItemsOnDelete = false; + + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) { + $this->objectStore = $parameters['objectstore']; + } else { + throw new \Exception('missing IObjectStore instance'); + } + if (isset($parameters['storageid'])) { + $this->id = 'object::store:' . $parameters['storageid']; + } else { + $this->id = 'object::store:' . $this->objectStore->getStorageId(); + } + if (isset($parameters['objectPrefix'])) { + $this->objectPrefix = $parameters['objectPrefix']; + } + if (isset($parameters['validateWrites'])) { + $this->validateWrites = (bool)$parameters['validateWrites']; + } + $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false); + + $this->logger = \OCP\Server::get(LoggerInterface::class); + } + + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { + $path = $this->normalizePath($path); + if (!$force && $this->file_exists($path)) { + $this->logger->warning("Tried to create an object store folder that already exists: $path"); + return false; + } + + $mTime = time(); + $data = [ + 'mimetype' => 'httpd/unix-directory', + 'size' => $metadata['size'] ?? 0, + 'mtime' => $mTime, + 'storage_mtime' => $mTime, + 'permissions' => \OCP\Constants::PERMISSION_ALL, + ]; + if ($path === '') { + //create root on the fly + $data['etag'] = $this->getETag(''); + $this->getCache()->put('', $data); + return true; + } else { + // if parent does not exist, create it + $parent = $this->normalizePath(dirname($path)); + $parentType = $this->filetype($parent); + if ($parentType === false) { + if (!$this->mkdir($parent)) { + // something went wrong + $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created"); + return false; + } + } elseif ($parentType === 'file') { + // parent is a file + $this->logger->warning("Parent ($parent) is a file"); + return false; + } + // finally create the new dir + $mTime = time(); // update mtime + $data['mtime'] = $mTime; + $data['storage_mtime'] = $mTime; + $data['etag'] = $this->getETag($path); + $this->getCache()->put($path, $data); + return true; + } + } + + private function normalizePath(string $path): string { + $path = trim($path, '/'); + //FIXME why do we sometimes get a path like 'files//username'? + $path = str_replace('//', '/', $path); + + // dirname('/folder') returns '.' but internally (in the cache) we store the root as '' + if (!$path || $path === '.') { + $path = ''; + } + + return $path; + } + + /** + * Object Stores use a NoopScanner because metadata is directly stored in + * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. + */ + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { + if (!$storage) { + $storage = $this; + } + if (!isset($this->scanner)) { + $this->scanner = new ObjectStoreScanner($storage); + } + /** @var \OC\Files\ObjectStore\ObjectStoreScanner */ + return $this->scanner; + } + + public function getId(): string { + return $this->id; + } + + public function rmdir(string $path): bool { + $path = $this->normalizePath($path); + $entry = $this->getCache()->get($path); + + if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) { + return false; + } + + return $this->rmObjects($entry); + } + + private function rmObjects(ICacheEntry $entry): bool { + $children = $this->getCache()->getFolderContentsById($entry->getId()); + foreach ($children as $child) { + if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + if (!$this->rmObjects($child)) { + return false; + } + } else { + if (!$this->rmObject($child)) { + return false; + } + } + } + + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + + return true; + } + + public function unlink(string $path): bool { + $path = $this->normalizePath($path); + $entry = $this->getCache()->get($path); + + if ($entry instanceof ICacheEntry) { + if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + return $this->rmObjects($entry); + } else { + return $this->rmObject($entry); + } + } + return false; + } + + public function rmObject(ICacheEntry $entry): bool { + try { + $this->objectStore->deleteObject($this->getURN($entry->getId())); + } catch (\Exception $ex) { + if ($ex->getCode() !== 404) { + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); + return false; + } + //removing from cache is ok as it does not exist in the objectstore anyway + } + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + return true; + } + + public function stat(string $path): array|false { + $path = $this->normalizePath($path); + $cacheEntry = $this->getCache()->get($path); + if ($cacheEntry instanceof CacheEntry) { + return $cacheEntry->getData(); + } else { + if ($path === '') { + $this->mkdir('', true); + $cacheEntry = $this->getCache()->get($path); + if ($cacheEntry instanceof CacheEntry) { + return $cacheEntry->getData(); + } + } + return false; + } + } + + public function getPermissions(string $path): int { + $stat = $this->stat($path); + + if (is_array($stat) && isset($stat['permissions'])) { + return $stat['permissions']; + } + + return parent::getPermissions($path); + } + + /** + * Override this method if you need a different unique resource identifier for your object storage implementation. + * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. + * You may need a mapping table to store your URN if it cannot be generated from the fileid. + * + * @return string the unified resource name used to identify the object + */ + public function getURN(int $fileId): string { + return $this->objectPrefix . $fileId; + } + + public function opendir(string $path) { + $path = $this->normalizePath($path); + + try { + $files = []; + $folderContents = $this->getCache()->getFolderContents($path); + foreach ($folderContents as $file) { + $files[] = $file['name']; + } + + return IteratorDirectory::wrap($files); + } catch (\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + return false; + } + } + + public function filetype(string $path): string|false { + $path = $this->normalizePath($path); + $stat = $this->stat($path); + if ($stat) { + if ($stat['mimetype'] === 'httpd/unix-directory') { + return 'dir'; + } + return 'file'; + } else { + return false; + } + } + + public function fopen(string $path, string $mode) { + $path = $this->normalizePath($path); + + if (strrpos($path, '.') !== false) { + $ext = substr($path, strrpos($path, '.')); + } else { + $ext = ''; + } + + switch ($mode) { + case 'r': + case 'rb': + $stat = $this->stat($path); + if (is_array($stat)) { + $filesize = $stat['size'] ?? 0; + // Reading 0 sized files is a waste of time + if ($filesize === 0) { + return fopen('php://memory', $mode); + } + + try { + $handle = $this->objectStore->readObject($this->getURN($stat['fileid'])); + if ($handle === false) { + return false; // keep backward compatibility + } + $streamStat = fstat($handle); + $actualSize = $streamStat['size'] ?? -1; + if ($actualSize > -1 && $actualSize !== $filesize) { + $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]); + } + return $handle; + } catch (NotFoundException $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); + throw $e; + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); + return false; + } + } else { + return false; + } + // no break + case 'w': + case 'wb': + case 'w+': + case 'wb+': + $dirName = dirname($path); + $parentExists = $this->is_dir($dirName); + if (!$parentExists) { + return false; + } + + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext); + $handle = fopen($tmpFile, $mode); + return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { + $this->writeBack($tmpFile, $path); + unlink($tmpFile); + }); + case 'a': + case 'ab': + case 'r+': + case 'a+': + case 'x': + case 'x+': + case 'c': + case 'c+': + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext); + if ($this->file_exists($path)) { + $source = $this->fopen($path, 'r'); + file_put_contents($tmpFile, $source); + } + $handle = fopen($tmpFile, $mode); + return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { + $this->writeBack($tmpFile, $path); + unlink($tmpFile); + }); + } + return false; + } + + public function file_exists(string $path): bool { + $path = $this->normalizePath($path); + return (bool)$this->stat($path); + } + + public function rename(string $source, string $target): bool { + $source = $this->normalizePath($source); + $target = $this->normalizePath($target); + $this->remove($target); + $this->getCache()->move($source, $target); + $this->touch(dirname($target)); + return true; + } + + public function getMimeType(string $path): string|false { + $path = $this->normalizePath($path); + return parent::getMimeType($path); + } + + public function touch(string $path, ?int $mtime = null): bool { + if (is_null($mtime)) { + $mtime = time(); + } + + $path = $this->normalizePath($path); + $dirName = dirname($path); + $parentExists = $this->is_dir($dirName); + if (!$parentExists) { + return false; + } + + $stat = $this->stat($path); + if (is_array($stat)) { + // update existing mtime in db + $stat['mtime'] = $mtime; + $this->getCache()->update($stat['fileid'], $stat); + } else { + try { + //create a empty file, need to have at least on char to make it + // work with all object storage implementations + $this->file_put_contents($path, ' '); + } catch (\Exception $ex) { + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); + throw $ex; + } + } + return true; + } + + public function writeBack(string $tmpFile, string $path) { + $size = filesize($tmpFile); + $this->writeStream($path, fopen($tmpFile, 'r'), $size); + } + + public function hasUpdated(string $path, int $time): bool { + return false; + } + + public function needsPartFile(): bool { + return false; + } + + public function file_put_contents(string $path, mixed $data): int { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $data); + rewind($fh); + return $this->writeStream($path, $fh, strlen($data)); + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + if ($size === null) { + $stats = fstat($stream); + if (is_array($stats) && isset($stats['size'])) { + $size = $stats['size']; + } + } + + $stat = $this->stat($path); + if (empty($stat)) { + // create new file + $stat = [ + 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, + ]; + } + // update stat with new data + $mTime = time(); + $stat['size'] = (int)$size; + $stat['mtime'] = $mTime; + $stat['storage_mtime'] = $mTime; + + $mimetypeDetector = \OC::$server->getMimeTypeDetector(); + $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } + + $stat['mimetype'] = $mimetype; + $stat['etag'] = $this->getETag($path); + $stat['checksum'] = ''; + + $exists = $this->getCache()->inCache($path); + $uploadPath = $exists ? $path : $path . '.part'; + + if ($exists) { + $fileId = $stat['fileid']; + } else { + $parent = $this->normalizePath(dirname($path)); + if (!$this->is_dir($parent)) { + throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)"); + } + $fileId = $this->getCache()->put($uploadPath, $stat); + } + + $urn = $this->getURN($fileId); + try { + //upload to object storage + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { + $this->getCache()->update($fileId, [ + 'size' => $writtenSize, + ]); + } + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); + } else { + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); + } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; + } catch (\Exception $ex) { + if (!$exists) { + /* + * Only remove the entry if we are dealing with a new file. + * Else people lose access to existing files + */ + $this->getCache()->remove($uploadPath); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); + } else { + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); + } + throw new GenericFileException('Error while writing stream to object store', 0, $ex); + } + + if ($exists) { + // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways + $stat['unencrypted_size'] = $stat['size']; + $this->getCache()->update($fileId, $stat); + } else { + if (!$this->validateWrites || $this->objectStore->objectExists($urn)) { + $this->getCache()->move($uploadPath, $path); + } else { + $this->getCache()->remove($uploadPath); + throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404); + } + } + + return $totalWritten; + } + + public function getObjectStore(): IObjectStore { + return $this->objectStore; + } + + public function copyFromStorage( + IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime = false, + ): bool { + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { + /** @var CacheEntry $sourceEntry */ + $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath); + $sourceEntryData = $sourceEntry->getData(); + // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current + // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the + // unjailed storage. + if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) { + $sourceEntry['permissions'] = $sourceEntryData['scan_permissions']; + } + $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath); + return true; + } + } + + return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { + $sourceCache = $sourceStorage->getCache(); + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + // Do not import any data when source and target bucket are identical. + return true; + } + if (!$sourceCacheEntry) { + $sourceCacheEntry = $sourceCache->get($sourceInternalPath); + } + + $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry); + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $sourceStorage->rmdir($sourceInternalPath); + } else { + $sourceStorage->unlink($sourceInternalPath); + } + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + + return true; + } + + /** + * Copy the object(s) of a file or folder into this storage, without touching the cache + */ + private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) { + $copiedFiles = []; + try { + foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) { + $sourceStream = $sourceStorage->fopen($file->getPath(), 'r'); + if (!$sourceStream) { + throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})"); + } + $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType()); + if (is_resource($sourceStream)) { + fclose($sourceStream); + } + $copiedFiles[] = $file->getId(); + } + } catch (\Exception $e) { + foreach ($copiedFiles as $fileId) { + try { + $this->objectStore->deleteObject($this->getURN($fileId)); + } catch (\Exception $e) { + // ignore + } + } + throw $e; + } + } + + /** + * @return \Iterator<ICacheEntry> + */ + private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + foreach ($cache->getFolderContentsById($entry->getId()) as $child) { + yield from $this->getAllChildObjects($cache, $child); + } + } else { + yield $entry; + } + } + + public function copy(string $source, string $target): bool { + $source = $this->normalizePath($source); + $target = $this->normalizePath($target); + + $cache = $this->getCache(); + $sourceEntry = $cache->get($source); + if (!$sourceEntry) { + throw new NotFoundException('Source object not found'); + } + + $this->copyInner($cache, $sourceEntry, $target); + + return true; + } + + private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) { + $cache = $this->getCache(); + + if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + if ($cache->inCache($to)) { + $cache->remove($to); + } + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); + + foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { + $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); + } + } else { + $this->copyFile($sourceEntry, $to); + } + } + + private function copyFile(ICacheEntry $sourceEntry, string $to) { + $cache = $this->getCache(); + + $sourceUrn = $this->getURN($sourceEntry->getId()); + + if (!$cache instanceof Cache) { + throw new \Exception('Invalid source cache for object store copy'); + } + + $targetId = $cache->copyFromCache($cache, $sourceEntry, $to); + + $targetUrn = $this->getURN($targetId); + + try { + $this->objectStore->copyObject($sourceUrn, $targetUrn); + if ($this->handleCopiesAsOwned) { + // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage ! + $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]); + } + } catch (\Exception $e) { + $cache->remove($to); + + throw $e; + } + } + + public function startChunkedWrite(string $targetPath): string { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + return $this->objectStore->initiateMultipartUpload($urn); + } + + /** + * @throws GenericFileException + */ + public function putChunkedWritePart( + string $targetPath, + string $writeToken, + string $chunkId, + $data, + $size = null, + ): ?array { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + + $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size); + + $parts[$chunkId] = [ + 'PartNumber' => $chunkId, + 'ETag' => trim($result->get('ETag'), '"'), + ]; + return $parts[$chunkId]; + } + + public function completeChunkedWrite(string $targetPath, string $writeToken): int { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $parts = $this->objectStore->getMultipartUploads($urn, $writeToken); + $sortedParts = array_values($parts); + sort($sortedParts); + try { + $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts); + $stat = $this->stat($targetPath); + $mtime = time(); + if (is_array($stat)) { + $stat['size'] = $size; + $stat['mtime'] = $mtime; + $stat['mimetype'] = $this->getMimeType($targetPath); + $this->getCache()->update($stat['fileid'], $stat); + } + } catch (S3MultipartUploadException|S3Exception $e) { + $this->objectStore->abortMultipartUpload($urn, $writeToken); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); + throw new GenericFileException('Could not write chunked file'); + } + return $size; + } + + public function cancelChunkedWrite(string $targetPath, string $writeToken): void { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $this->objectStore->abortMultipartUpload($urn, $writeToken); + } + + public function setPreserveCacheOnDelete(bool $preserve) { + $this->preserveCacheItemsOnDelete = $preserve; + } +} diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php new file mode 100644 index 00000000000..ffc33687340 --- /dev/null +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -0,0 +1,225 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\ObjectStore; + +use OCP\App\IAppManager; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +use OCP\IUser; + +/** + * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}} + */ +class PrimaryObjectStoreConfig { + public function __construct( + private readonly IConfig $config, + private readonly IAppManager $appManager, + ) { + } + + /** + * @param ObjectStoreConfig $config + */ + public function buildObjectStore(array $config): IObjectStore { + return new $config['class']($config['arguments']); + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForRoot(): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $config = $this->getObjectStoreConfiguration('root'); + + if ($config['arguments']['multibucket']) { + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + // put the root FS always in first bucket for multibucket configuration + $config['arguments']['bucket'] .= '0'; + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForUser(IUser $user): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $store = $this->getObjectStoreForUser($user); + $config = $this->getObjectStoreConfiguration($store); + + if ($config['arguments']['multibucket']) { + $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); + } + return $config; + } + + /** + * @param string $name + * @return ObjectStoreConfig + */ + public function getObjectStoreConfiguration(string $name): array { + $configs = $this->getObjectStoreConfigs(); + $name = $this->resolveAlias($name); + if (!isset($configs[$name])) { + throw new \Exception("Object store configuration for '$name' not found"); + } + if (is_string($configs[$name])) { + throw new \Exception("Object store configuration for '{$configs[$name]}' not found"); + } + return $configs[$name]; + } + + public function resolveAlias(string $name): string { + $configs = $this->getObjectStoreConfigs(); + + while (isset($configs[$name]) && is_string($configs[$name])) { + $name = $configs[$name]; + } + return $name; + } + + public function hasObjectStore(): bool { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + return $objectStore || $objectStoreMultiBucket; + } + + public function hasMultipleObjectStorages(): bool { + $objectStore = $this->config->getSystemValue('objectstore', []); + return isset($objectStore['default']); + } + + /** + * @return ?array<string, ObjectStoreConfig|string> + * @throws InvalidObjectStoreConfigurationException + */ + public function getObjectStoreConfigs(): ?array { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + + // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config + if ($objectStoreMultiBucket) { + $objectStoreMultiBucket['arguments']['multibucket'] = true; + return [ + 'default' => 'server1', + 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), + 'root' => 'server1', + ]; + } elseif ($objectStore) { + if (!isset($objectStore['default'])) { + $objectStore = [ + 'default' => 'server1', + 'root' => 'server1', + 'server1' => $objectStore, + ]; + } + if (!isset($objectStore['root'])) { + $objectStore['root'] = 'default'; + } + + if (!is_string($objectStore['default'])) { + throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); + } + return array_map($this->validateObjectStoreConfig(...), $objectStore); + } else { + return null; + } + } + + /** + * @param array|string $config + * @return string|ObjectStoreConfig + */ + private function validateObjectStoreConfig(array|string $config): array|string { + if (is_string($config)) { + return $config; + } + if (!isset($config['class'])) { + throw new InvalidObjectStoreConfigurationException('No class configured for object store'); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $class = $config['class']; + $arguments = $config['arguments']; + if (!is_array($arguments)) { + throw new InvalidObjectStoreConfigurationException('Configured object store arguments are not an array'); + } + if (!isset($arguments['multibucket'])) { + $arguments['multibucket'] = false; + } + if (!is_bool($arguments['multibucket'])) { + throw new InvalidObjectStoreConfigurationException('arguments.multibucket must be a boolean in object store configuration'); + } + + if (!is_string($class)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not a string'); + } + + if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { + [$appId] = explode('\\', $class); + $this->appManager->loadApp(strtolower($appId)); + } + + if (!is_a($class, IObjectStore::class, true)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not an object store'); + } + return [ + 'class' => $class, + 'arguments' => $arguments, + ]; + } + + public function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->getSetBucketForUser($user); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new Mapper($user, $this->config); + $numBuckets = $config['arguments']['num_buckets'] ?? 64; + $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); + } + + return $bucket; + } + + public function getSetBucketForUser(IUser $user): ?string { + return $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + } + + public function getObjectStoreForUser(IUser $user): string { + if ($this->hasMultipleObjectStorages()) { + $value = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null); + if ($value === null) { + $value = $this->resolveAlias('default'); + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $value); + } + return $value; + } else { + return 'default'; + } + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php new file mode 100644 index 00000000000..72e1751e23d --- /dev/null +++ b/lib/private/Files/ObjectStore/S3.php @@ -0,0 +1,142 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\ObjectStore; + +use Aws\Result; +use Exception; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; + +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { + use S3ConnectionTrait; + use S3ObjectTrait; + + public function __construct(array $parameters) { + $parameters['primary_storage'] = true; + $this->parseParams($parameters); + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + public function getStorageId() { + return $this->id; + } + + public function initiateMultipartUpload(string $urn): string { + $upload = $this->getConnection()->createMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ] + $this->getSSECParameters()); + $uploadId = $upload->get('UploadId'); + if ($uploadId === null) { + throw new Exception('No upload id returned'); + } + return (string)$uploadId; + } + + public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result { + return $this->getConnection()->uploadPart([ + 'Body' => $stream, + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'ContentLength' => $size, + 'PartNumber' => $partId, + 'UploadId' => $uploadId, + ] + $this->getSSECParameters()); + } + + public function getMultipartUploads(string $urn, string $uploadId): array { + $parts = []; + $isTruncated = true; + $partNumberMarker = 0; + + while ($isTruncated) { + $result = $this->getConnection()->listParts([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + 'MaxParts' => 1000, + 'PartNumberMarker' => $partNumberMarker, + ] + $this->getSSECParameters()); + $parts = array_merge($parts, $result->get('Parts') ?? []); + $isTruncated = $result->get('IsTruncated'); + $partNumberMarker = $result->get('NextPartNumberMarker'); + } + + return $parts; + } + + public function completeMultipartUpload(string $urn, string $uploadId, array $result): int { + $this->getConnection()->completeMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + 'MultipartUpload' => ['Parts' => $result], + ] + $this->getSSECParameters()); + $stat = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ] + $this->getSSECParameters()); + return (int)$stat->get('ContentLength'); + } + + public function abortMultipartUpload($urn, $uploadId): void { + $this->getConnection()->abortMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + ]); + } + + private function parseS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + if (str_starts_with($key, 'x-amz-meta-')) { + $result[substr($key, strlen('x-amz-meta-'))] = $value; + } + } + return $result; + } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ] + $this->parseS3Metadata($object['Metadata'] ?? []); + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } +} diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php new file mode 100644 index 00000000000..5b086db8f77 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -0,0 +1,41 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +/** + * Shared configuration between ConnectionTrait and ObjectTrait to ensure both to be in sync + */ +trait S3ConfigTrait { + protected array $params; + + protected string $bucket; + + /** Maximum number of concurrent multipart uploads */ + protected int $concurrency; + + /** Timeout, in seconds, for the connection to S3 server, not for the + * request. */ + protected float $connectTimeout; + + protected int $timeout; + + protected string|false $proxy; + + protected string $storageClass; + + /** @var int Part size in bytes (float is added for 32bit support) */ + protected int|float $uploadPartSize; + + /** @var int Limit on PUT in bytes (float is added for 32bit support) */ + private int|float $putSizeLimit; + + /** @var int Limit on COPY in bytes (float is added for 32bit support) */ + private int|float $copySizeLimit; + + private bool $useMultipartCopy = true; +} diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php new file mode 100644 index 00000000000..67b82a44ab7 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -0,0 +1,251 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use Aws\ClientResolver; +use Aws\Credentials\CredentialProvider; +use Aws\Credentials\Credentials; +use Aws\Exception\CredentialsException; +use Aws\S3\Exception\S3Exception; +use Aws\S3\S3Client; +use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; +use OCP\Files\StorageNotAvailableException; +use OCP\ICertificateManager; +use OCP\Server; +use Psr\Log\LoggerInterface; + +trait S3ConnectionTrait { + use S3ConfigTrait; + + protected string $id; + + protected bool $test; + + protected ?S3Client $connection = null; + + protected function parseParams($params) { + if (empty($params['bucket'])) { + throw new \Exception('Bucket has to be configured.'); + } + + $this->id = 'amazon::' . $params['bucket']; + + $this->test = isset($params['test']); + $this->bucket = $params['bucket']; + // Default to 5 like the S3 SDK does + $this->concurrency = $params['concurrency'] ?? 5; + $this->proxy = $params['proxy'] ?? false; + $this->connectTimeout = $params['connect_timeout'] ?? 5; + $this->timeout = $params['timeout'] ?? 15; + $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; + $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; + $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600; + $this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000; + $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true); + $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; + $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; + $params['s3-accelerate'] = $params['hostname'] === 's3-accelerate.amazonaws.com' || $params['hostname'] === 's3-accelerate.dualstack.amazonaws.com'; + if (!isset($params['port']) || $params['port'] === '') { + $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; + } + $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true; + + if ($params['s3-accelerate']) { + $params['verify_bucket_exists'] = false; + } + + $this->params = $params; + } + + public function getBucket() { + return $this->bucket; + } + + public function getProxy() { + return $this->proxy; + } + + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ + public function getConnection() { + if ($this->connection !== null) { + return $this->connection; + } + + $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; + $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; + + // Adding explicit credential provider to the beginning chain. + // Including default credential provider (skipping AWS shared config files). + $provider = CredentialProvider::memoize( + CredentialProvider::chain( + $this->paramCredentialProvider(), + CredentialProvider::defaultProvider(['use_aws_shared_config_files' => false]) + ) + ); + + $options = [ + 'version' => $this->params['version'] ?? 'latest', + 'credentials' => $provider, + 'endpoint' => $base_url, + 'region' => $this->params['region'], + 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, + 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()), + 'csm' => false, + 'use_arn_region' => false, + 'http' => [ + 'verify' => $this->getCertificateBundlePath(), + 'connect_timeout' => $this->connectTimeout, + ], + 'use_aws_shared_config_files' => false, + 'retries' => [ + 'mode' => 'standard', + 'max_attempts' => 5, + ], + ]; + + if ($this->params['s3-accelerate']) { + $options['use_accelerate_endpoint'] = true; + } else { + $options['endpoint'] = $base_url; + } + + if ($this->getProxy()) { + $options['http']['proxy'] = $this->getProxy(); + } + if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) { + $options['signature_version'] = 'v2'; + } + $this->connection = new S3Client($options); + + try { + $logger = Server::get(LoggerInterface::class); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', + ['app' => 'objectstore']); + } + + if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { + try { + $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket); + } + $this->connection->createBucket(['Bucket' => $this->bucket]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->debug('Invalid remote storage.', [ + 'exception' => $e, + 'app' => 'objectstore', + ]); + if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') { + throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); + } + } + } + + // google cloud's s3 compatibility doesn't like the EncodingType parameter + if (strpos($base_url, 'storage.googleapis.com')) { + $this->connection->getHandlerList()->remove('s3.auto_encode'); + } + } catch (S3Exception $e) { + throw new StorageNotAvailableException('S3 service is unable to handle request: ' . $e->getMessage()); + } + + return $this->connection; + } + + /** + * when running the tests wait to let the buckets catch up + */ + private function testTimeout() { + if ($this->test) { + sleep($this->timeout); + } + } + + public static function legacySignatureProvider($version, $service, $region) { + switch ($version) { + case 'v2': + case 's3': + return new S3Signature(); + default: + return null; + } + } + + /** + * This function creates a credential provider based on user parameter file + */ + protected function paramCredentialProvider(): callable { + return function () { + $key = empty($this->params['key']) ? null : $this->params['key']; + $secret = empty($this->params['secret']) ? null : $this->params['secret']; + $sessionToken = empty($this->params['session_token']) ? null : $this->params['session_token']; + + if ($key && $secret) { + return Create::promiseFor( + // a null sessionToken match the default signature of the constructor + new Credentials($key, $secret, $sessionToken) + ); + } + + $msg = 'Could not find parameters set for credentials in config file.'; + return new RejectedPromise(new CredentialsException($msg)); + }; + } + + protected function getCertificateBundlePath(): ?string { + if ((int)($this->params['use_nextcloud_bundle'] ?? '0')) { + // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage + if (!isset($this->params['primary_storage'])) { + /** @var ICertificateManager $certManager */ + $certManager = Server::get(ICertificateManager::class); + return $certManager->getAbsoluteBundlePath(); + } else { + return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt'; + } + } else { + return null; + } + } + + protected function getSSECKey(): ?string { + if (isset($this->params['sse_c_key']) && !empty($this->params['sse_c_key'])) { + return $this->params['sse_c_key']; + } + + return null; + } + + protected function getSSECParameters(bool $copy = false): array { + $key = $this->getSSECKey(); + + if ($key === null) { + return []; + } + + $rawKey = base64_decode($key); + if ($copy) { + return [ + 'CopySourceSSECustomerAlgorithm' => 'AES256', + 'CopySourceSSECustomerKey' => $rawKey, + 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true) + ]; + } + return [ + 'SSECustomerAlgorithm' => 'AES256', + 'SSECustomerKey' => $rawKey, + 'SSECustomerKeyMD5' => md5($rawKey, true) + ]; + } +} diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php new file mode 100644 index 00000000000..89405de2e8e --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -0,0 +1,282 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use Aws\Command; +use Aws\Exception\MultipartUploadException; +use Aws\S3\Exception\S3MultipartUploadException; +use Aws\S3\MultipartCopy; +use Aws\S3\MultipartUploader; +use Aws\S3\S3Client; +use GuzzleHttp\Psr7; +use GuzzleHttp\Psr7\Utils; +use OC\Files\Stream\SeekableHttpStream; +use Psr\Http\Message\StreamInterface; + +trait S3ObjectTrait { + use S3ConfigTrait; + + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ + abstract protected function getConnection(); + + abstract protected function getCertificateBundlePath(): ?string; + abstract protected function getSSECParameters(bool $copy = false): array; + + /** + * @param string $urn the unified resource name used to identify the object + * + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + public function readObject($urn) { + $fh = SeekableHttpStream::open(function ($range) use ($urn) { + $command = $this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Range' => 'bytes=' . $range, + ] + $this->getSSECParameters()); + $request = \Aws\serialize($command); + $headers = []; + foreach ($request->getHeaders() as $key => $values) { + foreach ($values as $value) { + $headers[] = "$key: $value"; + } + } + $opts = [ + 'http' => [ + 'protocol_version' => $request->getProtocolVersion(), + 'header' => $headers, + ] + ]; + $bundle = $this->getCertificateBundlePath(); + if ($bundle) { + $opts['ssl'] = [ + 'cafile' => $bundle + ]; + } + + if ($this->getProxy()) { + $opts['http']['proxy'] = $this->getProxy(); + $opts['http']['request_fulluri'] = true; + } + + $context = stream_context_create($opts); + return fopen($request->getUri(), 'r', false, $context); + }); + if (!$fh) { + throw new \Exception("Failed to read object $urn"); + } + return $fh; + } + + private function buildS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + $result['x-amz-meta-' . $key] = $value; + } + return $result; + } + + /** + * Single object put helper + * + * @param string $urn the unified resource name used to identify the object + * @param StreamInterface $stream stream with the data to write + * @param array $metaData the metadata to set for the object + * @throws \Exception when something goes wrong, message will be logged + */ + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $args = [ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream, + 'ACL' => 'private', + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); + } + + + /** + * Multipart upload helper that tries to avoid orphaned fragments in S3 + * + * @param string $urn the unified resource name used to identify the object + * @param StreamInterface $stream stream with the data to write + * @param array $metaData the metadata to set for the object + * @throws \Exception when something goes wrong, message will be logged + */ + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $attempts = 0; + $uploaded = false; + $concurrency = $this->concurrency; + $exception = null; + $state = null; + $size = $stream->getSize(); + $totalWritten = 0; + + // retry multipart upload once with concurrency at half on failure + while (!$uploaded && $attempts <= 1) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'concurrency' => $concurrency, + 'key' => $urn, + 'part_size' => $this->uploadPartSize, + 'state' => $state, + 'params' => [ + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, + ]); + + try { + $uploader->upload(); + $uploaded = true; + } catch (S3MultipartUploadException $e) { + $exception = $e; + $attempts++; + + if ($concurrency > 1) { + $concurrency = round($concurrency / 2); + } + + if ($stream->isSeekable()) { + $stream->rewind(); + } + } catch (MultipartUploadException $e) { + $exception = $e; + break; + } + } + + if (!$uploaded) { + // if anything goes wrong with multipart, make sure that you don´t poison and + // slow down s3 bucket with orphaned fragments + $uploadInfo = $exception->getState()->getId(); + if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { + $this->getConnection()->abortMultipartUpload($uploadInfo); + } + + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception); + } + } + + public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } + + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { + $canSeek = fseek($stream, 0, SEEK_CUR) === 0; + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); + + + $size = $psrStream->getSize(); + if ($size === null || !$canSeek) { + // The s3 single-part upload requires the size to be known for the stream. + // So for input streams that don't have a known size, we need to copy (part of) + // the input into a temporary stream so the size can be determined + $buffer = new Psr7\Stream(fopen('php://temp', 'rw+')); + Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); + $buffer->seek(0); + if ($buffer->getSize() < $this->putSizeLimit) { + // buffer is fully seekable, so use it directly for the small upload + $this->writeSingle($urn, $buffer, $metaData); + } else { + $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); + $this->writeMultiPart($urn, $loadStream, $metaData); + } + } else { + if ($size < $this->putSizeLimit) { + $this->writeSingle($urn, $psrStream, $metaData); + } else { + $this->writeMultiPart($urn, $psrStream, $metaData); + } + } + $psrStream->close(); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + public function deleteObject($urn) { + $this->getConnection()->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ]); + } + + public function objectExists($urn) { + return $this->getConnection()->doesObjectExist($this->bucket, $urn, $this->getSSECParameters()); + } + + public function copyObject($from, $to, array $options = []) { + $sourceMetadata = $this->getConnection()->headObject([ + 'Bucket' => $this->getBucket(), + 'Key' => $from, + ] + $this->getSSECParameters()); + + $size = (int)($sourceMetadata->get('Size') ?? $sourceMetadata->get('ContentLength')); + + if ($this->useMultipartCopy && $size > $this->copySizeLimit) { + $copy = new MultipartCopy($this->getConnection(), [ + 'source_bucket' => $this->getBucket(), + 'source_key' => $from + ], array_merge([ + 'bucket' => $this->getBucket(), + 'key' => $to, + 'acl' => 'private', + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'source_metadata' => $sourceMetadata + ], $options)); + $copy->copy(); + } else { + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([ + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'mup_threshold' => PHP_INT_MAX, + ], $options)); + } + } +} diff --git a/lib/private/Files/ObjectStore/S3Signature.php b/lib/private/Files/ObjectStore/S3Signature.php new file mode 100644 index 00000000000..b80382ff67d --- /dev/null +++ b/lib/private/Files/ObjectStore/S3Signature.php @@ -0,0 +1,204 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use Aws\Credentials\CredentialsInterface; +use Aws\S3\S3Client; +use Aws\S3\S3UriParser; +use Aws\Signature\SignatureInterface; +use GuzzleHttp\Psr7; +use Psr\Http\Message\RequestInterface; + +/** + * Legacy Amazon S3 signature implementation + */ +class S3Signature implements SignatureInterface { + /** @var array Query string values that must be signed */ + private $signableQueryString = [ + 'acl', 'cors', 'delete', 'lifecycle', 'location', 'logging', + 'notification', 'partNumber', 'policy', 'requestPayment', + 'response-cache-control', 'response-content-disposition', + 'response-content-encoding', 'response-content-language', + 'response-content-type', 'response-expires', 'restore', 'tagging', + 'torrent', 'uploadId', 'uploads', 'versionId', 'versioning', + 'versions', 'website' + ]; + + /** @var array Sorted headers that must be signed */ + private $signableHeaders = ['Content-MD5', 'Content-Type']; + + /** @var \Aws\S3\S3UriParser S3 URI parser */ + private $parser; + + public function __construct() { + $this->parser = new S3UriParser(); + // Ensure that the signable query string parameters are sorted + sort($this->signableQueryString); + } + + public function signRequest( + RequestInterface $request, + CredentialsInterface $credentials, + ) { + $request = $this->prepareRequest($request, $credentials); + $stringToSign = $this->createCanonicalizedString($request); + $auth = 'AWS ' + . $credentials->getAccessKeyId() . ':' + . $this->signString($stringToSign, $credentials); + + return $request->withHeader('Authorization', $auth); + } + + public function presign( + RequestInterface $request, + CredentialsInterface $credentials, + $expires, + array $options = [], + ) { + $query = []; + // URL encoding already occurs in the URI template expansion. Undo that + // and encode using the same encoding as GET object, PUT object, etc. + $uri = $request->getUri(); + $path = S3Client::encodeKey(rawurldecode($uri->getPath())); + $request = $request->withUri($uri->withPath($path)); + + // Make sure to handle temporary credentials + if ($token = $credentials->getSecurityToken()) { + $request = $request->withHeader('X-Amz-Security-Token', $token); + $query['X-Amz-Security-Token'] = $token; + } + + if ($expires instanceof \DateTime) { + $expires = $expires->getTimestamp(); + } elseif (!is_numeric($expires)) { + $expires = strtotime($expires); + } + + // Set query params required for pre-signed URLs + $query['AWSAccessKeyId'] = $credentials->getAccessKeyId(); + $query['Expires'] = $expires; + $query['Signature'] = $this->signString( + $this->createCanonicalizedString($request, $expires), + $credentials + ); + + // Move X-Amz-* headers to the query string + foreach ($request->getHeaders() as $name => $header) { + $name = strtolower($name); + if (str_starts_with($name, 'x-amz-')) { + $query[$name] = implode(',', $header); + } + } + + $queryString = http_build_query($query, '', '&', PHP_QUERY_RFC3986); + + return $request->withUri($request->getUri()->withQuery($queryString)); + } + + /** + * @param RequestInterface $request + * @param CredentialsInterface $creds + * + * @return RequestInterface + */ + private function prepareRequest( + RequestInterface $request, + CredentialsInterface $creds, + ) { + $modify = [ + 'remove_headers' => ['X-Amz-Date'], + 'set_headers' => ['Date' => gmdate(\DateTimeInterface::RFC2822)] + ]; + + // Add the security token header if one is being used by the credentials + if ($token = $creds->getSecurityToken()) { + $modify['set_headers']['X-Amz-Security-Token'] = $token; + } + + return Psr7\Utils::modifyRequest($request, $modify); + } + + private function signString($string, CredentialsInterface $credentials) { + return base64_encode( + hash_hmac('sha1', $string, $credentials->getSecretKey(), true) + ); + } + + private function createCanonicalizedString( + RequestInterface $request, + $expires = null, + ) { + $buffer = $request->getMethod() . "\n"; + + // Add the interesting headers + foreach ($this->signableHeaders as $header) { + $buffer .= $request->getHeaderLine($header) . "\n"; + } + + $date = $expires ?: $request->getHeaderLine('date'); + $buffer .= "{$date}\n" + . $this->createCanonicalizedAmzHeaders($request) + . $this->createCanonicalizedResource($request); + + return $buffer; + } + + private function createCanonicalizedAmzHeaders(RequestInterface $request) { + $headers = []; + foreach ($request->getHeaders() as $name => $header) { + $name = strtolower($name); + if (str_starts_with($name, 'x-amz-')) { + $value = implode(',', $header); + if (strlen($value) > 0) { + $headers[$name] = $name . ':' . $value; + } + } + } + + if (!$headers) { + return ''; + } + + ksort($headers); + + return implode("\n", $headers) . "\n"; + } + + private function createCanonicalizedResource(RequestInterface $request) { + $data = $this->parser->parse($request->getUri()); + $buffer = '/'; + + if ($data['bucket']) { + $buffer .= $data['bucket']; + if (!empty($data['key']) || !$data['path_style']) { + $buffer .= '/' . $data['key']; + } + } + + // Add sub resource parameters if present. + $query = $request->getUri()->getQuery(); + + if ($query) { + $params = Psr7\Query::parse($query); + $first = true; + foreach ($this->signableQueryString as $key) { + if (array_key_exists($key, $params)) { + $value = $params[$key]; + $buffer .= $first ? '?' : '&'; + $first = false; + $buffer .= $key; + // Don't add values for empty sub-resources + if (strlen($value)) { + $buffer .= "={$value}"; + } + } + } + } + + return $buffer; + } +} diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php new file mode 100644 index 00000000000..888602a62e4 --- /dev/null +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -0,0 +1,77 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\Storage\IStorage; +use function is_resource; + +/** + * Object store that wraps a storage backend, mostly for testing purposes + */ +class StorageObjectStore implements IObjectStore { + /** @var IStorage */ + private $storage; + + /** + * @param IStorage $storage + */ + public function __construct(IStorage $storage) { + $this->storage = $storage; + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + public function getStorageId(): string { + return $this->storage->getId(); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + public function readObject($urn) { + $handle = $this->storage->fopen($urn, 'r'); + if (is_resource($handle)) { + return $handle; + } + + throw new \Exception(); + } + + public function writeObject($urn, $stream, ?string $mimetype = null) { + $handle = $this->storage->fopen($urn, 'w'); + if ($handle) { + stream_copy_to_stream($stream, $handle); + fclose($handle); + } else { + throw new \Exception(); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + public function deleteObject($urn) { + $this->storage->unlink($urn); + } + + public function objectExists($urn) { + return $this->storage->file_exists($urn); + } + + public function copyObject($from, $to) { + $this->storage->copy($from, $to); + } +} diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php new file mode 100644 index 00000000000..aa8b3bb34ec --- /dev/null +++ b/lib/private/Files/ObjectStore/Swift.php @@ -0,0 +1,137 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use GuzzleHttp\Client; +use GuzzleHttp\Exception\BadResponseException; +use GuzzleHttp\Psr7\Utils; +use Icewind\Streams\RetryWrapper; +use OCP\Files\NotFoundException; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\StorageAuthException; +use Psr\Log\LoggerInterface; + +const SWIFT_SEGMENT_SIZE = 1073741824; // 1GB + +class Swift implements IObjectStore { + /** + * @var array + */ + private $params; + + /** @var SwiftFactory */ + private $swiftFactory; + + public function __construct($params, ?SwiftFactory $connectionFactory = null) { + $this->swiftFactory = $connectionFactory ?: new SwiftFactory( + \OC::$server->getMemCacheFactory()->createDistributed('swift::'), + $params, + \OC::$server->get(LoggerInterface::class) + ); + $this->params = $params; + } + + /** + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws \OCP\Files\StorageNotAvailableException + */ + private function getContainer() { + return $this->swiftFactory->getContainer(); + } + + /** + * @return string the container name where objects are stored + */ + public function getStorageId() { + if (isset($this->params['bucket'])) { + return $this->params['bucket']; + } + + return $this->params['container']; + } + + public function writeObject($urn, $stream, ?string $mimetype = null) { + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile('swiftwrite'); + file_put_contents($tmpFile, $stream); + $handle = fopen($tmpFile, 'rb'); + + if (filesize($tmpFile) < SWIFT_SEGMENT_SIZE) { + $this->getContainer()->createObject([ + 'name' => $urn, + 'stream' => Utils::streamFor($handle), + 'contentType' => $mimetype, + ]); + } else { + $this->getContainer()->createLargeObject([ + 'name' => $urn, + 'stream' => Utils::streamFor($handle), + 'segmentSize' => SWIFT_SEGMENT_SIZE, + 'contentType' => $mimetype, + ]); + } + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception from openstack or GuzzleHttp libs when something goes wrong + * @throws NotFoundException if file does not exist + */ + public function readObject($urn) { + try { + $publicUri = $this->getContainer()->getObject($urn)->getPublicUri(); + $tokenId = $this->swiftFactory->getCachedTokenId(); + + $response = (new Client())->request('GET', $publicUri, + [ + 'stream' => true, + 'headers' => [ + 'X-Auth-Token' => $tokenId, + 'Cache-Control' => 'no-cache', + ], + ] + ); + } catch (BadResponseException $e) { + if ($e->getResponse() && $e->getResponse()->getStatusCode() === 404) { + throw new NotFoundException("object $urn not found in object store"); + } else { + throw $e; + } + } + + return RetryWrapper::wrap($response->getBody()->detach()); + } + + /** + * @param string $urn Unified Resource Name + * @return void + * @throws \Exception from openstack lib when something goes wrong + */ + public function deleteObject($urn) { + $this->getContainer()->getObject($urn)->delete(); + } + + /** + * @return void + * @throws \Exception from openstack lib when something goes wrong + */ + public function deleteContainer() { + $this->getContainer()->delete(); + } + + public function objectExists($urn) { + return $this->getContainer()->objectExists($urn); + } + + public function copyObject($from, $to) { + $this->getContainer()->getObject($from)->copy([ + 'destination' => $this->getContainer()->name . '/' . $to + ]); + } +} diff --git a/lib/private/Files/ObjectStore/SwiftFactory.php b/lib/private/Files/ObjectStore/SwiftFactory.php new file mode 100644 index 00000000000..118724159e5 --- /dev/null +++ b/lib/private/Files/ObjectStore/SwiftFactory.php @@ -0,0 +1,262 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use GuzzleHttp\Client; +use GuzzleHttp\Exception\ClientException; +use GuzzleHttp\Exception\ConnectException; +use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\HandlerStack; +use OCP\Files\StorageAuthException; +use OCP\Files\StorageNotAvailableException; +use OCP\ICache; +use OpenStack\Common\Auth\Token; +use OpenStack\Common\Error\BadResponseError; +use OpenStack\Common\Transport\Utils as TransportUtils; +use OpenStack\Identity\v2\Models\Catalog; +use OpenStack\Identity\v2\Service as IdentityV2Service; +use OpenStack\Identity\v3\Service as IdentityV3Service; +use OpenStack\ObjectStore\v1\Models\Container; +use OpenStack\OpenStack; +use Psr\Http\Message\RequestInterface; +use Psr\Log\LoggerInterface; + +class SwiftFactory { + private $cache; + private $params; + /** @var Container|null */ + private $container = null; + private LoggerInterface $logger; + + public const DEFAULT_OPTIONS = [ + 'autocreate' => false, + 'urlType' => 'publicURL', + 'catalogName' => 'swift', + 'catalogType' => 'object-store' + ]; + + public function __construct(ICache $cache, array $params, LoggerInterface $logger) { + $this->cache = $cache; + $this->params = $params; + $this->logger = $logger; + } + + /** + * Gets currently cached token id + * + * @return string + * @throws StorageAuthException + */ + public function getCachedTokenId() { + if (!isset($this->params['cachedToken'])) { + throw new StorageAuthException('Unauthenticated ObjectStore connection'); + } + + // Is it V2 token? + if (isset($this->params['cachedToken']['token'])) { + return $this->params['cachedToken']['token']['id']; + } + + return $this->params['cachedToken']['id']; + } + + private function getCachedToken(string $cacheKey) { + $cachedTokenString = $this->cache->get($cacheKey . '/token'); + if ($cachedTokenString) { + return json_decode($cachedTokenString, true); + } else { + return null; + } + } + + private function cacheToken(Token $token, string $serviceUrl, string $cacheKey) { + if ($token instanceof \OpenStack\Identity\v3\Models\Token) { + // for v3 the catalog is cached as part of the token, so no need to cache $serviceUrl separately + $value = $token->export(); + } else { + /** @var \OpenStack\Identity\v2\Models\Token $token */ + $value = [ + 'serviceUrl' => $serviceUrl, + 'token' => [ + 'issued_at' => $token->issuedAt->format('c'), + 'expires' => $token->expires->format('c'), + 'id' => $token->id, + 'tenant' => $token->tenant + ] + ]; + } + + $this->params['cachedToken'] = $value; + $this->cache->set($cacheKey . '/token', json_encode($value)); + } + + /** + * @return OpenStack + * @throws StorageAuthException + */ + private function getClient() { + if (isset($this->params['bucket'])) { + $this->params['container'] = $this->params['bucket']; + } + if (!isset($this->params['container'])) { + $this->params['container'] = 'nextcloud'; + } + if (isset($this->params['user']) && is_array($this->params['user'])) { + $userName = $this->params['user']['name']; + } else { + if (!isset($this->params['username']) && isset($this->params['user'])) { + $this->params['username'] = $this->params['user']; + } + $userName = $this->params['username']; + } + if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) { + $this->params['tenantName'] = $this->params['tenant']; + } + if (isset($this->params['domain'])) { + $this->params['scope']['project']['name'] = $this->params['tenant']; + $this->params['scope']['project']['domain']['name'] = $this->params['domain']; + } + $this->params = array_merge(self::DEFAULT_OPTIONS, $this->params); + + $cacheKey = $userName . '@' . $this->params['url'] . '/' . $this->params['container']; + $token = $this->getCachedToken($cacheKey); + $this->params['cachedToken'] = $token; + + $httpClient = new Client([ + 'base_uri' => TransportUtils::normalizeUrl($this->params['url']), + 'handler' => HandlerStack::create() + ]); + + if (isset($this->params['user']) && is_array($this->params['user']) && isset($this->params['user']['name'])) { + if (!isset($this->params['scope'])) { + throw new StorageAuthException('Scope has to be defined for V3 requests'); + } + + return $this->auth(IdentityV3Service::factory($httpClient), $cacheKey); + } else { + return $this->auth(SwiftV2CachingAuthService::factory($httpClient), $cacheKey); + } + } + + /** + * @param IdentityV2Service|IdentityV3Service $authService + * @param string $cacheKey + * @return OpenStack + * @throws StorageAuthException + */ + private function auth($authService, string $cacheKey) { + $this->params['identityService'] = $authService; + $this->params['authUrl'] = $this->params['url']; + + $cachedToken = $this->params['cachedToken']; + $hasValidCachedToken = false; + if (\is_array($cachedToken)) { + if ($authService instanceof IdentityV3Service) { + $token = $authService->generateTokenFromCache($cachedToken); + if (\is_null($token->catalog)) { + $this->logger->warning('Invalid cached token for swift, no catalog set: ' . json_encode($cachedToken)); + } elseif ($token->hasExpired()) { + $this->logger->debug('Cached token for swift expired'); + } else { + $hasValidCachedToken = true; + } + } else { + try { + /** @var \OpenStack\Identity\v2\Models\Token $token */ + $token = $authService->model(\OpenStack\Identity\v2\Models\Token::class, $cachedToken['token']); + $now = new \DateTimeImmutable('now'); + if ($token->expires > $now) { + $hasValidCachedToken = true; + $this->params['v2cachedToken'] = $token; + $this->params['v2serviceUrl'] = $cachedToken['serviceUrl']; + } else { + $this->logger->debug('Cached token for swift expired'); + } + } catch (\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + } + } + } + + if (!$hasValidCachedToken) { + unset($this->params['cachedToken']); + try { + [$token, $serviceUrl] = $authService->authenticate($this->params); + $this->cacheToken($token, $serviceUrl, $cacheKey); + } catch (ConnectException $e) { + throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e); + } catch (ClientException $e) { + $statusCode = $e->getResponse()->getStatusCode(); + if ($statusCode === 404) { + throw new StorageAuthException('Keystone not found while connecting to object storage, verify the keystone url', $e); + } elseif ($statusCode === 412) { + throw new StorageAuthException('Precondition failed while connecting to object storage, verify the keystone url', $e); + } elseif ($statusCode === 401) { + throw new StorageAuthException('Authentication failed while connecting to object storage, verify the username, password and possibly tenant', $e); + } else { + throw new StorageAuthException('Unknown error while connecting to object storage', $e); + } + } catch (RequestException $e) { + throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e); + } + } + + + $client = new OpenStack($this->params); + + return $client; + } + + /** + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws StorageNotAvailableException + */ + public function getContainer() { + if (is_null($this->container)) { + $this->container = $this->createContainer(); + } + + return $this->container; + } + + /** + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws StorageNotAvailableException + */ + private function createContainer() { + $client = $this->getClient(); + $objectStoreService = $client->objectStoreV1(); + + $autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true; + try { + $container = $objectStoreService->getContainer($this->params['container']); + if ($autoCreate) { + $container->getMetadata(); + } + return $container; + } catch (BadResponseError $ex) { + // if the container does not exist and autocreate is true try to create the container on the fly + if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) { + return $objectStoreService->createContainer([ + 'name' => $this->params['container'] + ]); + } else { + throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $ex); + } + } catch (ConnectException $e) { + /** @var RequestInterface $request */ + $request = $e->getRequest(); + $host = $request->getUri()->getHost() . ':' . $request->getUri()->getPort(); + $this->logger->error("Can't connect to object storage server at $host", ['exception' => $e]); + throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e); + } + } +} diff --git a/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php new file mode 100644 index 00000000000..266781af142 --- /dev/null +++ b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php @@ -0,0 +1,24 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +use OpenStack\Common\Auth\Token; +use OpenStack\Identity\v2\Service; + +class SwiftV2CachingAuthService extends Service { + public function authenticate(array $options = []): array { + if (isset($options['v2cachedToken'], $options['v2serviceUrl']) + && $options['v2cachedToken'] instanceof Token + && is_string($options['v2serviceUrl'])) { + return [$options['v2cachedToken'], $options['v2serviceUrl']]; + } else { + return parent::authenticate($options); + } + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php new file mode 100644 index 00000000000..bb7bef2ed63 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php @@ -0,0 +1,33 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +class FlattenNestedBool extends QueryOptimizerStep { + public function processOperator(ISearchOperator &$operator) { + if ( + $operator instanceof SearchBinaryOperator && ( + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + || $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + ) + ) { + $newArguments = []; + foreach ($operator->getArguments() as $oldArgument) { + if ($oldArgument instanceof SearchBinaryOperator && $oldArgument->getType() === $operator->getType()) { + $newArguments = array_merge($newArguments, $oldArgument->getArguments()); + } else { + $newArguments[] = $oldArgument; + } + } + $operator->setArguments($newArguments); + } + parent::processOperator($operator); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php new file mode 100644 index 00000000000..7e99c04f197 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php @@ -0,0 +1,31 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * replace single argument AND and OR operations with their single argument + */ +class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + parent::processOperator($operator); + if ( + $operator instanceof ISearchBinaryOperator + && count($operator->getArguments()) === 1 + && ( + $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + || $operator->getType() === ISearchBinaryOperator::OPERATOR_AND + ) + ) { + $operator = $operator->getArguments()[0]; + return true; + } + return false; + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php b/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php new file mode 100644 index 00000000000..4949ca7396b --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php @@ -0,0 +1,99 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * Attempt to transform + * + * (A AND B) OR (A AND C) OR (A AND D AND E) into A AND (B OR C OR (D AND E)) + * + * This is always valid because logical 'AND' and 'OR' are distributive[1]. + * + * [1]: https://en.wikipedia.org/wiki/Distributive_property + */ +class MergeDistributiveOperations extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof SearchBinaryOperator) { + // either 'AND' or 'OR' + $topLevelType = $operator->getType(); + + // split the arguments into groups that share a first argument + $groups = $this->groupBinaryOperatorsByChild($operator->getArguments(), 0); + $outerOperations = array_map(function (array $operators) use ($topLevelType) { + // no common operations, no need to change anything + if (count($operators) === 1) { + return $operators[0]; + } + + // for groups with size >1 we know they are binary operators with at least 1 child + /** @var ISearchBinaryOperator $firstArgument */ + $firstArgument = $operators[0]; + + // we already checked that all arguments have the same type, so this type applies for all, either 'AND' or 'OR' + $innerType = $firstArgument->getType(); + + // the common operation we move out ('A' from the example) + $extractedLeftHand = $firstArgument->getArguments()[0]; + + // for each argument we remove the extracted operation to get the leftovers ('B', 'C' and '(D AND E)' in the example) + // note that we leave them inside the "inner" binary operation for when the "inner" operation contained more than two parts + // in the (common) case where the "inner" operation only has 1 item left it will be cleaned up in a follow up step + $rightHandArguments = array_map(function (ISearchOperator $inner) { + /** @var ISearchBinaryOperator $inner */ + $arguments = $inner->getArguments(); + array_shift($arguments); + if (count($arguments) === 1) { + return $arguments[0]; + } + return new SearchBinaryOperator($inner->getType(), $arguments); + }, $operators); + + // combine the extracted operation ('A') with the remaining bit ('(B OR C OR (D AND E))') + // note that because of how distribution work, we use the "outer" type "inside" and the "inside" type "outside". + $extractedRightHand = new SearchBinaryOperator($topLevelType, $rightHandArguments); + return new SearchBinaryOperator( + $innerType, + [$extractedLeftHand, $extractedRightHand] + ); + }, $groups); + + // combine all groups again + $operator = new SearchBinaryOperator($topLevelType, $outerOperations); + parent::processOperator($operator); + return true; + } + return parent::processOperator($operator); + } + + /** + * Group a list of binary search operators that have a common argument + * + * Non-binary operators, or empty binary operators will each get their own 1-sized group + * + * @param ISearchOperator[] $operators + * @return ISearchOperator[][] + */ + private function groupBinaryOperatorsByChild(array $operators, int $index = 0): array { + $result = []; + foreach ($operators as $operator) { + if ($operator instanceof ISearchBinaryOperator && count($operator->getArguments()) > 0) { + /** @var SearchBinaryOperator|SearchComparison $child */ + $child = $operator->getArguments()[$index]; + $childKey = (string)$child; + $result[$childKey][] = $operator; + } else { + $result[] = [$operator]; + } + } + return array_values($result); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php new file mode 100644 index 00000000000..6df35c9c9a2 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php @@ -0,0 +1,74 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * transform (field == A OR field == B ...) into field IN (A, B, ...) + */ +class OrEqualsToIn extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ( + $operator instanceof ISearchBinaryOperator + && $operator->getType() === ISearchBinaryOperator::OPERATOR_OR + ) { + $groups = $this->groupEqualsComparisonsByField($operator->getArguments()); + $newParts = array_map(function (array $group) { + if (count($group) > 1) { + // because of the logic from `groupEqualsComparisonsByField` we now that group is all comparisons on the same field + /** @var ISearchComparison[] $group */ + $field = $group[0]->getField(); + $values = array_map(function (ISearchComparison $comparison) { + /** @var string|integer|bool|\DateTime $value */ + $value = $comparison->getValue(); + return $value; + }, $group); + $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values, $group[0]->getExtra()); + $pathEqHash = array_reduce($group, function ($pathEqHash, ISearchComparison $comparison) { + return $comparison->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true) && $pathEqHash; + }, true); + $in->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, $pathEqHash); + return $in; + } else { + return $group[0]; + } + }, $groups); + if (count($newParts) === 1) { + $operator = $newParts[0]; + } else { + $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $newParts); + } + parent::processOperator($operator); + return true; + } + parent::processOperator($operator); + return false; + } + + /** + * Non-equals operators are put in a separate group for each + * + * @param ISearchOperator[] $operators + * @return ISearchOperator[][] + */ + private function groupEqualsComparisonsByField(array $operators): array { + $result = []; + foreach ($operators as $operator) { + if ($operator instanceof ISearchComparison && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + $result[$operator->getField()][] = $operator; + } else { + $result[] = [$operator]; + } + } + return array_values($result); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php new file mode 100644 index 00000000000..2994a9365a7 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +class PathPrefixOptimizer extends QueryOptimizerStep { + private bool $useHashEq = true; + + public function inspectOperator(ISearchOperator $operator): void { + // normally any `path = "$path"` search filter would be generated as an `path_hash = md5($path)` sql query + // since the `path_hash` sql column usually provides much faster querying that selecting on the `path` sql column + // + // however, if we're already doing a filter on the `path` column in the form of `path LIKE "$prefix/%"` + // generating a `path = "$prefix"` sql query lets the database handle use the same column for both expressions and potentially use the same index + // + // If there is any operator in the query that matches this pattern, we change all `path = "$path"` instances to not the `path_hash` equality, + // otherwise mariadb has a tendency of ignoring the path_prefix index + if ($this->useHashEq && $this->isPathPrefixOperator($operator)) { + $this->useHashEq = false; + } + + parent::inspectOperator($operator); + } + + public function processOperator(ISearchOperator &$operator) { + if (!$this->useHashEq && $operator instanceof ISearchComparison && !$operator->getExtra() && $operator->getField() === 'path' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + $operator->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, false); + } + + parent::processOperator($operator); + } + + private function isPathPrefixOperator(ISearchOperator $operator): bool { + if ($operator instanceof ISearchBinaryOperator && $operator->getType() === ISearchBinaryOperator::OPERATOR_OR && count($operator->getArguments()) == 2) { + $a = $operator->getArguments()[0]; + $b = $operator->getArguments()[1]; + if ($this->operatorPairIsPathPrefix($a, $b) || $this->operatorPairIsPathPrefix($b, $a)) { + return true; + } + } + return false; + } + + private function operatorPairIsPathPrefix(ISearchOperator $like, ISearchOperator $equal): bool { + return ( + $like instanceof ISearchComparison && $equal instanceof ISearchComparison + && !$like->getExtra() && !$equal->getExtra() && $like->getField() === 'path' && $equal->getField() === 'path' + && $like->getType() === ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE && $equal->getType() === ISearchComparison::COMPARE_EQUAL + && $like->getValue() === SearchComparison::escapeLikeParameter($equal->getValue()) . '/%' + ); + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php new file mode 100644 index 00000000000..5259ca25ad3 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php @@ -0,0 +1,38 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Search\QueryOptimizer; + +use OCP\Files\Search\ISearchOperator; + +class QueryOptimizer { + /** @var QueryOptimizerStep[] */ + private $steps = []; + + public function __construct() { + // note that the order here is relevant + $this->steps = [ + new PathPrefixOptimizer(), + new MergeDistributiveOperations(), + new FlattenSingleArgumentBinaryOperation(), + new FlattenNestedBool(), + new OrEqualsToIn(), + new FlattenNestedBool(), + new SplitLargeIn(), + ]; + } + + public function processOperator(ISearchOperator &$operator) { + foreach ($this->steps as $step) { + $step->inspectOperator($operator); + } + foreach ($this->steps as $step) { + $step->processOperator($operator); + } + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php b/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php new file mode 100644 index 00000000000..15b5b580ec1 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php @@ -0,0 +1,42 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Search\QueryOptimizer; + +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +class QueryOptimizerStep { + /** + * Allow optimizer steps to inspect the entire query before starting processing + * + * @param ISearchOperator $operator + * @return void + */ + public function inspectOperator(ISearchOperator $operator): void { + if ($operator instanceof ISearchBinaryOperator) { + foreach ($operator->getArguments() as $argument) { + $this->inspectOperator($argument); + } + } + } + + /** + * Allow optimizer steps to modify query operators + * + * @param ISearchOperator $operator + * @return void + */ + public function processOperator(ISearchOperator &$operator) { + if ($operator instanceof ISearchBinaryOperator) { + foreach ($operator->getArguments() as $argument) { + $this->processOperator($argument); + } + } + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php b/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php new file mode 100644 index 00000000000..a9c9ba876bc --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php @@ -0,0 +1,37 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +/** + * Optimizer step that can replace the $operator altogether instead of just modifying it + * These steps need some extra logic to properly replace the arguments of binary operators + */ +class ReplacingOptimizerStep extends QueryOptimizerStep { + /** + * Allow optimizer steps to modify query operators + * + * Returns true if the reference $operator points to a new value + */ + public function processOperator(ISearchOperator &$operator): bool { + if ($operator instanceof SearchBinaryOperator) { + $modified = false; + $arguments = $operator->getArguments(); + foreach ($arguments as &$argument) { + if ($this->processOperator($argument)) { + $modified = true; + } + } + if ($modified) { + $operator->setArguments($arguments); + } + } + return false; + } +} diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php new file mode 100644 index 00000000000..8aee1975708 --- /dev/null +++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php @@ -0,0 +1,36 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search\QueryOptimizer; + +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOperator; + +/** + * transform IN (1000+ element) into (IN (1000 elements) OR IN(...)) + */ +class SplitLargeIn extends ReplacingOptimizerStep { + public function processOperator(ISearchOperator &$operator): bool { + if ( + $operator instanceof ISearchComparison + && $operator->getType() === ISearchComparison::COMPARE_IN + && count($operator->getValue()) > 1000 + ) { + $chunks = array_chunk($operator->getValue(), 1000); + $chunkComparisons = array_map(function (array $values) use ($operator) { + return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values, $operator->getExtra()); + }, $chunks); + + $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons); + return true; + } + parent::processOperator($operator); + return false; + } +} diff --git a/lib/private/Files/Search/SearchBinaryOperator.php b/lib/private/Files/Search/SearchBinaryOperator.php new file mode 100644 index 00000000000..49f599933f4 --- /dev/null +++ b/lib/private/Files/Search/SearchBinaryOperator.php @@ -0,0 +1,66 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search; + +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchOperator; + +class SearchBinaryOperator implements ISearchBinaryOperator { + /** @var string */ + private $type; + /** @var (SearchBinaryOperator|SearchComparison)[] */ + private $arguments; + private $hints = []; + + /** + * SearchBinaryOperator constructor. + * + * @param string $type + * @param (SearchBinaryOperator|SearchComparison)[] $arguments + */ + public function __construct($type, array $arguments) { + $this->type = $type; + $this->arguments = $arguments; + } + + /** + * @return string + */ + public function getType() { + return $this->type; + } + + /** + * @return ISearchOperator[] + */ + public function getArguments() { + return $this->arguments; + } + + /** + * @param ISearchOperator[] $arguments + * @return void + */ + public function setArguments(array $arguments): void { + $this->arguments = $arguments; + } + + public function getQueryHint(string $name, $default) { + return $this->hints[$name] ?? $default; + } + + public function setQueryHint(string $name, $value): void { + $this->hints[$name] = $value; + } + + public function __toString(): string { + if ($this->type === ISearchBinaryOperator::OPERATOR_NOT) { + return '(not ' . $this->arguments[0] . ')'; + } + return '(' . implode(' ' . $this->type . ' ', $this->arguments) . ')'; + } +} diff --git a/lib/private/Files/Search/SearchComparison.php b/lib/private/Files/Search/SearchComparison.php new file mode 100644 index 00000000000..c1f0176afd9 --- /dev/null +++ b/lib/private/Files/Search/SearchComparison.php @@ -0,0 +1,68 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search; + +use OCP\Files\Search\ISearchComparison; + +/** + * @psalm-import-type ParamValue from ISearchComparison + */ +class SearchComparison implements ISearchComparison { + private array $hints = []; + + public function __construct( + private string $type, + private string $field, + /** @var ParamValue $value */ + private \DateTime|int|string|bool|array $value, + private string $extra = '', + ) { + } + + /** + * @return string + */ + public function getType(): string { + return $this->type; + } + + /** + * @return string + */ + public function getField(): string { + return $this->field; + } + + public function getValue(): string|int|bool|\DateTime|array { + return $this->value; + } + + /** + * @return string + * @since 28.0.0 + */ + public function getExtra(): string { + return $this->extra; + } + + public function getQueryHint(string $name, $default) { + return $this->hints[$name] ?? $default; + } + + public function setQueryHint(string $name, $value): void { + $this->hints[$name] = $value; + } + + public static function escapeLikeParameter(string $param): string { + return addcslashes($param, '\\_%'); + } + + public function __toString(): string { + return $this->field . ' ' . $this->type . ' ' . json_encode($this->value); + } +} diff --git a/lib/private/Files/Search/SearchOrder.php b/lib/private/Files/Search/SearchOrder.php new file mode 100644 index 00000000000..5a036653f4e --- /dev/null +++ b/lib/private/Files/Search/SearchOrder.php @@ -0,0 +1,65 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search; + +use OCP\Files\FileInfo; +use OCP\Files\Search\ISearchOrder; + +class SearchOrder implements ISearchOrder { + public function __construct( + private string $direction, + private string $field, + private string $extra = '', + ) { + } + + /** + * @return string + */ + public function getDirection(): string { + return $this->direction; + } + + /** + * @return string + */ + public function getField(): string { + return $this->field; + } + + /** + * @return string + * @since 28.0.0 + */ + public function getExtra(): string { + return $this->extra; + } + + public function sortFileInfo(FileInfo $a, FileInfo $b): int { + $cmp = $this->sortFileInfoNoDirection($a, $b); + return $cmp * ($this->direction === ISearchOrder::DIRECTION_ASCENDING ? 1 : -1); + } + + private function sortFileInfoNoDirection(FileInfo $a, FileInfo $b): int { + switch ($this->field) { + case 'name': + return $a->getName() <=> $b->getName(); + case 'mimetype': + return $a->getMimetype() <=> $b->getMimetype(); + case 'mtime': + return $a->getMtime() <=> $b->getMtime(); + case 'size': + return $a->getSize() <=> $b->getSize(); + case 'fileid': + return $a->getId() <=> $b->getId(); + case 'permissions': + return $a->getPermissions() <=> $b->getPermissions(); + default: + return 0; + } + } +} diff --git a/lib/private/Files/Search/SearchQuery.php b/lib/private/Files/Search/SearchQuery.php new file mode 100644 index 00000000000..592749cf4a0 --- /dev/null +++ b/lib/private/Files/Search/SearchQuery.php @@ -0,0 +1,91 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Search; + +use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; +use OCP\Files\Search\ISearchQuery; +use OCP\IUser; + +class SearchQuery implements ISearchQuery { + /** @var ISearchOperator */ + private $searchOperation; + /** @var integer */ + private $limit; + /** @var integer */ + private $offset; + /** @var ISearchOrder[] */ + private $order; + /** @var ?IUser */ + private $user; + private $limitToHome; + + /** + * SearchQuery constructor. + * + * @param ISearchOperator $searchOperation + * @param int $limit + * @param int $offset + * @param array $order + * @param ?IUser $user + * @param bool $limitToHome + */ + public function __construct( + ISearchOperator $searchOperation, + int $limit, + int $offset, + array $order, + ?IUser $user = null, + bool $limitToHome = false, + ) { + $this->searchOperation = $searchOperation; + $this->limit = $limit; + $this->offset = $offset; + $this->order = $order; + $this->user = $user; + $this->limitToHome = $limitToHome; + } + + /** + * @return ISearchOperator + */ + public function getSearchOperation() { + return $this->searchOperation; + } + + /** + * @return int + */ + public function getLimit() { + return $this->limit; + } + + /** + * @return int + */ + public function getOffset() { + return $this->offset; + } + + /** + * @return ISearchOrder[] + */ + public function getOrder() { + return $this->order; + } + + /** + * @return ?IUser + */ + public function getUser() { + return $this->user; + } + + public function limitToHome(): bool { + return $this->limitToHome; + } +} diff --git a/lib/private/Files/SetupManager.php b/lib/private/Files/SetupManager.php new file mode 100644 index 00000000000..b92c608a81d --- /dev/null +++ b/lib/private/Files/SetupManager.php @@ -0,0 +1,609 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files; + +use OC\Files\Config\MountProviderCollection; +use OC\Files\Mount\HomeMountPoint; +use OC\Files\Mount\MountPoint; +use OC\Files\Storage\Common; +use OC\Files\Storage\Home; +use OC\Files\Storage\Storage; +use OC\Files\Storage\Wrapper\Availability; +use OC\Files\Storage\Wrapper\Encoding; +use OC\Files\Storage\Wrapper\PermissionsMask; +use OC\Files\Storage\Wrapper\Quota; +use OC\Lockdown\Filesystem\NullStorage; +use OC\Share\Share; +use OC\Share20\ShareDisableChecker; +use OC_Hook; +use OCA\Files_External\Config\ExternalMountPoint; +use OCA\Files_Sharing\External\Mount; +use OCA\Files_Sharing\ISharedMountPoint; +use OCA\Files_Sharing\SharedMount; +use OCP\App\IAppManager; +use OCP\Constants; +use OCP\Diagnostics\IEventLogger; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Config\ICachedMountInfo; +use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Config\IMountProvider; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Config\IUserMountCache; +use OCP\Files\Events\BeforeFileSystemSetupEvent; +use OCP\Files\Events\InvalidateMountCacheEvent; +use OCP\Files\Events\Node\FilesystemTornDownEvent; +use OCP\Files\Mount\IMountManager; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\IStorage; +use OCP\Group\Events\UserAddedEvent; +use OCP\Group\Events\UserRemovedEvent; +use OCP\ICache; +use OCP\ICacheFactory; +use OCP\IConfig; +use OCP\IUser; +use OCP\IUserManager; +use OCP\IUserSession; +use OCP\Lockdown\ILockdownManager; +use OCP\Share\Events\ShareCreatedEvent; +use Psr\Log\LoggerInterface; + +class SetupManager { + private bool $rootSetup = false; + // List of users for which at least one mount is setup + private array $setupUsers = []; + // List of users for which all mounts are setup + private array $setupUsersComplete = []; + /** @var array<string, string[]> */ + private array $setupUserMountProviders = []; + private ICache $cache; + private bool $listeningForProviders; + private array $fullSetupRequired = []; + private bool $setupBuiltinWrappersDone = false; + private bool $forceFullSetup = false; + + public function __construct( + private IEventLogger $eventLogger, + private MountProviderCollection $mountProviderCollection, + private IMountManager $mountManager, + private IUserManager $userManager, + private IEventDispatcher $eventDispatcher, + private IUserMountCache $userMountCache, + private ILockdownManager $lockdownManager, + private IUserSession $userSession, + ICacheFactory $cacheFactory, + private LoggerInterface $logger, + private IConfig $config, + private ShareDisableChecker $shareDisableChecker, + private IAppManager $appManager, + ) { + $this->cache = $cacheFactory->createDistributed('setupmanager::'); + $this->listeningForProviders = false; + $this->forceFullSetup = $this->config->getSystemValueBool('debug.force-full-fs-setup'); + + $this->setupListeners(); + } + + private function isSetupStarted(IUser $user): bool { + return in_array($user->getUID(), $this->setupUsers, true); + } + + public function isSetupComplete(IUser $user): bool { + return in_array($user->getUID(), $this->setupUsersComplete, true); + } + + private function setupBuiltinWrappers() { + if ($this->setupBuiltinWrappersDone) { + return; + } + $this->setupBuiltinWrappersDone = true; + + // load all filesystem apps before, so no setup-hook gets lost + $this->appManager->loadApps(['filesystem']); + $prevLogging = Filesystem::logWarningWhenAddingStorageWrapper(false); + + Filesystem::addStorageWrapper('mount_options', function ($mountPoint, IStorage $storage, IMountPoint $mount) { + if ($storage->instanceOfStorage(Common::class)) { + $options = array_merge($mount->getOptions(), ['mount_point' => $mountPoint]); + $storage->setMountOptions($options); + } + return $storage; + }); + + $reSharingEnabled = Share::isResharingAllowed(); + $user = $this->userSession->getUser(); + $sharingEnabledForUser = $user ? !$this->shareDisableChecker->sharingDisabledForUser($user->getUID()) : true; + Filesystem::addStorageWrapper( + 'sharing_mask', + function ($mountPoint, IStorage $storage, IMountPoint $mount) use ($reSharingEnabled, $sharingEnabledForUser) { + $sharingEnabledForMount = $mount->getOption('enable_sharing', true); + $isShared = $mount instanceof ISharedMountPoint; + if (!$sharingEnabledForMount || !$sharingEnabledForUser || (!$reSharingEnabled && $isShared)) { + return new PermissionsMask([ + 'storage' => $storage, + 'mask' => Constants::PERMISSION_ALL - Constants::PERMISSION_SHARE, + ]); + } + return $storage; + } + ); + + // install storage availability wrapper, before most other wrappers + Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, IStorage $storage, IMountPoint $mount) { + $externalMount = $mount instanceof ExternalMountPoint || $mount instanceof Mount; + if ($externalMount && !$storage->isLocal()) { + return new Availability(['storage' => $storage]); + } + return $storage; + }); + + Filesystem::addStorageWrapper('oc_encoding', function ($mountPoint, IStorage $storage, IMountPoint $mount) { + if ($mount->getOption('encoding_compatibility', false) && !$mount instanceof SharedMount) { + return new Encoding(['storage' => $storage]); + } + return $storage; + }); + + $quotaIncludeExternal = $this->config->getSystemValue('quota_include_external_storage', false); + Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage, IMountPoint $mount) use ($quotaIncludeExternal) { + // set up quota for home storages, even for other users + // which can happen when using sharing + if ($mount instanceof HomeMountPoint) { + $user = $mount->getUser(); + return new Quota(['storage' => $storage, 'quotaCallback' => function () use ($user) { + return $user->getQuotaBytes(); + }, 'root' => 'files', 'include_external_storage' => $quotaIncludeExternal]); + } + + return $storage; + }); + + Filesystem::addStorageWrapper('readonly', function ($mountPoint, IStorage $storage, IMountPoint $mount) { + /* + * Do not allow any operations that modify the storage + */ + if ($mount->getOption('readonly', false)) { + return new PermissionsMask([ + 'storage' => $storage, + 'mask' => Constants::PERMISSION_ALL & ~( + Constants::PERMISSION_UPDATE + | Constants::PERMISSION_CREATE + | Constants::PERMISSION_DELETE + ), + ]); + } + return $storage; + }); + + Filesystem::logWarningWhenAddingStorageWrapper($prevLogging); + } + + /** + * Setup the full filesystem for the specified user + */ + public function setupForUser(IUser $user): void { + if ($this->isSetupComplete($user)) { + return; + } + $this->setupUsersComplete[] = $user->getUID(); + + $this->eventLogger->start('fs:setup:user:full', 'Setup full filesystem for user'); + + if (!isset($this->setupUserMountProviders[$user->getUID()])) { + $this->setupUserMountProviders[$user->getUID()] = []; + } + + $previouslySetupProviders = $this->setupUserMountProviders[$user->getUID()]; + + $this->setupForUserWith($user, function () use ($user) { + $this->mountProviderCollection->addMountForUser($user, $this->mountManager, function ( + IMountProvider $provider, + ) use ($user) { + return !in_array(get_class($provider), $this->setupUserMountProviders[$user->getUID()]); + }); + }); + $this->afterUserFullySetup($user, $previouslySetupProviders); + $this->eventLogger->end('fs:setup:user:full'); + } + + /** + * part of the user setup that is run only once per user + */ + private function oneTimeUserSetup(IUser $user) { + if ($this->isSetupStarted($user)) { + return; + } + $this->setupUsers[] = $user->getUID(); + + $this->setupRoot(); + + $this->eventLogger->start('fs:setup:user:onetime', 'Onetime filesystem for user'); + + $this->setupBuiltinWrappers(); + + $prevLogging = Filesystem::logWarningWhenAddingStorageWrapper(false); + + // TODO remove hook + OC_Hook::emit('OC_Filesystem', 'preSetup', ['user' => $user->getUID()]); + + $event = new BeforeFileSystemSetupEvent($user); + $this->eventDispatcher->dispatchTyped($event); + + Filesystem::logWarningWhenAddingStorageWrapper($prevLogging); + + $userDir = '/' . $user->getUID() . '/files'; + + Filesystem::initInternal($userDir); + + if ($this->lockdownManager->canAccessFilesystem()) { + $this->eventLogger->start('fs:setup:user:home', 'Setup home filesystem for user'); + // home mounts are handled separate since we need to ensure this is mounted before we call the other mount providers + $homeMount = $this->mountProviderCollection->getHomeMountForUser($user); + $this->mountManager->addMount($homeMount); + + if ($homeMount->getStorageRootId() === -1) { + $this->eventLogger->start('fs:setup:user:home:scan', 'Scan home filesystem for user'); + $homeMount->getStorage()->mkdir(''); + $homeMount->getStorage()->getScanner()->scan(''); + $this->eventLogger->end('fs:setup:user:home:scan'); + } + $this->eventLogger->end('fs:setup:user:home'); + } else { + $this->mountManager->addMount(new MountPoint( + new NullStorage([]), + '/' . $user->getUID() + )); + $this->mountManager->addMount(new MountPoint( + new NullStorage([]), + '/' . $user->getUID() . '/files' + )); + $this->setupUsersComplete[] = $user->getUID(); + } + + $this->listenForNewMountProviders(); + + $this->eventLogger->end('fs:setup:user:onetime'); + } + + /** + * Final housekeeping after a user has been fully setup + */ + private function afterUserFullySetup(IUser $user, array $previouslySetupProviders): void { + $this->eventLogger->start('fs:setup:user:full:post', 'Housekeeping after user is setup'); + $userRoot = '/' . $user->getUID() . '/'; + $mounts = $this->mountManager->getAll(); + $mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot) { + return str_starts_with($mount->getMountPoint(), $userRoot); + }); + $allProviders = array_map(function (IMountProvider|IHomeMountProvider|IRootMountProvider $provider) { + return get_class($provider); + }, array_merge( + $this->mountProviderCollection->getProviders(), + $this->mountProviderCollection->getHomeProviders(), + $this->mountProviderCollection->getRootProviders(), + )); + $newProviders = array_diff($allProviders, $previouslySetupProviders); + $mounts = array_filter($mounts, function (IMountPoint $mount) use ($previouslySetupProviders) { + return !in_array($mount->getMountProvider(), $previouslySetupProviders); + }); + $this->registerMounts($user, $mounts, $newProviders); + + $cacheDuration = $this->config->getSystemValueInt('fs_mount_cache_duration', 5 * 60); + if ($cacheDuration > 0) { + $this->cache->set($user->getUID(), true, $cacheDuration); + $this->fullSetupRequired[$user->getUID()] = false; + } + $this->eventLogger->end('fs:setup:user:full:post'); + } + + /** + * @param IUser $user + * @param IMountPoint $mounts + * @return void + * @throws \OCP\HintException + * @throws \OC\ServerNotAvailableException + */ + private function setupForUserWith(IUser $user, callable $mountCallback): void { + $this->oneTimeUserSetup($user); + + if ($this->lockdownManager->canAccessFilesystem()) { + $mountCallback(); + } + $this->eventLogger->start('fs:setup:user:post-init-mountpoint', 'post_initMountPoints legacy hook'); + \OC_Hook::emit('OC_Filesystem', 'post_initMountPoints', ['user' => $user->getUID()]); + $this->eventLogger->end('fs:setup:user:post-init-mountpoint'); + + $userDir = '/' . $user->getUID() . '/files'; + $this->eventLogger->start('fs:setup:user:setup-hook', 'setup legacy hook'); + OC_Hook::emit('OC_Filesystem', 'setup', ['user' => $user->getUID(), 'user_dir' => $userDir]); + $this->eventLogger->end('fs:setup:user:setup-hook'); + } + + /** + * Set up the root filesystem + */ + public function setupRoot(): void { + //setting up the filesystem twice can only lead to trouble + if ($this->rootSetup) { + return; + } + + $this->setupBuiltinWrappers(); + + $this->rootSetup = true; + + $this->eventLogger->start('fs:setup:root', 'Setup root filesystem'); + + $rootMounts = $this->mountProviderCollection->getRootMounts(); + foreach ($rootMounts as $rootMountProvider) { + $this->mountManager->addMount($rootMountProvider); + } + + $this->eventLogger->end('fs:setup:root'); + } + + /** + * Get the user to setup for a path or `null` if the root needs to be setup + * + * @param string $path + * @return IUser|null + */ + private function getUserForPath(string $path) { + if (str_starts_with($path, '/__groupfolders')) { + return null; + } elseif (substr_count($path, '/') < 2) { + if ($user = $this->userSession->getUser()) { + return $user; + } else { + return null; + } + } elseif (str_starts_with($path, '/appdata_' . \OC_Util::getInstanceId()) || str_starts_with($path, '/files_external/')) { + return null; + } else { + [, $userId] = explode('/', $path); + } + + return $this->userManager->get($userId); + } + + /** + * Set up the filesystem for the specified path + */ + public function setupForPath(string $path, bool $includeChildren = false): void { + $user = $this->getUserForPath($path); + if (!$user) { + $this->setupRoot(); + return; + } + + if ($this->isSetupComplete($user)) { + return; + } + + if ($this->fullSetupRequired($user)) { + $this->setupForUser($user); + return; + } + + // for the user's home folder, and includes children we need everything always + if (rtrim($path) === '/' . $user->getUID() . '/files' && $includeChildren) { + $this->setupForUser($user); + return; + } + + if (!isset($this->setupUserMountProviders[$user->getUID()])) { + $this->setupUserMountProviders[$user->getUID()] = []; + } + $setupProviders = &$this->setupUserMountProviders[$user->getUID()]; + $currentProviders = []; + + try { + $cachedMount = $this->userMountCache->getMountForPath($user, $path); + } catch (NotFoundException $e) { + $this->setupForUser($user); + return; + } + + $this->oneTimeUserSetup($user); + + $this->eventLogger->start('fs:setup:user:path', "Setup $path filesystem for user"); + $this->eventLogger->start('fs:setup:user:path:find', "Find mountpoint for $path"); + + $mounts = []; + if (!in_array($cachedMount->getMountProvider(), $setupProviders)) { + $currentProviders[] = $cachedMount->getMountProvider(); + if ($cachedMount->getMountProvider()) { + $setupProviders[] = $cachedMount->getMountProvider(); + $mounts = $this->mountProviderCollection->getUserMountsForProviderClasses($user, [$cachedMount->getMountProvider()]); + } else { + $this->logger->debug('mount at ' . $cachedMount->getMountPoint() . ' has no provider set, performing full setup'); + $this->eventLogger->end('fs:setup:user:path:find'); + $this->setupForUser($user); + $this->eventLogger->end('fs:setup:user:path'); + return; + } + } + + if ($includeChildren) { + $subCachedMounts = $this->userMountCache->getMountsInPath($user, $path); + $this->eventLogger->end('fs:setup:user:path:find'); + + $needsFullSetup = array_reduce($subCachedMounts, function (bool $needsFullSetup, ICachedMountInfo $cachedMountInfo) { + return $needsFullSetup || $cachedMountInfo->getMountProvider() === ''; + }, false); + + if ($needsFullSetup) { + $this->logger->debug('mount has no provider set, performing full setup'); + $this->setupForUser($user); + $this->eventLogger->end('fs:setup:user:path'); + return; + } else { + foreach ($subCachedMounts as $cachedMount) { + if (!in_array($cachedMount->getMountProvider(), $setupProviders)) { + $currentProviders[] = $cachedMount->getMountProvider(); + $setupProviders[] = $cachedMount->getMountProvider(); + $mounts = array_merge($mounts, $this->mountProviderCollection->getUserMountsForProviderClasses($user, [$cachedMount->getMountProvider()])); + } + } + } + } else { + $this->eventLogger->end('fs:setup:user:path:find'); + } + + if (count($mounts)) { + $this->registerMounts($user, $mounts, $currentProviders); + $this->setupForUserWith($user, function () use ($mounts) { + array_walk($mounts, [$this->mountManager, 'addMount']); + }); + } elseif (!$this->isSetupStarted($user)) { + $this->oneTimeUserSetup($user); + } + $this->eventLogger->end('fs:setup:user:path'); + } + + private function fullSetupRequired(IUser $user): bool { + if ($this->forceFullSetup) { + return true; + } + + // we perform a "cached" setup only after having done the full setup recently + // this is also used to trigger a full setup after handling events that are likely + // to change the available mounts + if (!isset($this->fullSetupRequired[$user->getUID()])) { + $this->fullSetupRequired[$user->getUID()] = !$this->cache->get($user->getUID()); + } + return $this->fullSetupRequired[$user->getUID()]; + } + + /** + * @param string $path + * @param string[] $providers + */ + public function setupForProvider(string $path, array $providers): void { + $user = $this->getUserForPath($path); + if (!$user) { + $this->setupRoot(); + return; + } + + if ($this->isSetupComplete($user)) { + return; + } + + if ($this->fullSetupRequired($user)) { + $this->setupForUser($user); + return; + } + + $this->eventLogger->start('fs:setup:user:providers', 'Setup filesystem for ' . implode(', ', $providers)); + + $this->oneTimeUserSetup($user); + + // home providers are always used + $providers = array_filter($providers, function (string $provider) { + return !is_subclass_of($provider, IHomeMountProvider::class); + }); + + if (in_array('', $providers)) { + $this->setupForUser($user); + return; + } + $setupProviders = $this->setupUserMountProviders[$user->getUID()] ?? []; + + $providers = array_diff($providers, $setupProviders); + if (count($providers) === 0) { + if (!$this->isSetupStarted($user)) { + $this->oneTimeUserSetup($user); + } + $this->eventLogger->end('fs:setup:user:providers'); + return; + } else { + $this->setupUserMountProviders[$user->getUID()] = array_merge($setupProviders, $providers); + $mounts = $this->mountProviderCollection->getUserMountsForProviderClasses($user, $providers); + } + + $this->registerMounts($user, $mounts, $providers); + $this->setupForUserWith($user, function () use ($mounts) { + array_walk($mounts, [$this->mountManager, 'addMount']); + }); + $this->eventLogger->end('fs:setup:user:providers'); + } + + public function tearDown() { + $this->setupUsers = []; + $this->setupUsersComplete = []; + $this->setupUserMountProviders = []; + $this->fullSetupRequired = []; + $this->rootSetup = false; + $this->mountManager->clear(); + $this->eventDispatcher->dispatchTyped(new FilesystemTornDownEvent()); + } + + /** + * Get mounts from mount providers that are registered after setup + */ + private function listenForNewMountProviders() { + if (!$this->listeningForProviders) { + $this->listeningForProviders = true; + $this->mountProviderCollection->listen('\OC\Files\Config', 'registerMountProvider', function ( + IMountProvider $provider, + ) { + foreach ($this->setupUsers as $userId) { + $user = $this->userManager->get($userId); + if ($user) { + $mounts = $provider->getMountsForUser($user, Filesystem::getLoader()); + array_walk($mounts, [$this->mountManager, 'addMount']); + } + } + }); + } + } + + private function setupListeners() { + // note that this event handling is intentionally pessimistic + // clearing the cache to often is better than not enough + + $this->eventDispatcher->addListener(UserAddedEvent::class, function (UserAddedEvent $event) { + $this->cache->remove($event->getUser()->getUID()); + }); + $this->eventDispatcher->addListener(UserRemovedEvent::class, function (UserRemovedEvent $event) { + $this->cache->remove($event->getUser()->getUID()); + }); + $this->eventDispatcher->addListener(ShareCreatedEvent::class, function (ShareCreatedEvent $event) { + $this->cache->remove($event->getShare()->getSharedWith()); + }); + $this->eventDispatcher->addListener(InvalidateMountCacheEvent::class, function (InvalidateMountCacheEvent $event, + ) { + if ($user = $event->getUser()) { + $this->cache->remove($user->getUID()); + } else { + $this->cache->clear(); + } + }); + + $genericEvents = [ + 'OCA\Circles\Events\CreatingCircleEvent', + 'OCA\Circles\Events\DestroyingCircleEvent', + 'OCA\Circles\Events\AddingCircleMemberEvent', + 'OCA\Circles\Events\RemovingCircleMemberEvent', + ]; + + foreach ($genericEvents as $genericEvent) { + $this->eventDispatcher->addListener($genericEvent, function ($event) { + $this->cache->clear(); + }); + } + } + + private function registerMounts(IUser $user, array $mounts, ?array $mountProviderClasses = null): void { + if ($this->lockdownManager->canAccessFilesystem()) { + $this->userMountCache->registerMounts($user, $mounts, $mountProviderClasses); + } + } +} diff --git a/lib/private/Files/SetupManagerFactory.php b/lib/private/Files/SetupManagerFactory.php new file mode 100644 index 00000000000..d2fe978fa9e --- /dev/null +++ b/lib/private/Files/SetupManagerFactory.php @@ -0,0 +1,65 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files; + +use OC\Share20\ShareDisableChecker; +use OCP\App\IAppManager; +use OCP\Diagnostics\IEventLogger; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Config\IMountProviderCollection; +use OCP\Files\Config\IUserMountCache; +use OCP\Files\Mount\IMountManager; +use OCP\ICacheFactory; +use OCP\IConfig; +use OCP\IUserManager; +use OCP\IUserSession; +use OCP\Lockdown\ILockdownManager; +use Psr\Log\LoggerInterface; + +class SetupManagerFactory { + private ?SetupManager $setupManager; + + public function __construct( + private IEventLogger $eventLogger, + private IMountProviderCollection $mountProviderCollection, + private IUserManager $userManager, + private IEventDispatcher $eventDispatcher, + private IUserMountCache $userMountCache, + private ILockdownManager $lockdownManager, + private IUserSession $userSession, + private ICacheFactory $cacheFactory, + private LoggerInterface $logger, + private IConfig $config, + private ShareDisableChecker $shareDisableChecker, + private IAppManager $appManager, + ) { + $this->setupManager = null; + } + + public function create(IMountManager $mountManager): SetupManager { + if (!$this->setupManager) { + $this->setupManager = new SetupManager( + $this->eventLogger, + $this->mountProviderCollection, + $mountManager, + $this->userManager, + $this->eventDispatcher, + $this->userMountCache, + $this->lockdownManager, + $this->userSession, + $this->cacheFactory, + $this->logger, + $this->config, + $this->shareDisableChecker, + $this->appManager, + ); + } + return $this->setupManager; + } +} diff --git a/lib/private/Files/SimpleFS/NewSimpleFile.php b/lib/private/Files/SimpleFS/NewSimpleFile.php new file mode 100644 index 00000000000..d0986592c03 --- /dev/null +++ b/lib/private/Files/SimpleFS/NewSimpleFile.php @@ -0,0 +1,211 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\SimpleFS; + +use Icewind\Streams\CallbackWrapper; +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Files\SimpleFS\ISimpleFile; + +class NewSimpleFile implements ISimpleFile { + private Folder $parentFolder; + private string $name; + private ?File $file = null; + + /** + * File constructor. + */ + public function __construct(Folder $parentFolder, string $name) { + $this->parentFolder = $parentFolder; + $this->name = $name; + } + + /** + * Get the name + */ + public function getName(): string { + return $this->name; + } + + /** + * Get the size in bytes + */ + public function getSize(): int|float { + if ($this->file) { + return $this->file->getSize(); + } else { + return 0; + } + } + + /** + * Get the ETag + */ + public function getETag(): string { + if ($this->file) { + return $this->file->getEtag(); + } else { + return ''; + } + } + + /** + * Get the last modification time + */ + public function getMTime(): int { + if ($this->file) { + return $this->file->getMTime(); + } else { + return time(); + } + } + + /** + * Get the content + * + * @throws NotFoundException + * @throws NotPermittedException + */ + public function getContent(): string { + if ($this->file) { + $result = $this->file->getContent(); + + if ($result === false) { + $this->checkFile(); + } + + return $result; + } else { + return ''; + } + } + + /** + * Overwrite the file + * + * @param string|resource $data + * @throws NotPermittedException + * @throws NotFoundException + */ + public function putContent($data): void { + try { + if ($this->file) { + $this->file->putContent($data); + } else { + $this->file = $this->parentFolder->newFile($this->name, $data); + } + } catch (NotFoundException $e) { + $this->checkFile(); + } + } + + /** + * Sometimes there are some issues with the AppData. Most of them are from + * user error. But we should handle them gracefully anyway. + * + * If for some reason the current file can't be found. We remove it. + * Then traverse up and check all folders if they exists. This so that the + * next request will have a valid appdata structure again. + * + * @throws NotFoundException + */ + private function checkFile(): void { + if (!$this->file) { + throw new NotFoundException('File not set'); + } + + $cur = $this->file; + + while ($cur->stat() === false) { + $parent = $cur->getParent(); + try { + $cur->delete(); + } catch (NotFoundException $e) { + // Just continue then + } + $cur = $parent; + } + + if ($cur !== $this->file) { + throw new NotFoundException('File does not exist'); + } + } + + + /** + * Delete the file + * + * @throws NotPermittedException + */ + public function delete(): void { + if ($this->file) { + $this->file->delete(); + } + } + + /** + * Get the MimeType + * + * @return string + */ + public function getMimeType(): string { + if ($this->file) { + return $this->file->getMimeType(); + } else { + return 'text/plain'; + } + } + + /** + * {@inheritDoc} + */ + public function getExtension(): string { + if ($this->file) { + return $this->file->getExtension(); + } else { + return \pathinfo($this->name, PATHINFO_EXTENSION); + } + } + + /** + * Open the file as stream for reading, resulting resource can be operated as stream like the result from php's own fopen + * + * @return resource|false + * @throws \OCP\Files\NotPermittedException + * @since 14.0.0 + */ + public function read() { + if ($this->file) { + return $this->file->fopen('r'); + } else { + return fopen('php://temp', 'r'); + } + } + + /** + * Open the file as stream for writing, resulting resource can be operated as stream like the result from php's own fopen + * + * @return resource|bool + * @throws \OCP\Files\NotPermittedException + * @since 14.0.0 + */ + public function write() { + if ($this->file) { + return $this->file->fopen('w'); + } else { + $source = fopen('php://temp', 'w+'); + return CallbackWrapper::wrap($source, null, null, null, null, function () use ($source) { + rewind($source); + $this->putContent($source); + }); + } + } +} diff --git a/lib/private/Files/SimpleFS/SimpleFile.php b/lib/private/Files/SimpleFS/SimpleFile.php new file mode 100644 index 00000000000..d9c1b47d2f1 --- /dev/null +++ b/lib/private/Files/SimpleFS/SimpleFile.php @@ -0,0 +1,163 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\SimpleFS; + +use OCP\Files\File; +use OCP\Files\GenericFileException; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Files\SimpleFS\ISimpleFile; +use OCP\Lock\LockedException; + +class SimpleFile implements ISimpleFile { + private File $file; + + public function __construct(File $file) { + $this->file = $file; + } + + /** + * Get the name + */ + public function getName(): string { + return $this->file->getName(); + } + + /** + * Get the size in bytes + */ + public function getSize(): int|float { + return $this->file->getSize(); + } + + /** + * Get the ETag + */ + public function getETag(): string { + return $this->file->getEtag(); + } + + /** + * Get the last modification time + */ + public function getMTime(): int { + return $this->file->getMTime(); + } + + /** + * Get the content + * + * @throws GenericFileException + * @throws LockedException + * @throws NotFoundException + * @throws NotPermittedException + */ + public function getContent(): string { + $result = $this->file->getContent(); + + if ($result === false) { + $this->checkFile(); + } + + return $result; + } + + /** + * Overwrite the file + * + * @param string|resource $data + * @throws GenericFileException + * @throws LockedException + * @throws NotFoundException + * @throws NotPermittedException + */ + public function putContent($data): void { + try { + $this->file->putContent($data); + } catch (NotFoundException $e) { + $this->checkFile(); + } + } + + /** + * Sometimes there are some issues with the AppData. Most of them are from + * user error. But we should handle them gracefully anyway. + * + * If for some reason the current file can't be found. We remove it. + * Then traverse up and check all folders if they exists. This so that the + * next request will have a valid appdata structure again. + * + * @throws NotFoundException + */ + private function checkFile(): void { + $cur = $this->file; + + while ($cur->stat() === false) { + $parent = $cur->getParent(); + try { + $cur->delete(); + } catch (NotFoundException $e) { + // Just continue then + } + $cur = $parent; + } + + if ($cur !== $this->file) { + throw new NotFoundException('File does not exist'); + } + } + + + /** + * Delete the file + * + * @throws NotPermittedException + */ + public function delete(): void { + $this->file->delete(); + } + + /** + * Get the MimeType + */ + public function getMimeType(): string { + return $this->file->getMimeType(); + } + + /** + * {@inheritDoc} + */ + public function getExtension(): string { + return $this->file->getExtension(); + } + + /** + * Open the file as stream for reading, resulting resource can be operated as stream like the result from php's own fopen + * + * @return resource|false + * @throws \OCP\Files\NotPermittedException + * @since 14.0.0 + */ + public function read() { + return $this->file->fopen('r'); + } + + /** + * Open the file as stream for writing, resulting resource can be operated as stream like the result from php's own fopen + * + * @return resource|false + * @throws \OCP\Files\NotPermittedException + * @since 14.0.0 + */ + public function write() { + return $this->file->fopen('w'); + } + + public function getId(): int { + return $this->file->getId(); + } +} diff --git a/lib/private/Files/SimpleFS/SimpleFolder.php b/lib/private/Files/SimpleFS/SimpleFolder.php new file mode 100644 index 00000000000..62f3db25e9b --- /dev/null +++ b/lib/private/Files/SimpleFS/SimpleFolder.php @@ -0,0 +1,90 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\SimpleFS; + +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\SimpleFS\ISimpleFile; +use OCP\Files\SimpleFS\ISimpleFolder; + +class SimpleFolder implements ISimpleFolder { + /** @var Folder */ + private $folder; + + /** + * Folder constructor. + * + * @param Folder $folder + */ + public function __construct(Folder $folder) { + $this->folder = $folder; + } + + public function getName(): string { + return $this->folder->getName(); + } + + public function getDirectoryListing(): array { + $listing = $this->folder->getDirectoryListing(); + + $fileListing = array_map(function (Node $file) { + if ($file instanceof File) { + return new SimpleFile($file); + } + return null; + }, $listing); + + $fileListing = array_filter($fileListing); + + return array_values($fileListing); + } + + public function delete(): void { + $this->folder->delete(); + } + + public function fileExists(string $name): bool { + return $this->folder->nodeExists($name); + } + + public function getFile(string $name): ISimpleFile { + $file = $this->folder->get($name); + + if (!($file instanceof File)) { + throw new NotFoundException(); + } + + return new SimpleFile($file); + } + + public function newFile(string $name, $content = null): ISimpleFile { + if ($content === null) { + // delay creating the file until it's written to + return new NewSimpleFile($this->folder, $name); + } else { + $file = $this->folder->newFile($name, $content); + return new SimpleFile($file); + } + } + + public function getFolder(string $name): ISimpleFolder { + $folder = $this->folder->get($name); + + if (!($folder instanceof Folder)) { + throw new NotFoundException(); + } + + return new SimpleFolder($folder); + } + + public function newFolder(string $path): ISimpleFolder { + $folder = $this->folder->newFolder($path); + return new SimpleFolder($folder); + } +} diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php new file mode 100644 index 00000000000..2dc359169d7 --- /dev/null +++ b/lib/private/Files/Storage/Common.php @@ -0,0 +1,769 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OC\Files\Cache\Propagator; +use OC\Files\Cache\Scanner; +use OC\Files\Cache\Updater; +use OC\Files\Cache\Watcher; +use OC\Files\FilenameValidator; +use OC\Files\Filesystem; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OC\Files\Storage\Wrapper\Wrapper; +use OCP\Files; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Cache\IScanner; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Cache\IWatcher; +use OCP\Files\ForbiddenException; +use OCP\Files\GenericFileException; +use OCP\Files\IFilenameValidator; +use OCP\Files\InvalidPathException; +use OCP\Files\Storage\IConstructableStorage; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IStorage; +use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\IConfig; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; +use OCP\Server; +use Psr\Log\LoggerInterface; + +/** + * Storage backend class for providing common filesystem operation methods + * which are not storage-backend specific. + * + * \OC\Files\Storage\Common is never used directly; it is extended by all other + * storage backends, where its methods may be overridden, and additional + * (backend-specific) methods are defined. + * + * Some \OC\Files\Storage\Common methods call functions which are first defined + * in classes which extend it, e.g. $this->stat() . + */ +abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, IConstructableStorage { + use LocalTempFileTrait; + + protected ?Cache $cache = null; + protected ?Scanner $scanner = null; + protected ?Watcher $watcher = null; + protected ?Propagator $propagator = null; + protected $storageCache; + protected ?Updater $updater = null; + + protected array $mountOptions = []; + protected $owner = null; + + private ?bool $shouldLogLocks = null; + private ?LoggerInterface $logger = null; + private ?IFilenameValidator $filenameValidator = null; + + public function __construct(array $parameters) { + } + + protected function remove(string $path): bool { + if ($this->file_exists($path)) { + if ($this->is_dir($path)) { + return $this->rmdir($path); + } elseif ($this->is_file($path)) { + return $this->unlink($path); + } + } + return false; + } + + public function is_dir(string $path): bool { + return $this->filetype($path) === 'dir'; + } + + public function is_file(string $path): bool { + return $this->filetype($path) === 'file'; + } + + public function filesize(string $path): int|float|false { + if ($this->is_dir($path)) { + return 0; //by definition + } else { + $stat = $this->stat($path); + return isset($stat['size']) ? $stat['size'] : 0; + } + } + + public function isReadable(string $path): bool { + // at least check whether it exists + // subclasses might want to implement this more thoroughly + return $this->file_exists($path); + } + + public function isUpdatable(string $path): bool { + // at least check whether it exists + // subclasses might want to implement this more thoroughly + // a non-existing file/folder isn't updatable + return $this->file_exists($path); + } + + public function isCreatable(string $path): bool { + if ($this->is_dir($path) && $this->isUpdatable($path)) { + return true; + } + return false; + } + + public function isDeletable(string $path): bool { + if ($path === '' || $path === '/') { + return $this->isUpdatable($path); + } + $parent = dirname($path); + return $this->isUpdatable($parent) && $this->isUpdatable($path); + } + + public function isSharable(string $path): bool { + return $this->isReadable($path); + } + + public function getPermissions(string $path): int { + $permissions = 0; + if ($this->isCreatable($path)) { + $permissions |= \OCP\Constants::PERMISSION_CREATE; + } + if ($this->isReadable($path)) { + $permissions |= \OCP\Constants::PERMISSION_READ; + } + if ($this->isUpdatable($path)) { + $permissions |= \OCP\Constants::PERMISSION_UPDATE; + } + if ($this->isDeletable($path)) { + $permissions |= \OCP\Constants::PERMISSION_DELETE; + } + if ($this->isSharable($path)) { + $permissions |= \OCP\Constants::PERMISSION_SHARE; + } + return $permissions; + } + + public function filemtime(string $path): int|false { + $stat = $this->stat($path); + if (isset($stat['mtime']) && $stat['mtime'] > 0) { + return $stat['mtime']; + } else { + return 0; + } + } + + public function file_get_contents(string $path): string|false { + $handle = $this->fopen($path, 'r'); + if (!$handle) { + return false; + } + $data = stream_get_contents($handle); + fclose($handle); + return $data; + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + $handle = $this->fopen($path, 'w'); + if (!$handle) { + return false; + } + $this->removeCachedFile($path); + $count = fwrite($handle, $data); + fclose($handle); + return $count; + } + + public function rename(string $source, string $target): bool { + $this->remove($target); + + $this->removeCachedFile($source); + return $this->copy($source, $target) and $this->remove($source); + } + + public function copy(string $source, string $target): bool { + if ($this->is_dir($source)) { + $this->remove($target); + $dir = $this->opendir($source); + $this->mkdir($target); + while (($file = readdir($dir)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + if (!$this->copy($source . '/' . $file, $target . '/' . $file)) { + closedir($dir); + return false; + } + } + } + closedir($dir); + return true; + } else { + $sourceStream = $this->fopen($source, 'r'); + $targetStream = $this->fopen($target, 'w'); + [, $result] = Files::streamCopy($sourceStream, $targetStream, true); + if (!$result) { + Server::get(LoggerInterface::class)->warning("Failed to write data while copying $source to $target"); + } + $this->removeCachedFile($target); + return $result; + } + } + + public function getMimeType(string $path): string|false { + if ($this->is_dir($path)) { + return 'httpd/unix-directory'; + } elseif ($this->file_exists($path)) { + return \OC::$server->getMimeTypeDetector()->detectPath($path); + } else { + return false; + } + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + $fh = $this->fopen($path, 'rb'); + if (!$fh) { + return false; + } + $ctx = hash_init($type); + hash_update_stream($ctx, $fh); + fclose($fh); + return hash_final($ctx, $raw); + } + + public function getLocalFile(string $path): string|false { + return $this->getCachedFile($path); + } + + private function addLocalFolder(string $path, string $target): void { + $dh = $this->opendir($path); + if (is_resource($dh)) { + while (($file = readdir($dh)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + if ($this->is_dir($path . '/' . $file)) { + mkdir($target . '/' . $file); + $this->addLocalFolder($path . '/' . $file, $target . '/' . $file); + } else { + $tmp = $this->toTmpFile($path . '/' . $file); + rename($tmp, $target . '/' . $file); + } + } + } + } + } + + protected function searchInDir(string $query, string $dir = ''): array { + $files = []; + $dh = $this->opendir($dir); + if (is_resource($dh)) { + while (($item = readdir($dh)) !== false) { + if (Filesystem::isIgnoredDir($item)) { + continue; + } + if (strstr(strtolower($item), strtolower($query)) !== false) { + $files[] = $dir . '/' . $item; + } + if ($this->is_dir($dir . '/' . $item)) { + $files = array_merge($files, $this->searchInDir($query, $dir . '/' . $item)); + } + } + } + closedir($dh); + return $files; + } + + /** + * @inheritDoc + * Check if a file or folder has been updated since $time + * + * The method is only used to check if the cache needs to be updated. Storage backends that don't support checking + * the mtime should always return false here. As a result storage implementations that always return false expect + * exclusive access to the backend and will not pick up files that have been added in a way that circumvents + * Nextcloud filesystem. + */ + public function hasUpdated(string $path, int $time): bool { + return $this->filemtime($path) > $time; + } + + protected function getCacheDependencies(): CacheDependencies { + static $dependencies = null; + if (!$dependencies) { + $dependencies = Server::get(CacheDependencies::class); + } + return $dependencies; + } + + public function getCache(string $path = '', ?IStorage $storage = null): ICache { + if (!$storage) { + $storage = $this; + } + /** @var self $storage */ + if (!isset($storage->cache)) { + $storage->cache = new Cache($storage, $this->getCacheDependencies()); + } + return $storage->cache; + } + + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { + if (!$storage) { + $storage = $this; + } + if (!$storage->instanceOfStorage(self::class)) { + throw new \InvalidArgumentException('Storage is not of the correct class'); + } + if (!isset($storage->scanner)) { + $storage->scanner = new Scanner($storage); + } + return $storage->scanner; + } + + public function getWatcher(string $path = '', ?IStorage $storage = null): IWatcher { + if (!$storage) { + $storage = $this; + } + if (!isset($this->watcher)) { + $this->watcher = new Watcher($storage); + $globalPolicy = Server::get(IConfig::class)->getSystemValueInt('filesystem_check_changes', Watcher::CHECK_NEVER); + $this->watcher->setPolicy((int)$this->getMountOption('filesystem_check_changes', $globalPolicy)); + } + return $this->watcher; + } + + public function getPropagator(?IStorage $storage = null): IPropagator { + if (!$storage) { + $storage = $this; + } + if (!$storage->instanceOfStorage(self::class)) { + throw new \InvalidArgumentException('Storage is not of the correct class'); + } + /** @var self $storage */ + if (!isset($storage->propagator)) { + $config = Server::get(IConfig::class); + $storage->propagator = new Propagator($storage, \OC::$server->getDatabaseConnection(), ['appdata_' . $config->getSystemValueString('instanceid')]); + } + return $storage->propagator; + } + + public function getUpdater(?IStorage $storage = null): IUpdater { + if (!$storage) { + $storage = $this; + } + if (!$storage->instanceOfStorage(self::class)) { + throw new \InvalidArgumentException('Storage is not of the correct class'); + } + /** @var self $storage */ + if (!isset($storage->updater)) { + $storage->updater = new Updater($storage); + } + return $storage->updater; + } + + public function getStorageCache(?IStorage $storage = null): \OC\Files\Cache\Storage { + /** @var Cache $cache */ + $cache = $this->getCache(storage: $storage); + return $cache->getStorageCache(); + } + + public function getOwner(string $path): string|false { + if ($this->owner === null) { + $this->owner = \OC_User::getUser(); + } + + return $this->owner; + } + + public function getETag(string $path): string|false { + return uniqid(); + } + + /** + * clean a path, i.e. remove all redundant '.' and '..' + * making sure that it can't point to higher than '/' + * + * @param string $path The path to clean + * @return string cleaned path + */ + public function cleanPath(string $path): string { + if (strlen($path) == 0 || $path[0] != '/') { + $path = '/' . $path; + } + + $output = []; + foreach (explode('/', $path) as $chunk) { + if ($chunk == '..') { + array_pop($output); + } elseif ($chunk == '.') { + } else { + $output[] = $chunk; + } + } + return implode('/', $output); + } + + /** + * Test a storage for availability + */ + public function test(): bool { + try { + if ($this->stat('')) { + return true; + } + Server::get(LoggerInterface::class)->info('External storage not available: stat() failed'); + return false; + } catch (\Exception $e) { + Server::get(LoggerInterface::class)->warning( + 'External storage not available: ' . $e->getMessage(), + ['exception' => $e] + ); + return false; + } + } + + public function free_space(string $path): int|float|false { + return \OCP\Files\FileInfo::SPACE_UNKNOWN; + } + + public function isLocal(): bool { + // the common implementation returns a temporary file by + // default, which is not local + return false; + } + + /** + * Check if the storage is an instance of $class or is a wrapper for a storage that is an instance of $class + */ + public function instanceOfStorage(string $class): bool { + if (ltrim($class, '\\') === 'OC\Files\Storage\Shared') { + // FIXME Temporary fix to keep existing checks working + $class = '\OCA\Files_Sharing\SharedStorage'; + } + return is_a($this, $class); + } + + /** + * A custom storage implementation can return an url for direct download of a give file. + * + * For now the returned array can hold the parameter url - in future more attributes might follow. + */ + public function getDirectDownload(string $path): array|false { + return []; + } + + public function verifyPath(string $path, string $fileName): void { + $this->getFilenameValidator() + ->validateFilename($fileName); + + // verify also the path is valid + if ($path && $path !== '/' && $path !== '.') { + try { + $this->verifyPath(dirname($path), basename($path)); + } catch (InvalidPathException $e) { + // Ignore invalid file type exceptions on directories + if ($e->getCode() !== FilenameValidator::INVALID_FILE_TYPE) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Invalid parent path'), previous: $e); + } + } + } + } + + /** + * Get the filename validator + * (cached for performance) + */ + protected function getFilenameValidator(): IFilenameValidator { + if ($this->filenameValidator === null) { + $this->filenameValidator = Server::get(IFilenameValidator::class); + } + return $this->filenameValidator; + } + + public function setMountOptions(array $options): void { + $this->mountOptions = $options; + } + + public function getMountOption(string $name, mixed $default = null): mixed { + return $this->mountOptions[$name] ?? $default; + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, bool $preserveMtime = false): bool { + if ($sourceStorage === $this) { + return $this->copy($sourceInternalPath, $targetInternalPath); + } + + if ($sourceStorage->is_dir($sourceInternalPath)) { + $dh = $sourceStorage->opendir($sourceInternalPath); + $result = $this->mkdir($targetInternalPath); + if (is_resource($dh)) { + $result = true; + while ($result && ($file = readdir($dh)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + $result &= $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file); + } + } + } + } else { + $source = $sourceStorage->fopen($sourceInternalPath, 'r'); + $result = false; + if ($source) { + try { + $this->writeStream($targetInternalPath, $source); + $result = true; + } catch (\Exception $e) { + Server::get(LoggerInterface::class)->warning('Failed to copy stream to storage', ['exception' => $e]); + } + } + + if ($result && $preserveMtime) { + $mtime = $sourceStorage->filemtime($sourceInternalPath); + $this->touch($targetInternalPath, is_int($mtime) ? $mtime : null); + } + + if (!$result) { + // delete partially written target file + $this->unlink($targetInternalPath); + // delete cache entry that was created by fopen + $this->getCache()->remove($targetInternalPath); + } + } + return (bool)$result; + } + + /** + * Check if a storage is the same as the current one, including wrapped storages + */ + private function isSameStorage(IStorage $storage): bool { + while ($storage->instanceOfStorage(Wrapper::class)) { + /** + * @var Wrapper $storage + */ + $storage = $storage->getWrapperStorage(); + } + + return $storage === $this; + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ( + !$sourceStorage->instanceOfStorage(Encryption::class) + && $this->isSameStorage($sourceStorage) + ) { + // resolve any jailed paths + while ($sourceStorage->instanceOfStorage(Jail::class)) { + /** + * @var Jail $sourceStorage + */ + $sourceInternalPath = $sourceStorage->getUnjailedPath($sourceInternalPath); + $sourceStorage = $sourceStorage->getUnjailedStorage(); + } + + return $this->rename($sourceInternalPath, $targetInternalPath); + } + + if (!$sourceStorage->isDeletable($sourceInternalPath)) { + return false; + } + + $result = $this->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath, true); + if ($result) { + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + try { + if ($sourceStorage->is_dir($sourceInternalPath)) { + $result = $sourceStorage->rmdir($sourceInternalPath); + } else { + $result = $sourceStorage->unlink($sourceInternalPath); + } + } finally { + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + } + } + return $result; + } + + public function getMetaData(string $path): ?array { + if (Filesystem::isFileBlacklisted($path)) { + throw new ForbiddenException('Invalid path: ' . $path, false); + } + + $permissions = $this->getPermissions($path); + if (!$permissions & \OCP\Constants::PERMISSION_READ) { + //can't read, nothing we can do + return null; + } + + $data = []; + $data['mimetype'] = $this->getMimeType($path); + $data['mtime'] = $this->filemtime($path); + if ($data['mtime'] === false) { + $data['mtime'] = time(); + } + if ($data['mimetype'] == 'httpd/unix-directory') { + $data['size'] = -1; //unknown + } else { + $data['size'] = $this->filesize($path); + } + $data['etag'] = $this->getETag($path); + $data['storage_mtime'] = $data['mtime']; + $data['permissions'] = $permissions; + $data['name'] = basename($path); + + return $data; + } + + public function acquireLock(string $path, int $type, ILockingProvider $provider): void { + $logger = $this->getLockLogger(); + if ($logger) { + $typeString = ($type === ILockingProvider::LOCK_SHARED) ? 'shared' : 'exclusive'; + $logger->info( + sprintf( + 'acquire %s lock on "%s" on storage "%s"', + $typeString, + $path, + $this->getId() + ), + [ + 'app' => 'locking', + ] + ); + } + try { + $provider->acquireLock('files/' . md5($this->getId() . '::' . trim($path, '/')), $type, $this->getId() . '::' . $path); + } catch (LockedException $e) { + $e = new LockedException($e->getPath(), $e, $e->getExistingLock(), $path); + if ($logger) { + $logger->info($e->getMessage(), ['exception' => $e]); + } + throw $e; + } + } + + public function releaseLock(string $path, int $type, ILockingProvider $provider): void { + $logger = $this->getLockLogger(); + if ($logger) { + $typeString = ($type === ILockingProvider::LOCK_SHARED) ? 'shared' : 'exclusive'; + $logger->info( + sprintf( + 'release %s lock on "%s" on storage "%s"', + $typeString, + $path, + $this->getId() + ), + [ + 'app' => 'locking', + ] + ); + } + try { + $provider->releaseLock('files/' . md5($this->getId() . '::' . trim($path, '/')), $type); + } catch (LockedException $e) { + $e = new LockedException($e->getPath(), $e, $e->getExistingLock(), $path); + if ($logger) { + $logger->info($e->getMessage(), ['exception' => $e]); + } + throw $e; + } + } + + public function changeLock(string $path, int $type, ILockingProvider $provider): void { + $logger = $this->getLockLogger(); + if ($logger) { + $typeString = ($type === ILockingProvider::LOCK_SHARED) ? 'shared' : 'exclusive'; + $logger->info( + sprintf( + 'change lock on "%s" to %s on storage "%s"', + $path, + $typeString, + $this->getId() + ), + [ + 'app' => 'locking', + ] + ); + } + try { + $provider->changeLock('files/' . md5($this->getId() . '::' . trim($path, '/')), $type); + } catch (LockedException $e) { + $e = new LockedException($e->getPath(), $e, $e->getExistingLock(), $path); + if ($logger) { + $logger->info($e->getMessage(), ['exception' => $e]); + } + throw $e; + } + } + + private function getLockLogger(): ?LoggerInterface { + if (is_null($this->shouldLogLocks)) { + $this->shouldLogLocks = Server::get(IConfig::class)->getSystemValueBool('filelocking.debug', false); + $this->logger = $this->shouldLogLocks ? Server::get(LoggerInterface::class) : null; + } + return $this->logger; + } + + /** + * @return array [ available, last_checked ] + */ + public function getAvailability(): array { + return $this->getStorageCache()->getAvailability(); + } + + public function setAvailability(bool $isAvailable): void { + $this->getStorageCache()->setAvailability($isAvailable); + } + + public function setOwner(?string $user): void { + $this->owner = $user; + } + + public function needsPartFile(): bool { + return true; + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + $target = $this->fopen($path, 'w'); + if (!$target) { + throw new GenericFileException("Failed to open $path for writing"); + } + try { + [$count, $result] = Files::streamCopy($stream, $target, true); + if (!$result) { + throw new GenericFileException('Failed to copy stream'); + } + } finally { + fclose($target); + fclose($stream); + } + return $count; + } + + public function getDirectoryContent(string $directory): \Traversable { + $dh = $this->opendir($directory); + + if ($dh === false) { + throw new StorageNotAvailableException('Directory listing failed'); + } + + if (is_resource($dh)) { + $basePath = rtrim($directory, '/'); + while (($file = readdir($dh)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + $childPath = $basePath . '/' . trim($file, '/'); + $metadata = $this->getMetaData($childPath); + if ($metadata !== null) { + yield $metadata; + } + } + } + } + } +} diff --git a/lib/private/Files/Storage/CommonTest.php b/lib/private/Files/Storage/CommonTest.php new file mode 100644 index 00000000000..da796130899 --- /dev/null +++ b/lib/private/Files/Storage/CommonTest.php @@ -0,0 +1,60 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +class CommonTest extends \OC\Files\Storage\Common { + /** + * underlying local storage used for missing functions + * @var \OC\Files\Storage\Local + */ + private $storage; + + public function __construct(array $parameters) { + $this->storage = new \OC\Files\Storage\Local($parameters); + } + + public function getId(): string { + return 'test::' . $this->storage->getId(); + } + public function mkdir(string $path): bool { + return $this->storage->mkdir($path); + } + public function rmdir(string $path): bool { + return $this->storage->rmdir($path); + } + public function opendir(string $path) { + return $this->storage->opendir($path); + } + public function stat(string $path): array|false { + return $this->storage->stat($path); + } + public function filetype(string $path): string|false { + return @$this->storage->filetype($path); + } + public function isReadable(string $path): bool { + return $this->storage->isReadable($path); + } + public function isUpdatable(string $path): bool { + return $this->storage->isUpdatable($path); + } + public function file_exists(string $path): bool { + return $this->storage->file_exists($path); + } + public function unlink(string $path): bool { + return $this->storage->unlink($path); + } + public function fopen(string $path, string $mode) { + return $this->storage->fopen($path, $mode); + } + public function free_space(string $path): int|float|false { + return $this->storage->free_space($path); + } + public function touch(string $path, ?int $mtime = null): bool { + return $this->storage->touch($path, $mtime); + } +} diff --git a/lib/private/Files/Storage/DAV.php b/lib/private/Files/Storage/DAV.php new file mode 100644 index 00000000000..2d166b5438d --- /dev/null +++ b/lib/private/Files/Storage/DAV.php @@ -0,0 +1,841 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use Exception; +use Icewind\Streams\CallbackWrapper; +use Icewind\Streams\IteratorDirectory; +use OC\Files\Filesystem; +use OC\MemCache\ArrayCache; +use OCP\AppFramework\Http; +use OCP\Constants; +use OCP\Diagnostics\IEventLogger; +use OCP\Files\FileInfo; +use OCP\Files\ForbiddenException; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\StorageInvalidException; +use OCP\Files\StorageNotAvailableException; +use OCP\Http\Client\IClient; +use OCP\Http\Client\IClientService; +use OCP\ICertificateManager; +use OCP\IConfig; +use OCP\Server; +use OCP\Util; +use Psr\Http\Message\ResponseInterface; +use Psr\Log\LoggerInterface; +use Sabre\DAV\Client; +use Sabre\DAV\Xml\Property\ResourceType; +use Sabre\HTTP\ClientException; +use Sabre\HTTP\ClientHttpException; +use Sabre\HTTP\RequestInterface; + +/** + * Class DAV + * + * @package OC\Files\Storage + */ +class DAV extends Common { + /** @var string */ + protected $password; + /** @var string */ + protected $user; + /** @var string|null */ + protected $authType; + /** @var string */ + protected $host; + /** @var bool */ + protected $secure; + /** @var string */ + protected $root; + /** @var string */ + protected $certPath; + /** @var bool */ + protected $ready; + /** @var Client */ + protected $client; + /** @var ArrayCache */ + protected $statCache; + /** @var IClientService */ + protected $httpClientService; + /** @var ICertificateManager */ + protected $certManager; + protected LoggerInterface $logger; + protected IEventLogger $eventLogger; + protected IMimeTypeDetector $mimeTypeDetector; + + /** @var int */ + private $timeout; + + protected const PROPFIND_PROPS = [ + '{DAV:}getlastmodified', + '{DAV:}getcontentlength', + '{DAV:}getcontenttype', + '{http://owncloud.org/ns}permissions', + '{http://open-collaboration-services.org/ns}share-permissions', + '{DAV:}resourcetype', + '{DAV:}getetag', + '{DAV:}quota-available-bytes', + ]; + + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + $this->statCache = new ArrayCache(); + $this->httpClientService = Server::get(IClientService::class); + if (isset($parameters['host']) && isset($parameters['user']) && isset($parameters['password'])) { + $host = $parameters['host']; + //remove leading http[s], will be generated in createBaseUri() + if (str_starts_with($host, 'https://')) { + $host = substr($host, 8); + } elseif (str_starts_with($host, 'http://')) { + $host = substr($host, 7); + } + $this->host = $host; + $this->user = $parameters['user']; + $this->password = $parameters['password']; + if (isset($parameters['authType'])) { + $this->authType = $parameters['authType']; + } + if (isset($parameters['secure'])) { + if (is_string($parameters['secure'])) { + $this->secure = ($parameters['secure'] === 'true'); + } else { + $this->secure = (bool)$parameters['secure']; + } + } else { + $this->secure = false; + } + if ($this->secure === true) { + // inject mock for testing + $this->certManager = \OC::$server->getCertificateManager(); + } + $this->root = rawurldecode($parameters['root'] ?? '/'); + $this->root = '/' . ltrim($this->root, '/'); + $this->root = rtrim($this->root, '/') . '/'; + } else { + throw new \Exception('Invalid webdav storage configuration'); + } + $this->logger = Server::get(LoggerInterface::class); + $this->eventLogger = Server::get(IEventLogger::class); + // This timeout value will be used for the download and upload of files + $this->timeout = Server::get(IConfig::class)->getSystemValueInt('davstorage.request_timeout', IClient::DEFAULT_REQUEST_TIMEOUT); + $this->mimeTypeDetector = \OC::$server->getMimeTypeDetector(); + } + + protected function init(): void { + if ($this->ready) { + return; + } + $this->ready = true; + + $settings = [ + 'baseUri' => $this->createBaseUri(), + 'userName' => $this->user, + 'password' => $this->password, + ]; + if ($this->authType !== null) { + $settings['authType'] = $this->authType; + } + + $proxy = Server::get(IConfig::class)->getSystemValueString('proxy', ''); + if ($proxy !== '') { + $settings['proxy'] = $proxy; + } + + $this->client = new Client($settings); + $this->client->setThrowExceptions(true); + + if ($this->secure === true) { + $certPath = $this->certManager->getAbsoluteBundlePath(); + if (file_exists($certPath)) { + $this->certPath = $certPath; + } + if ($this->certPath) { + $this->client->addCurlSetting(CURLOPT_CAINFO, $this->certPath); + } + } + + $lastRequestStart = 0; + $this->client->on('beforeRequest', function (RequestInterface $request) use (&$lastRequestStart) { + $this->logger->debug('sending dav ' . $request->getMethod() . ' request to external storage: ' . $request->getAbsoluteUrl(), ['app' => 'dav']); + $lastRequestStart = microtime(true); + $this->eventLogger->start('fs:storage:dav:request', 'Sending dav request to external storage'); + }); + $this->client->on('afterRequest', function (RequestInterface $request) use (&$lastRequestStart) { + $elapsed = microtime(true) - $lastRequestStart; + $this->logger->debug('dav ' . $request->getMethod() . ' request to external storage: ' . $request->getAbsoluteUrl() . ' took ' . round($elapsed * 1000, 1) . 'ms', ['app' => 'dav']); + $this->eventLogger->end('fs:storage:dav:request'); + }); + } + + /** + * Clear the stat cache + */ + public function clearStatCache(): void { + $this->statCache->clear(); + } + + public function getId(): string { + return 'webdav::' . $this->user . '@' . $this->host . '/' . $this->root; + } + + public function createBaseUri(): string { + $baseUri = 'http'; + if ($this->secure) { + $baseUri .= 's'; + } + $baseUri .= '://' . $this->host . $this->encodePath($this->root); + return $baseUri; + } + + public function mkdir(string $path): bool { + $this->init(); + $path = $this->cleanPath($path); + $result = $this->simpleResponse('MKCOL', $path, null, 201); + if ($result) { + $this->statCache->set($path, true); + } + return $result; + } + + public function rmdir(string $path): bool { + $this->init(); + $path = $this->cleanPath($path); + // FIXME: some WebDAV impl return 403 when trying to DELETE + // a non-empty folder + $result = $this->simpleResponse('DELETE', $path . '/', null, 204); + $this->statCache->clear($path . '/'); + $this->statCache->remove($path); + return $result; + } + + public function opendir(string $path) { + $this->init(); + $path = $this->cleanPath($path); + try { + $content = $this->getDirectoryContent($path); + $files = []; + foreach ($content as $child) { + $files[] = $child['name']; + } + return IteratorDirectory::wrap($files); + } catch (\Exception $e) { + $this->convertException($e, $path); + } + return false; + } + + /** + * Propfind call with cache handling. + * + * First checks if information is cached. + * If not, request it from the server then store to cache. + * + * @param string $path path to propfind + * + * @return array|false propfind response or false if the entry was not found + * + * @throws ClientHttpException + */ + protected function propfind(string $path): array|false { + $path = $this->cleanPath($path); + $cachedResponse = $this->statCache->get($path); + // we either don't know it, or we know it exists but need more details + if (is_null($cachedResponse) || $cachedResponse === true) { + $this->init(); + $response = false; + try { + $response = $this->client->propFind( + $this->encodePath($path), + self::PROPFIND_PROPS + ); + $this->statCache->set($path, $response); + } catch (ClientHttpException $e) { + if ($e->getHttpStatus() === 404 || $e->getHttpStatus() === 405) { + $this->statCache->clear($path . '/'); + $this->statCache->set($path, false); + } else { + $this->convertException($e, $path); + } + } catch (\Exception $e) { + $this->convertException($e, $path); + } + } else { + $response = $cachedResponse; + } + return $response; + } + + public function filetype(string $path): string|false { + try { + $response = $this->propfind($path); + if ($response === false) { + return false; + } + $responseType = []; + if (isset($response['{DAV:}resourcetype'])) { + /** @var ResourceType[] $response */ + $responseType = $response['{DAV:}resourcetype']->getValue(); + } + return (count($responseType) > 0 && $responseType[0] == '{DAV:}collection') ? 'dir' : 'file'; + } catch (\Exception $e) { + $this->convertException($e, $path); + } + return false; + } + + public function file_exists(string $path): bool { + try { + $path = $this->cleanPath($path); + $cachedState = $this->statCache->get($path); + if ($cachedState === false) { + // we know the file doesn't exist + return false; + } elseif (!is_null($cachedState)) { + return true; + } + // need to get from server + return ($this->propfind($path) !== false); + } catch (\Exception $e) { + $this->convertException($e, $path); + } + return false; + } + + public function unlink(string $path): bool { + $this->init(); + $path = $this->cleanPath($path); + $result = $this->simpleResponse('DELETE', $path, null, 204); + $this->statCache->clear($path . '/'); + $this->statCache->remove($path); + return $result; + } + + public function fopen(string $path, string $mode) { + $this->init(); + $path = $this->cleanPath($path); + switch ($mode) { + case 'r': + case 'rb': + try { + $response = $this->httpClientService + ->newClient() + ->get($this->createBaseUri() . $this->encodePath($path), [ + 'auth' => [$this->user, $this->password], + 'stream' => true, + // set download timeout for users with slow connections or large files + 'timeout' => $this->timeout + ]); + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getResponse() instanceof ResponseInterface + && $e->getResponse()->getStatusCode() === 404) { + return false; + } else { + throw $e; + } + } + + if ($response->getStatusCode() !== Http::STATUS_OK) { + if ($response->getStatusCode() === Http::STATUS_LOCKED) { + throw new \OCP\Lock\LockedException($path); + } else { + $this->logger->error('Guzzle get returned status code ' . $response->getStatusCode(), ['app' => 'webdav client']); + } + } + + $content = $response->getBody(); + + if ($content === null || is_string($content)) { + return false; + } + + return $content; + case 'w': + case 'wb': + case 'a': + case 'ab': + case 'r+': + case 'w+': + case 'wb+': + case 'a+': + case 'x': + case 'x+': + case 'c': + case 'c+': + //emulate these + $tempManager = \OC::$server->getTempManager(); + if (strrpos($path, '.') !== false) { + $ext = substr($path, strrpos($path, '.')); + } else { + $ext = ''; + } + if ($this->file_exists($path)) { + if (!$this->isUpdatable($path)) { + return false; + } + if ($mode === 'w' || $mode === 'w+') { + $tmpFile = $tempManager->getTemporaryFile($ext); + } else { + $tmpFile = $this->getCachedFile($path); + } + } else { + if (!$this->isCreatable(dirname($path))) { + return false; + } + $tmpFile = $tempManager->getTemporaryFile($ext); + } + $handle = fopen($tmpFile, $mode); + return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { + $this->writeBack($tmpFile, $path); + }); + } + + return false; + } + + public function writeBack(string $tmpFile, string $path): void { + $this->uploadFile($tmpFile, $path); + unlink($tmpFile); + } + + public function free_space(string $path): int|float|false { + $this->init(); + $path = $this->cleanPath($path); + try { + $response = $this->propfind($path); + if ($response === false) { + return FileInfo::SPACE_UNKNOWN; + } + if (isset($response['{DAV:}quota-available-bytes'])) { + return Util::numericToNumber($response['{DAV:}quota-available-bytes']); + } else { + return FileInfo::SPACE_UNKNOWN; + } + } catch (\Exception $e) { + return FileInfo::SPACE_UNKNOWN; + } + } + + public function touch(string $path, ?int $mtime = null): bool { + $this->init(); + if (is_null($mtime)) { + $mtime = time(); + } + $path = $this->cleanPath($path); + + // if file exists, update the mtime, else create a new empty file + if ($this->file_exists($path)) { + try { + $this->statCache->remove($path); + $this->client->proppatch($this->encodePath($path), ['{DAV:}lastmodified' => $mtime]); + // non-owncloud clients might not have accepted the property, need to recheck it + $response = $this->client->propfind($this->encodePath($path), ['{DAV:}getlastmodified'], 0); + if (isset($response['{DAV:}getlastmodified'])) { + $remoteMtime = strtotime($response['{DAV:}getlastmodified']); + if ($remoteMtime !== $mtime) { + // server has not accepted the mtime + return false; + } + } + } catch (ClientHttpException $e) { + if ($e->getHttpStatus() === 501) { + return false; + } + $this->convertException($e, $path); + return false; + } catch (\Exception $e) { + $this->convertException($e, $path); + return false; + } + } else { + $this->file_put_contents($path, ''); + } + return true; + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + $path = $this->cleanPath($path); + $result = parent::file_put_contents($path, $data); + $this->statCache->remove($path); + return $result; + } + + protected function uploadFile(string $path, string $target): void { + $this->init(); + + // invalidate + $target = $this->cleanPath($target); + $this->statCache->remove($target); + $source = fopen($path, 'r'); + + $this->httpClientService + ->newClient() + ->put($this->createBaseUri() . $this->encodePath($target), [ + 'body' => $source, + 'auth' => [$this->user, $this->password], + // set upload timeout for users with slow connections or large files + 'timeout' => $this->timeout + ]); + + $this->removeCachedFile($target); + } + + public function rename(string $source, string $target): bool { + $this->init(); + $source = $this->cleanPath($source); + $target = $this->cleanPath($target); + try { + // overwrite directory ? + if ($this->is_dir($target)) { + // needs trailing slash in destination + $target = rtrim($target, '/') . '/'; + } + $this->client->request( + 'MOVE', + $this->encodePath($source), + null, + [ + 'Destination' => $this->createBaseUri() . $this->encodePath($target), + ] + ); + $this->statCache->clear($source . '/'); + $this->statCache->clear($target . '/'); + $this->statCache->set($source, false); + $this->statCache->set($target, true); + $this->removeCachedFile($source); + $this->removeCachedFile($target); + return true; + } catch (\Exception $e) { + $this->convertException($e); + } + return false; + } + + public function copy(string $source, string $target): bool { + $this->init(); + $source = $this->cleanPath($source); + $target = $this->cleanPath($target); + try { + // overwrite directory ? + if ($this->is_dir($target)) { + // needs trailing slash in destination + $target = rtrim($target, '/') . '/'; + } + $this->client->request( + 'COPY', + $this->encodePath($source), + null, + [ + 'Destination' => $this->createBaseUri() . $this->encodePath($target), + ] + ); + $this->statCache->clear($target . '/'); + $this->statCache->set($target, true); + $this->removeCachedFile($target); + return true; + } catch (\Exception $e) { + $this->convertException($e); + } + return false; + } + + public function getMetaData(string $path): ?array { + if (Filesystem::isFileBlacklisted($path)) { + throw new ForbiddenException('Invalid path: ' . $path, false); + } + $response = $this->propfind($path); + if (!$response) { + return null; + } else { + return $this->getMetaFromPropfind($path, $response); + } + } + private function getMetaFromPropfind(string $path, array $response): array { + if (isset($response['{DAV:}getetag'])) { + $etag = trim($response['{DAV:}getetag'], '"'); + if (strlen($etag) > 40) { + $etag = md5($etag); + } + } else { + $etag = parent::getETag($path); + } + + $responseType = []; + if (isset($response['{DAV:}resourcetype'])) { + /** @var ResourceType[] $response */ + $responseType = $response['{DAV:}resourcetype']->getValue(); + } + $type = (count($responseType) > 0 && $responseType[0] == '{DAV:}collection') ? 'dir' : 'file'; + if ($type === 'dir') { + $mimeType = 'httpd/unix-directory'; + } elseif (isset($response['{DAV:}getcontenttype'])) { + $mimeType = $response['{DAV:}getcontenttype']; + } else { + $mimeType = $this->mimeTypeDetector->detectPath($path); + } + + if (isset($response['{http://owncloud.org/ns}permissions'])) { + $permissions = $this->parsePermissions($response['{http://owncloud.org/ns}permissions']); + } elseif ($type === 'dir') { + $permissions = Constants::PERMISSION_ALL; + } else { + $permissions = Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE; + } + + $mtime = isset($response['{DAV:}getlastmodified']) ? strtotime($response['{DAV:}getlastmodified']) : null; + + if ($type === 'dir') { + $size = -1; + } else { + $size = Util::numericToNumber($response['{DAV:}getcontentlength'] ?? 0); + } + + return [ + 'name' => basename($path), + 'mtime' => $mtime, + 'storage_mtime' => $mtime, + 'size' => $size, + 'permissions' => $permissions, + 'etag' => $etag, + 'mimetype' => $mimeType, + ]; + } + + public function stat(string $path): array|false { + $meta = $this->getMetaData($path); + return $meta ?: false; + + } + + public function getMimeType(string $path): string|false { + $meta = $this->getMetaData($path); + return $meta ? $meta['mimetype'] : false; + } + + public function cleanPath(string $path): string { + if ($path === '') { + return $path; + } + $path = Filesystem::normalizePath($path); + // remove leading slash + return substr($path, 1); + } + + /** + * URL encodes the given path but keeps the slashes + * + * @param string $path to encode + * @return string encoded path + */ + protected function encodePath(string $path): string { + // slashes need to stay + return str_replace('%2F', '/', rawurlencode($path)); + } + + /** + * @return bool + * @throws StorageInvalidException + * @throws StorageNotAvailableException + */ + protected function simpleResponse(string $method, string $path, ?string $body, int $expected): bool { + $path = $this->cleanPath($path); + try { + $response = $this->client->request($method, $this->encodePath($path), $body); + return $response['statusCode'] == $expected; + } catch (ClientHttpException $e) { + if ($e->getHttpStatus() === 404 && $method === 'DELETE') { + $this->statCache->clear($path . '/'); + $this->statCache->set($path, false); + return false; + } + + $this->convertException($e, $path); + } catch (\Exception $e) { + $this->convertException($e, $path); + } + return false; + } + + /** + * check if curl is installed + */ + public static function checkDependencies(): bool { + return true; + } + + public function isUpdatable(string $path): bool { + return (bool)($this->getPermissions($path) & Constants::PERMISSION_UPDATE); + } + + public function isCreatable(string $path): bool { + return (bool)($this->getPermissions($path) & Constants::PERMISSION_CREATE); + } + + public function isSharable(string $path): bool { + return (bool)($this->getPermissions($path) & Constants::PERMISSION_SHARE); + } + + public function isDeletable(string $path): bool { + return (bool)($this->getPermissions($path) & Constants::PERMISSION_DELETE); + } + + public function getPermissions(string $path): int { + $stat = $this->getMetaData($path); + return $stat ? $stat['permissions'] : 0; + } + + public function getETag(string $path): string|false { + $meta = $this->getMetaData($path); + return $meta ? $meta['etag'] : false; + } + + protected function parsePermissions(string $permissionsString): int { + $permissions = Constants::PERMISSION_READ; + if (str_contains($permissionsString, 'R')) { + $permissions |= Constants::PERMISSION_SHARE; + } + if (str_contains($permissionsString, 'D')) { + $permissions |= Constants::PERMISSION_DELETE; + } + if (str_contains($permissionsString, 'W')) { + $permissions |= Constants::PERMISSION_UPDATE; + } + if (str_contains($permissionsString, 'CK')) { + $permissions |= Constants::PERMISSION_CREATE; + $permissions |= Constants::PERMISSION_UPDATE; + } + return $permissions; + } + + public function hasUpdated(string $path, int $time): bool { + $this->init(); + $path = $this->cleanPath($path); + try { + // force refresh for $path + $this->statCache->remove($path); + $response = $this->propfind($path); + if ($response === false) { + if ($path === '') { + // if root is gone it means the storage is not available + throw new StorageNotAvailableException('root is gone'); + } + return false; + } + if (isset($response['{DAV:}getetag'])) { + $cachedData = $this->getCache()->get($path); + $etag = trim($response['{DAV:}getetag'], '"'); + if (($cachedData === false) || (!empty($etag) && ($cachedData['etag'] !== $etag))) { + return true; + } elseif (isset($response['{http://open-collaboration-services.org/ns}share-permissions'])) { + $sharePermissions = (int)$response['{http://open-collaboration-services.org/ns}share-permissions']; + return $sharePermissions !== $cachedData['permissions']; + } elseif (isset($response['{http://owncloud.org/ns}permissions'])) { + $permissions = $this->parsePermissions($response['{http://owncloud.org/ns}permissions']); + return $permissions !== $cachedData['permissions']; + } else { + return false; + } + } elseif (isset($response['{DAV:}getlastmodified'])) { + $remoteMtime = strtotime($response['{DAV:}getlastmodified']); + return $remoteMtime > $time; + } else { + // neither `getetag` nor `getlastmodified` is set + return false; + } + } catch (ClientHttpException $e) { + if ($e->getHttpStatus() === 405) { + if ($path === '') { + // if root is gone it means the storage is not available + throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage()); + } + return false; + } + $this->convertException($e, $path); + return false; + } catch (\Exception $e) { + $this->convertException($e, $path); + return false; + } + } + + /** + * Interpret the given exception and decide whether it is due to an + * unavailable storage, invalid storage or other. + * This will either throw StorageInvalidException, StorageNotAvailableException + * or do nothing. + * + * @param Exception $e sabre exception + * @param string $path optional path from the operation + * + * @throws StorageInvalidException if the storage is invalid, for example + * when the authentication expired or is invalid + * @throws StorageNotAvailableException if the storage is not available, + * which might be temporary + * @throws ForbiddenException if the action is not allowed + */ + protected function convertException(Exception $e, string $path = ''): void { + $this->logger->debug($e->getMessage(), ['app' => 'files_external', 'exception' => $e]); + if ($e instanceof ClientHttpException) { + if ($e->getHttpStatus() === Http::STATUS_LOCKED) { + throw new \OCP\Lock\LockedException($path); + } + if ($e->getHttpStatus() === Http::STATUS_UNAUTHORIZED) { + // either password was changed or was invalid all along + throw new StorageInvalidException(get_class($e) . ': ' . $e->getMessage()); + } elseif ($e->getHttpStatus() === Http::STATUS_METHOD_NOT_ALLOWED) { + // ignore exception for MethodNotAllowed, false will be returned + return; + } elseif ($e->getHttpStatus() === Http::STATUS_FORBIDDEN) { + // The operation is forbidden. Fail somewhat gracefully + throw new ForbiddenException(get_class($e) . ':' . $e->getMessage(), false); + } + throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage()); + } elseif ($e instanceof ClientException) { + // connection timeout or refused, server could be temporarily down + throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage()); + } elseif ($e instanceof \InvalidArgumentException) { + // parse error because the server returned HTML instead of XML, + // possibly temporarily down + throw new StorageNotAvailableException(get_class($e) . ': ' . $e->getMessage()); + } elseif (($e instanceof StorageNotAvailableException) || ($e instanceof StorageInvalidException)) { + // rethrow + throw $e; + } + + // TODO: only log for now, but in the future need to wrap/rethrow exception + } + + public function getDirectoryContent(string $directory): \Traversable { + $this->init(); + $directory = $this->cleanPath($directory); + try { + $responses = $this->client->propFind( + $this->encodePath($directory), + self::PROPFIND_PROPS, + 1 + ); + + array_shift($responses); //the first entry is the current directory + if (!$this->statCache->hasKey($directory)) { + $this->statCache->set($directory, true); + } + + foreach ($responses as $file => $response) { + $file = rawurldecode($file); + $file = substr($file, strlen($this->root)); + $file = $this->cleanPath($file); + $this->statCache->set($file, $response); + yield $this->getMetaFromPropfind($file, $response); + } + } catch (\Exception $e) { + $this->convertException($e, $directory); + } + } +} diff --git a/lib/private/Files/Storage/FailedStorage.php b/lib/private/Files/Storage/FailedStorage.php new file mode 100644 index 00000000000..a8288de48d0 --- /dev/null +++ b/lib/private/Files/Storage/FailedStorage.php @@ -0,0 +1,191 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OC\Files\Cache\FailedCache; +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\Lock\ILockingProvider; + +/** + * Storage placeholder to represent a missing precondition, storage unavailable + */ +class FailedStorage extends Common { + /** @var \Exception */ + protected $e; + + /** + * @param array $parameters ['exception' => \Exception] + */ + public function __construct(array $parameters) { + $this->e = $parameters['exception']; + if (!$this->e) { + throw new \InvalidArgumentException('Missing "exception" argument in FailedStorage constructor'); + } + } + + public function getId(): string { + // we can't return anything sane here + return 'failedstorage'; + } + + public function mkdir(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function rmdir(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function opendir(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function is_dir(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function is_file(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function stat(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function filetype(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function filesize(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function isCreatable(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function isReadable(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function isUpdatable(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function isDeletable(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function isSharable(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getPermissions(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function file_exists(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function filemtime(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function file_get_contents(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function file_put_contents(string $path, mixed $data): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function unlink(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function rename(string $source, string $target): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function copy(string $source, string $target): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function fopen(string $path, string $mode): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getMimeType(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function hash(string $type, string $path, bool $raw = false): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function free_space(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function touch(string $path, ?int $mtime = null): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getLocalFile(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function hasUpdated(string $path, int $time): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getETag(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getDirectDownload(string $path): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function verifyPath(string $path, string $fileName): void { + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, bool $preserveMtime = false): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function acquireLock(string $path, int $type, ILockingProvider $provider): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function releaseLock(string $path, int $type, ILockingProvider $provider): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function changeLock(string $path, int $type, ILockingProvider $provider): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getAvailability(): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function setAvailability(bool $isAvailable): never { + throw new StorageNotAvailableException($this->e->getMessage(), $this->e->getCode(), $this->e); + } + + public function getCache(string $path = '', ?IStorage $storage = null): FailedCache { + return new FailedCache(); + } +} diff --git a/lib/private/Files/Storage/Home.php b/lib/private/Files/Storage/Home.php new file mode 100644 index 00000000000..91b8071ac30 --- /dev/null +++ b/lib/private/Files/Storage/Home.php @@ -0,0 +1,76 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OC\Files\Cache\HomePropagator; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Storage\IStorage; +use OCP\IUser; + +/** + * Specialized version of Local storage for home directory usage + */ +class Home extends Local implements \OCP\Files\IHomeStorage { + /** + * @var string + */ + protected $id; + + /** + * @var \OC\User\User $user + */ + protected $user; + + /** + * Construct a Home storage instance + * + * @param array $parameters array with "user" containing the + * storage owner + */ + public function __construct(array $parameters) { + $this->user = $parameters['user']; + $datadir = $this->user->getHome(); + $this->id = 'home::' . $this->user->getUID(); + + parent::__construct(['datadir' => $datadir]); + } + + public function getId(): string { + return $this->id; + } + + public function getCache(string $path = '', ?IStorage $storage = null): ICache { + if (!$storage) { + $storage = $this; + } + if (!isset($this->cache)) { + $this->cache = new \OC\Files\Cache\HomeCache($storage, $this->getCacheDependencies()); + } + return $this->cache; + } + + public function getPropagator(?IStorage $storage = null): IPropagator { + if (!$storage) { + $storage = $this; + } + if (!isset($this->propagator)) { + $this->propagator = new HomePropagator($storage, \OC::$server->getDatabaseConnection()); + } + return $this->propagator; + } + + + public function getUser(): IUser { + return $this->user; + } + + public function getOwner(string $path): string|false { + return $this->user->getUID(); + } +} diff --git a/lib/private/Files/Storage/Local.php b/lib/private/Files/Storage/Local.php new file mode 100644 index 00000000000..260f9218a88 --- /dev/null +++ b/lib/private/Files/Storage/Local.php @@ -0,0 +1,594 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OC\Files\Filesystem; +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\Storage\Wrapper\Jail; +use OCP\Constants; +use OCP\Files\ForbiddenException; +use OCP\Files\GenericFileException; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\IConfig; +use OCP\Server; +use OCP\Util; +use Psr\Log\LoggerInterface; + +/** + * for local filestore, we only have to map the paths + */ +class Local extends \OC\Files\Storage\Common { + protected $datadir; + + protected $dataDirLength; + + protected $realDataDir; + + private IConfig $config; + + private IMimeTypeDetector $mimeTypeDetector; + + private $defUMask; + + protected bool $unlinkOnTruncate; + + protected bool $caseInsensitive = false; + + public function __construct(array $parameters) { + if (!isset($parameters['datadir']) || !is_string($parameters['datadir'])) { + throw new \InvalidArgumentException('No data directory set for local storage'); + } + $this->datadir = str_replace('//', '/', $parameters['datadir']); + // some crazy code uses a local storage on root... + if ($this->datadir === '/') { + $this->realDataDir = $this->datadir; + } else { + $realPath = realpath($this->datadir) ?: $this->datadir; + $this->realDataDir = rtrim($realPath, '/') . '/'; + } + if (!str_ends_with($this->datadir, '/')) { + $this->datadir .= '/'; + } + $this->dataDirLength = strlen($this->realDataDir); + $this->config = Server::get(IConfig::class); + $this->mimeTypeDetector = Server::get(IMimeTypeDetector::class); + $this->defUMask = $this->config->getSystemValue('localstorage.umask', 0022); + $this->caseInsensitive = $this->config->getSystemValueBool('localstorage.case_insensitive', false); + + // support Write-Once-Read-Many file systems + $this->unlinkOnTruncate = $this->config->getSystemValueBool('localstorage.unlink_on_truncate', false); + + if (isset($parameters['isExternal']) && $parameters['isExternal'] && !$this->stat('')) { + // data dir not accessible or available, can happen when using an external storage of type Local + // on an unmounted system mount point + throw new StorageNotAvailableException('Local storage path does not exist "' . $this->getSourcePath('') . '"'); + } + } + + public function __destruct() { + } + + public function getId(): string { + return 'local::' . $this->datadir; + } + + public function mkdir(string $path): bool { + $sourcePath = $this->getSourcePath($path); + $oldMask = umask($this->defUMask); + $result = @mkdir($sourcePath, 0777, true); + umask($oldMask); + return $result; + } + + public function rmdir(string $path): bool { + if (!$this->isDeletable($path)) { + return false; + } + try { + $it = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($this->getSourcePath($path)), + \RecursiveIteratorIterator::CHILD_FIRST + ); + /** + * RecursiveDirectoryIterator on an NFS path isn't iterable with foreach + * This bug is fixed in PHP 5.5.9 or before + * See #8376 + */ + $it->rewind(); + while ($it->valid()) { + /** + * @var \SplFileInfo $file + */ + $file = $it->current(); + clearstatcache(true, $file->getRealPath()); + if (in_array($file->getBasename(), ['.', '..'])) { + $it->next(); + continue; + } elseif ($file->isFile() || $file->isLink()) { + unlink($file->getPathname()); + } elseif ($file->isDir()) { + rmdir($file->getPathname()); + } + $it->next(); + } + unset($it); // Release iterator and thereby its potential directory lock (e.g. in case of VirtualBox shared folders) + clearstatcache(true, $this->getSourcePath($path)); + return rmdir($this->getSourcePath($path)); + } catch (\UnexpectedValueException $e) { + return false; + } + } + + public function opendir(string $path) { + return opendir($this->getSourcePath($path)); + } + + public function is_dir(string $path): bool { + if ($this->caseInsensitive && !$this->file_exists($path)) { + return false; + } + if (str_ends_with($path, '/')) { + $path = substr($path, 0, -1); + } + return is_dir($this->getSourcePath($path)); + } + + public function is_file(string $path): bool { + if ($this->caseInsensitive && !$this->file_exists($path)) { + return false; + } + return is_file($this->getSourcePath($path)); + } + + public function stat(string $path): array|false { + $fullPath = $this->getSourcePath($path); + clearstatcache(true, $fullPath); + if (!file_exists($fullPath)) { + return false; + } + $statResult = @stat($fullPath); + if (PHP_INT_SIZE === 4 && $statResult && !$this->is_dir($path)) { + $filesize = $this->filesize($path); + $statResult['size'] = $filesize; + $statResult[7] = $filesize; + } + if (is_array($statResult)) { + $statResult['full_path'] = $fullPath; + } + return $statResult; + } + + public function getMetaData(string $path): ?array { + try { + $stat = $this->stat($path); + } catch (ForbiddenException $e) { + return null; + } + if (!$stat) { + return null; + } + + $permissions = Constants::PERMISSION_SHARE; + $statPermissions = $stat['mode']; + $isDir = ($statPermissions & 0x4000) === 0x4000 && !($statPermissions & 0x8000); + if ($statPermissions & 0x0100) { + $permissions += Constants::PERMISSION_READ; + } + if ($statPermissions & 0x0080) { + $permissions += Constants::PERMISSION_UPDATE; + if ($isDir) { + $permissions += Constants::PERMISSION_CREATE; + } + } + + if (!($path === '' || $path === '/')) { // deletable depends on the parents unix permissions + $parent = dirname($stat['full_path']); + if (is_writable($parent)) { + $permissions += Constants::PERMISSION_DELETE; + } + } + + $data = []; + $data['mimetype'] = $isDir ? 'httpd/unix-directory' : $this->mimeTypeDetector->detectPath($path); + $data['mtime'] = $stat['mtime']; + if ($data['mtime'] === false) { + $data['mtime'] = time(); + } + if ($isDir) { + $data['size'] = -1; //unknown + } else { + $data['size'] = $stat['size']; + } + $data['etag'] = $this->calculateEtag($path, $stat); + $data['storage_mtime'] = $data['mtime']; + $data['permissions'] = $permissions; + $data['name'] = basename($path); + + return $data; + } + + public function filetype(string $path): string|false { + $filetype = filetype($this->getSourcePath($path)); + if ($filetype == 'link') { + $filetype = filetype(realpath($this->getSourcePath($path))); + } + return $filetype; + } + + public function filesize(string $path): int|float|false { + if (!$this->is_file($path)) { + return 0; + } + $fullPath = $this->getSourcePath($path); + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper; + return $helper->getFileSize($fullPath); + } + return filesize($fullPath); + } + + public function isReadable(string $path): bool { + return is_readable($this->getSourcePath($path)); + } + + public function isUpdatable(string $path): bool { + return is_writable($this->getSourcePath($path)); + } + + public function file_exists(string $path): bool { + if ($this->caseInsensitive) { + $fullPath = $this->getSourcePath($path); + $parentPath = dirname($fullPath); + if (!is_dir($parentPath)) { + return false; + } + $content = scandir($parentPath, SCANDIR_SORT_NONE); + return is_array($content) && array_search(basename($fullPath), $content) !== false; + } else { + return file_exists($this->getSourcePath($path)); + } + } + + public function filemtime(string $path): int|false { + $fullPath = $this->getSourcePath($path); + clearstatcache(true, $fullPath); + if (!$this->file_exists($path)) { + return false; + } + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper(); + return $helper->getFileMtime($fullPath); + } + return filemtime($fullPath); + } + + public function touch(string $path, ?int $mtime = null): bool { + // sets the modification time of the file to the given value. + // If mtime is nil the current time is set. + // note that the access time of the file always changes to the current time. + if ($this->file_exists($path) && !$this->isUpdatable($path)) { + return false; + } + $oldMask = umask($this->defUMask); + if (!is_null($mtime)) { + $result = @touch($this->getSourcePath($path), $mtime); + } else { + $result = @touch($this->getSourcePath($path)); + } + umask($oldMask); + if ($result) { + clearstatcache(true, $this->getSourcePath($path)); + } + + return $result; + } + + public function file_get_contents(string $path): string|false { + return file_get_contents($this->getSourcePath($path)); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + $oldMask = umask($this->defUMask); + if ($this->unlinkOnTruncate) { + $this->unlink($path); + } + $result = file_put_contents($this->getSourcePath($path), $data); + umask($oldMask); + return $result; + } + + public function unlink(string $path): bool { + if ($this->is_dir($path)) { + return $this->rmdir($path); + } elseif ($this->is_file($path)) { + return unlink($this->getSourcePath($path)); + } else { + return false; + } + } + + private function checkTreeForForbiddenItems(string $path): void { + $iterator = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($path)); + foreach ($iterator as $file) { + /** @var \SplFileInfo $file */ + if (Filesystem::isFileBlacklisted($file->getBasename())) { + throw new ForbiddenException('Invalid path: ' . $file->getPathname(), false); + } + } + } + + public function rename(string $source, string $target): bool { + $srcParent = dirname($source); + $dstParent = dirname($target); + + if (!$this->isUpdatable($srcParent)) { + Server::get(LoggerInterface::class)->error('unable to rename, source directory is not writable : ' . $srcParent, ['app' => 'core']); + return false; + } + + if (!$this->isUpdatable($dstParent)) { + Server::get(LoggerInterface::class)->error('unable to rename, destination directory is not writable : ' . $dstParent, ['app' => 'core']); + return false; + } + + if (!$this->file_exists($source)) { + Server::get(LoggerInterface::class)->error('unable to rename, file does not exists : ' . $source, ['app' => 'core']); + return false; + } + + if ($this->file_exists($target)) { + if ($this->is_dir($target)) { + $this->rmdir($target); + } elseif ($this->is_file($target)) { + $this->unlink($target); + } + } + + if ($this->is_dir($source)) { + $this->checkTreeForForbiddenItems($this->getSourcePath($source)); + } + + if (@rename($this->getSourcePath($source), $this->getSourcePath($target))) { + if ($this->caseInsensitive) { + if (mb_strtolower($target) === mb_strtolower($source) && !$this->file_exists($target)) { + return false; + } + } + return true; + } + + return $this->copy($source, $target) && $this->unlink($source); + } + + public function copy(string $source, string $target): bool { + if ($this->is_dir($source)) { + return parent::copy($source, $target); + } else { + $oldMask = umask($this->defUMask); + if ($this->unlinkOnTruncate) { + $this->unlink($target); + } + $result = copy($this->getSourcePath($source), $this->getSourcePath($target)); + umask($oldMask); + if ($this->caseInsensitive) { + if (mb_strtolower($target) === mb_strtolower($source) && !$this->file_exists($target)) { + return false; + } + } + return $result; + } + } + + public function fopen(string $path, string $mode) { + $sourcePath = $this->getSourcePath($path); + if (!file_exists($sourcePath) && $mode === 'r') { + return false; + } + $oldMask = umask($this->defUMask); + if (($mode === 'w' || $mode === 'w+') && $this->unlinkOnTruncate) { + $this->unlink($path); + } + $result = @fopen($sourcePath, $mode); + umask($oldMask); + return $result; + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + return hash_file($type, $this->getSourcePath($path), $raw); + } + + public function free_space(string $path): int|float|false { + $sourcePath = $this->getSourcePath($path); + // using !is_dir because $sourcePath might be a part file or + // non-existing file, so we'd still want to use the parent dir + // in such cases + if (!is_dir($sourcePath)) { + // disk_free_space doesn't work on files + $sourcePath = dirname($sourcePath); + } + $space = (function_exists('disk_free_space') && is_dir($sourcePath)) ? disk_free_space($sourcePath) : false; + if ($space === false || is_null($space)) { + return \OCP\Files\FileInfo::SPACE_UNKNOWN; + } + return Util::numericToNumber($space); + } + + public function search(string $query): array { + return $this->searchInDir($query); + } + + public function getLocalFile(string $path): string|false { + return $this->getSourcePath($path); + } + + protected function searchInDir(string $query, string $dir = ''): array { + $files = []; + $physicalDir = $this->getSourcePath($dir); + foreach (scandir($physicalDir) as $item) { + if (\OC\Files\Filesystem::isIgnoredDir($item)) { + continue; + } + $physicalItem = $physicalDir . '/' . $item; + + if (strstr(strtolower($item), strtolower($query)) !== false) { + $files[] = $dir . '/' . $item; + } + if (is_dir($physicalItem)) { + $files = array_merge($files, $this->searchInDir($query, $dir . '/' . $item)); + } + } + return $files; + } + + public function hasUpdated(string $path, int $time): bool { + if ($this->file_exists($path)) { + return $this->filemtime($path) > $time; + } else { + return true; + } + } + + /** + * Get the source path (on disk) of a given path + * + * @throws ForbiddenException + */ + public function getSourcePath(string $path): string { + if (Filesystem::isFileBlacklisted($path)) { + throw new ForbiddenException('Invalid path: ' . $path, false); + } + + $fullPath = $this->datadir . $path; + $currentPath = $path; + $allowSymlinks = $this->config->getSystemValueBool('localstorage.allowsymlinks', false); + if ($allowSymlinks || $currentPath === '') { + return $fullPath; + } + $pathToResolve = $fullPath; + $realPath = realpath($pathToResolve); + while ($realPath === false) { // for non existing files check the parent directory + $currentPath = dirname($currentPath); + /** @psalm-suppress TypeDoesNotContainType Let's be extra cautious and still check for empty string */ + if ($currentPath === '' || $currentPath === '.') { + return $fullPath; + } + $realPath = realpath($this->datadir . $currentPath); + } + if ($realPath) { + $realPath = $realPath . '/'; + } + if (substr($realPath, 0, $this->dataDirLength) === $this->realDataDir) { + return $fullPath; + } + + Server::get(LoggerInterface::class)->error("Following symlinks is not allowed ('$fullPath' -> '$realPath' not inside '{$this->realDataDir}')", ['app' => 'core']); + throw new ForbiddenException('Following symlinks is not allowed', false); + } + + public function isLocal(): bool { + return true; + } + + public function getETag(string $path): string|false { + return $this->calculateEtag($path, $this->stat($path)); + } + + private function calculateEtag(string $path, array $stat): string|false { + if ($stat['mode'] & 0x4000 && !($stat['mode'] & 0x8000)) { // is_dir & not socket + return parent::getETag($path); + } else { + if ($stat === false) { + return md5(''); + } + + $toHash = ''; + if (isset($stat['mtime'])) { + $toHash .= $stat['mtime']; + } + if (isset($stat['ino'])) { + $toHash .= $stat['ino']; + } + if (isset($stat['dev'])) { + $toHash .= $stat['dev']; + } + if (isset($stat['size'])) { + $toHash .= $stat['size']; + } + + return md5($toHash); + } + } + + private function canDoCrossStorageMove(IStorage $sourceStorage): bool { + /** @psalm-suppress UndefinedClass,InvalidArgument */ + return $sourceStorage->instanceOfStorage(Local::class) + // Don't treat ACLStorageWrapper like local storage where copy can be done directly. + // Instead, use the slower recursive copying in php from Common::copyFromStorage with + // more permissions checks. + && !$sourceStorage->instanceOfStorage('OCA\GroupFolders\ACL\ACLStorageWrapper') + // Same for access control + && !$sourceStorage->instanceOfStorage(\OCA\FilesAccessControl\StorageWrapper::class) + // when moving encrypted files we have to handle keys and the target might not be encrypted + && !$sourceStorage->instanceOfStorage(Encryption::class); + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, bool $preserveMtime = false): bool { + if ($this->canDoCrossStorageMove($sourceStorage)) { + // resolve any jailed paths + while ($sourceStorage->instanceOfStorage(Jail::class)) { + /** + * @var \OC\Files\Storage\Wrapper\Jail $sourceStorage + */ + $sourceInternalPath = $sourceStorage->getUnjailedPath($sourceInternalPath); + $sourceStorage = $sourceStorage->getUnjailedStorage(); + } + /** + * @var \OC\Files\Storage\Local $sourceStorage + */ + $rootStorage = new Local(['datadir' => '/']); + return $rootStorage->copy($sourceStorage->getSourcePath($sourceInternalPath), $this->getSourcePath($targetInternalPath)); + } else { + return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($this->canDoCrossStorageMove($sourceStorage)) { + // resolve any jailed paths + while ($sourceStorage->instanceOfStorage(Jail::class)) { + /** + * @var \OC\Files\Storage\Wrapper\Jail $sourceStorage + */ + $sourceInternalPath = $sourceStorage->getUnjailedPath($sourceInternalPath); + $sourceStorage = $sourceStorage->getUnjailedStorage(); + } + /** + * @var \OC\Files\Storage\Local $sourceStorage + */ + $rootStorage = new Local(['datadir' => '/']); + return $rootStorage->rename($sourceStorage->getSourcePath($sourceInternalPath), $this->getSourcePath($targetInternalPath)); + } else { + return parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + /** @var int|false $result We consider here that returned size will never be a float because we write less than 4GB */ + $result = $this->file_put_contents($path, $stream); + if (is_resource($stream)) { + fclose($stream); + } + if ($result === false) { + throw new GenericFileException("Failed write stream to $path"); + } else { + return $result; + } + } +} diff --git a/lib/private/Files/Storage/LocalRootStorage.php b/lib/private/Files/Storage/LocalRootStorage.php new file mode 100644 index 00000000000..2e0645e092a --- /dev/null +++ b/lib/private/Files/Storage/LocalRootStorage.php @@ -0,0 +1,22 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Storage; + +use OC\Files\Cache\LocalRootScanner; +use OCP\Files\Cache\IScanner; +use OCP\Files\Storage\IStorage; + +class LocalRootStorage extends Local { + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { + if (!$storage) { + $storage = $this; + } + return $storage->scanner ?? ($storage->scanner = new LocalRootScanner($storage)); + } +} diff --git a/lib/private/Files/Storage/LocalTempFileTrait.php b/lib/private/Files/Storage/LocalTempFileTrait.php new file mode 100644 index 00000000000..fffc3e789f3 --- /dev/null +++ b/lib/private/Files/Storage/LocalTempFileTrait.php @@ -0,0 +1,54 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OCP\Files; + +/** + * Storage backend class for providing common filesystem operation methods + * which are not storage-backend specific. + * + * \OC\Files\Storage\Common is never used directly; it is extended by all other + * storage backends, where its methods may be overridden, and additional + * (backend-specific) methods are defined. + * + * Some \OC\Files\Storage\Common methods call functions which are first defined + * in classes which extend it, e.g. $this->stat() . + */ +trait LocalTempFileTrait { + /** @var array<string,string|false> */ + protected array $cachedFiles = []; + + protected function getCachedFile(string $path): string|false { + if (!isset($this->cachedFiles[$path])) { + $this->cachedFiles[$path] = $this->toTmpFile($path); + } + return $this->cachedFiles[$path]; + } + + protected function removeCachedFile(string $path): void { + unset($this->cachedFiles[$path]); + } + + protected function toTmpFile(string $path): string|false { //no longer in the storage api, still useful here + $source = $this->fopen($path, 'r'); + if (!$source) { + return false; + } + if ($pos = strrpos($path, '.')) { + $extension = substr($path, $pos); + } else { + $extension = ''; + } + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($extension); + $target = fopen($tmpFile, 'w'); + Files::streamCopy($source, $target); + fclose($target); + return $tmpFile; + } +} diff --git a/lib/private/Files/Storage/PolyFill/CopyDirectory.php b/lib/private/Files/Storage/PolyFill/CopyDirectory.php new file mode 100644 index 00000000000..2f6167ef85e --- /dev/null +++ b/lib/private/Files/Storage/PolyFill/CopyDirectory.php @@ -0,0 +1,71 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\PolyFill; + +trait CopyDirectory { + /** + * Check if a path is a directory + */ + abstract public function is_dir(string $path): bool; + + /** + * Check if a file or folder exists + */ + abstract public function file_exists(string $path): bool; + + /** + * Delete a file or folder + */ + abstract public function unlink(string $path): bool; + + /** + * Open a directory handle for a folder + * + * @return resource|false + */ + abstract public function opendir(string $path); + + /** + * Create a new folder + */ + abstract public function mkdir(string $path): bool; + + public function copy(string $source, string $target): bool { + if ($this->is_dir($source)) { + if ($this->file_exists($target)) { + $this->unlink($target); + } + $this->mkdir($target); + return $this->copyRecursive($source, $target); + } else { + return parent::copy($source, $target); + } + } + + /** + * For adapters that don't support copying folders natively + */ + protected function copyRecursive(string $source, string $target): bool { + $dh = $this->opendir($source); + $result = true; + while (($file = readdir($dh)) !== false) { + if (!\OC\Files\Filesystem::isIgnoredDir($file)) { + if ($this->is_dir($source . '/' . $file)) { + $this->mkdir($target . '/' . $file); + $result = $this->copyRecursive($source . '/' . $file, $target . '/' . $file); + } else { + $result = parent::copy($source . '/' . $file, $target . '/' . $file); + } + if (!$result) { + break; + } + } + } + return $result; + } +} diff --git a/lib/private/Files/Storage/Storage.php b/lib/private/Files/Storage/Storage.php new file mode 100644 index 00000000000..aa17c12b309 --- /dev/null +++ b/lib/private/Files/Storage/Storage.php @@ -0,0 +1,53 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\Storage; + +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Cache\IScanner; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Cache\IWatcher; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IStorage; + +/** + * Provide a common interface to all different storage options + * + * All paths passed to the storage are relative to the storage and should NOT have a leading slash. + */ +interface Storage extends IStorage, ILockingStorage { + public function getCache(string $path = '', ?IStorage $storage = null): ICache; + + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner; + + public function getWatcher(string $path = '', ?IStorage $storage = null): IWatcher; + + public function getPropagator(?IStorage $storage = null): IPropagator; + + public function getUpdater(?IStorage $storage = null): IUpdater; + + public function getStorageCache(): \OC\Files\Cache\Storage; + + public function getMetaData(string $path): ?array; + + /** + * Get the contents of a directory with metadata + * + * The metadata array will contain the following fields + * + * - name + * - mimetype + * - mtime + * - size + * - etag + * - storage_mtime + * - permissions + */ + public function getDirectoryContent(string $directory): \Traversable; +} diff --git a/lib/private/Files/Storage/StorageFactory.php b/lib/private/Files/Storage/StorageFactory.php new file mode 100644 index 00000000000..603df7fe007 --- /dev/null +++ b/lib/private/Files/Storage/StorageFactory.php @@ -0,0 +1,73 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Storage\IConstructableStorage; +use OCP\Files\Storage\IStorage; +use OCP\Files\Storage\IStorageFactory; +use Psr\Log\LoggerInterface; + +class StorageFactory implements IStorageFactory { + /** + * @var array[] [$name=>['priority'=>$priority, 'wrapper'=>$callable] $storageWrappers + */ + private $storageWrappers = []; + + public function addStorageWrapper(string $wrapperName, callable $callback, int $priority = 50, array $existingMounts = []): bool { + if (isset($this->storageWrappers[$wrapperName])) { + return false; + } + + // apply to existing mounts before registering it to prevent applying it double in MountPoint::createStorage + foreach ($existingMounts as $mount) { + $mount->wrapStorage($callback); + } + + $this->storageWrappers[$wrapperName] = ['wrapper' => $callback, 'priority' => $priority]; + return true; + } + + /** + * Remove a storage wrapper by name. + * Note: internal method only to be used for cleanup + * + * @internal + */ + public function removeStorageWrapper(string $wrapperName): void { + unset($this->storageWrappers[$wrapperName]); + } + + /** + * Create an instance of a storage and apply the registered storage wrappers + */ + public function getInstance(IMountPoint $mountPoint, string $class, array $arguments): IStorage { + if (!is_a($class, IConstructableStorage::class, true)) { + \OCP\Server::get(LoggerInterface::class)->warning('Building a storage not implementing IConstructableStorage is deprecated since 31.0.0', ['class' => $class]); + } + return $this->wrap($mountPoint, new $class($arguments)); + } + + public function wrap(IMountPoint $mountPoint, IStorage $storage): IStorage { + $wrappers = array_values($this->storageWrappers); + usort($wrappers, function ($a, $b) { + return $b['priority'] - $a['priority']; + }); + /** @var callable[] $wrappers */ + $wrappers = array_map(function ($wrapper) { + return $wrapper['wrapper']; + }, $wrappers); + foreach ($wrappers as $wrapper) { + $storage = $wrapper($mountPoint->getMountPoint(), $storage, $mountPoint); + if (!($storage instanceof IStorage)) { + throw new \Exception('Invalid result from storage wrapper'); + } + } + return $storage; + } +} diff --git a/lib/private/Files/Storage/Temporary.php b/lib/private/Files/Storage/Temporary.php new file mode 100644 index 00000000000..ecf8a1315a9 --- /dev/null +++ b/lib/private/Files/Storage/Temporary.php @@ -0,0 +1,34 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage; + +use OCP\Files; +use OCP\ITempManager; +use OCP\Server; + +/** + * local storage backend in temporary folder for testing purpose + */ +class Temporary extends Local { + public function __construct(array $parameters = []) { + parent::__construct(['datadir' => Server::get(ITempManager::class)->getTemporaryFolder()]); + } + + public function cleanUp(): void { + Files::rmdirr($this->datadir); + } + + public function __destruct() { + parent::__destruct(); + $this->cleanUp(); + } + + public function getDataDir(): array|string { + return $this->datadir; + } +} diff --git a/lib/private/Files/Storage/Wrapper/Availability.php b/lib/private/Files/Storage/Wrapper/Availability.php new file mode 100644 index 00000000000..32c51a1b25e --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Availability.php @@ -0,0 +1,277 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageAuthException; +use OCP\Files\StorageNotAvailableException; +use OCP\IConfig; + +/** + * Availability checker for storages + * + * Throws a StorageNotAvailableException for storages with known failures + */ +class Availability extends Wrapper { + public const RECHECK_TTL_SEC = 600; // 10 minutes + + /** @var IConfig */ + protected $config; + + public function __construct(array $parameters) { + $this->config = $parameters['config'] ?? \OCP\Server::get(IConfig::class); + parent::__construct($parameters); + } + + public static function shouldRecheck($availability): bool { + if (!$availability['available']) { + // trigger a recheck if TTL reached + if ((time() - $availability['last_checked']) > self::RECHECK_TTL_SEC) { + return true; + } + } + return false; + } + + /** + * Only called if availability === false + */ + private function updateAvailability(): bool { + // reset availability to false so that multiple requests don't recheck concurrently + $this->setAvailability(false); + try { + $result = $this->test(); + } catch (\Exception $e) { + $result = false; + } + $this->setAvailability($result); + return $result; + } + + private function isAvailable(): bool { + $availability = $this->getAvailability(); + if (self::shouldRecheck($availability)) { + return $this->updateAvailability(); + } + return $availability['available']; + } + + /** + * @throws StorageNotAvailableException + */ + private function checkAvailability(): void { + if (!$this->isAvailable()) { + throw new StorageNotAvailableException(); + } + } + + /** + * Handles availability checks and delegates method calls dynamically + */ + private function handleAvailability(string $method, mixed ...$args): mixed { + $this->checkAvailability(); + try { + return call_user_func_array([parent::class, $method], $args); + } catch (StorageNotAvailableException $e) { + $this->setUnavailable($e); + return false; + } + } + + public function mkdir(string $path): bool { + return $this->handleAvailability('mkdir', $path); + } + + public function rmdir(string $path): bool { + return $this->handleAvailability('rmdir', $path); + } + + public function opendir(string $path) { + return $this->handleAvailability('opendir', $path); + } + + public function is_dir(string $path): bool { + return $this->handleAvailability('is_dir', $path); + } + + public function is_file(string $path): bool { + return $this->handleAvailability('is_file', $path); + } + + public function stat(string $path): array|false { + return $this->handleAvailability('stat', $path); + } + + public function filetype(string $path): string|false { + return $this->handleAvailability('filetype', $path); + } + + public function filesize(string $path): int|float|false { + return $this->handleAvailability('filesize', $path); + } + + public function isCreatable(string $path): bool { + return $this->handleAvailability('isCreatable', $path); + } + + public function isReadable(string $path): bool { + return $this->handleAvailability('isReadable', $path); + } + + public function isUpdatable(string $path): bool { + return $this->handleAvailability('isUpdatable', $path); + } + + public function isDeletable(string $path): bool { + return $this->handleAvailability('isDeletable', $path); + } + + public function isSharable(string $path): bool { + return $this->handleAvailability('isSharable', $path); + } + + public function getPermissions(string $path): int { + return $this->handleAvailability('getPermissions', $path); + } + + public function file_exists(string $path): bool { + if ($path === '') { + return true; + } + return $this->handleAvailability('file_exists', $path); + } + + public function filemtime(string $path): int|false { + return $this->handleAvailability('filemtime', $path); + } + + public function file_get_contents(string $path): string|false { + return $this->handleAvailability('file_get_contents', $path); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + return $this->handleAvailability('file_put_contents', $path, $data); + } + + public function unlink(string $path): bool { + return $this->handleAvailability('unlink', $path); + } + + public function rename(string $source, string $target): bool { + return $this->handleAvailability('rename', $source, $target); + } + + public function copy(string $source, string $target): bool { + return $this->handleAvailability('copy', $source, $target); + } + + public function fopen(string $path, string $mode) { + return $this->handleAvailability('fopen', $path, $mode); + } + + public function getMimeType(string $path): string|false { + return $this->handleAvailability('getMimeType', $path); + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + return $this->handleAvailability('hash', $type, $path, $raw); + } + + public function free_space(string $path): int|float|false { + return $this->handleAvailability('free_space', $path); + } + + public function touch(string $path, ?int $mtime = null): bool { + return $this->handleAvailability('touch', $path, $mtime); + } + + public function getLocalFile(string $path): string|false { + return $this->handleAvailability('getLocalFile', $path); + } + + public function hasUpdated(string $path, int $time): bool { + if (!$this->isAvailable()) { + return false; + } + try { + return parent::hasUpdated($path, $time); + } catch (StorageNotAvailableException $e) { + // set unavailable but don't rethrow + $this->setUnavailable(null); + return false; + } + } + + public function getOwner(string $path): string|false { + try { + return parent::getOwner($path); + } catch (StorageNotAvailableException $e) { + $this->setUnavailable($e); + return false; + } + } + + public function getETag(string $path): string|false { + return $this->handleAvailability('getETag', $path); + } + + public function getDirectDownload(string $path): array|false { + return $this->handleAvailability('getDirectDownload', $path); + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + return $this->handleAvailability('copyFromStorage', $sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + return $this->handleAvailability('moveFromStorage', $sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function getMetaData(string $path): ?array { + $this->checkAvailability(); + try { + return parent::getMetaData($path); + } catch (StorageNotAvailableException $e) { + $this->setUnavailable($e); + return null; + } + } + + /** + * @template T of StorageNotAvailableException|null + * @param T $e + * @psalm-return (T is null ? void : never) + * @throws StorageNotAvailableException + */ + protected function setUnavailable(?StorageNotAvailableException $e): void { + $delay = self::RECHECK_TTL_SEC; + if ($e instanceof StorageAuthException) { + $delay = max( + // 30min + $this->config->getSystemValueInt('external_storage.auth_availability_delay', 1800), + self::RECHECK_TTL_SEC + ); + } + $this->getStorageCache()->setAvailability(false, $delay); + if ($e !== null) { + throw $e; + } + } + + + + public function getDirectoryContent(string $directory): \Traversable { + $this->checkAvailability(); + try { + return parent::getDirectoryContent($directory); + } catch (StorageNotAvailableException $e) { + $this->setUnavailable($e); + return new \EmptyIterator(); + } + } +} diff --git a/lib/private/Files/Storage/Wrapper/Encoding.php b/lib/private/Files/Storage/Wrapper/Encoding.php new file mode 100644 index 00000000000..92e20cfb3df --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Encoding.php @@ -0,0 +1,296 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Files\Filesystem; +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Cache\IScanner; +use OCP\Files\Storage\IStorage; +use OCP\ICache; + +/** + * Encoding wrapper that deals with file names that use unsupported encodings like NFD. + * + * When applied and a UTF-8 path name was given, the wrapper will first attempt to access + * the actual given name and then try its NFD form. + */ +class Encoding extends Wrapper { + /** + * @var ICache + */ + private $namesCache; + + /** + * @param array $parameters + */ + public function __construct(array $parameters) { + $this->storage = $parameters['storage']; + $this->namesCache = new CappedMemoryCache(); + } + + /** + * Returns whether the given string is only made of ASCII characters + */ + private function isAscii(string $str): bool { + return !preg_match('/[\\x80-\\xff]+/', $str); + } + + /** + * Checks whether the given path exists in NFC or NFD form after checking + * each form for each path section and returns the correct form. + * If no existing path found, returns the path as it was given. + * + * @return string original or converted path + */ + private function findPathToUse(string $fullPath): string { + $cachedPath = $this->namesCache[$fullPath]; + if ($cachedPath !== null) { + return $cachedPath; + } + + $sections = explode('/', $fullPath); + $path = ''; + foreach ($sections as $section) { + $convertedPath = $this->findPathToUseLastSection($path, $section); + if ($convertedPath === null) { + // no point in continuing if the section was not found, use original path + return $fullPath; + } + $path = $convertedPath . '/'; + } + $path = rtrim($path, '/'); + return $path; + } + + /** + * Checks whether the last path section of the given path exists in NFC or NFD form + * and returns the correct form. If no existing path found, returns null. + * + * @param string $lastSection last section of the path to check for NFD/NFC variations + * + * @return string|null original or converted path, or null if none of the forms was found + */ + private function findPathToUseLastSection(string $basePath, string $lastSection): ?string { + $fullPath = $basePath . $lastSection; + if ($lastSection === '' || $this->isAscii($lastSection) || $this->storage->file_exists($fullPath)) { + $this->namesCache[$fullPath] = $fullPath; + return $fullPath; + } + + // swap encoding + if (\Normalizer::isNormalized($lastSection, \Normalizer::FORM_C)) { + $otherFormPath = \Normalizer::normalize($lastSection, \Normalizer::FORM_D); + } else { + $otherFormPath = \Normalizer::normalize($lastSection, \Normalizer::FORM_C); + } + $otherFullPath = $basePath . $otherFormPath; + if ($this->storage->file_exists($otherFullPath)) { + $this->namesCache[$fullPath] = $otherFullPath; + return $otherFullPath; + } + + // return original path, file did not exist at all + $this->namesCache[$fullPath] = $fullPath; + return null; + } + + public function mkdir(string $path): bool { + // note: no conversion here, method should not be called with non-NFC names! + $result = $this->storage->mkdir($path); + if ($result) { + $this->namesCache[$path] = $path; + } + return $result; + } + + public function rmdir(string $path): bool { + $result = $this->storage->rmdir($this->findPathToUse($path)); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + public function opendir(string $path) { + $handle = $this->storage->opendir($this->findPathToUse($path)); + return EncodingDirectoryWrapper::wrap($handle); + } + + public function is_dir(string $path): bool { + return $this->storage->is_dir($this->findPathToUse($path)); + } + + public function is_file(string $path): bool { + return $this->storage->is_file($this->findPathToUse($path)); + } + + public function stat(string $path): array|false { + return $this->storage->stat($this->findPathToUse($path)); + } + + public function filetype(string $path): string|false { + return $this->storage->filetype($this->findPathToUse($path)); + } + + public function filesize(string $path): int|float|false { + return $this->storage->filesize($this->findPathToUse($path)); + } + + public function isCreatable(string $path): bool { + return $this->storage->isCreatable($this->findPathToUse($path)); + } + + public function isReadable(string $path): bool { + return $this->storage->isReadable($this->findPathToUse($path)); + } + + public function isUpdatable(string $path): bool { + return $this->storage->isUpdatable($this->findPathToUse($path)); + } + + public function isDeletable(string $path): bool { + return $this->storage->isDeletable($this->findPathToUse($path)); + } + + public function isSharable(string $path): bool { + return $this->storage->isSharable($this->findPathToUse($path)); + } + + public function getPermissions(string $path): int { + return $this->storage->getPermissions($this->findPathToUse($path)); + } + + public function file_exists(string $path): bool { + return $this->storage->file_exists($this->findPathToUse($path)); + } + + public function filemtime(string $path): int|false { + return $this->storage->filemtime($this->findPathToUse($path)); + } + + public function file_get_contents(string $path): string|false { + return $this->storage->file_get_contents($this->findPathToUse($path)); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + return $this->storage->file_put_contents($this->findPathToUse($path), $data); + } + + public function unlink(string $path): bool { + $result = $this->storage->unlink($this->findPathToUse($path)); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + public function rename(string $source, string $target): bool { + // second name always NFC + return $this->storage->rename($this->findPathToUse($source), $this->findPathToUse($target)); + } + + public function copy(string $source, string $target): bool { + return $this->storage->copy($this->findPathToUse($source), $this->findPathToUse($target)); + } + + public function fopen(string $path, string $mode) { + $result = $this->storage->fopen($this->findPathToUse($path), $mode); + if ($result && $mode !== 'r' && $mode !== 'rb') { + unset($this->namesCache[$path]); + } + return $result; + } + + public function getMimeType(string $path): string|false { + return $this->storage->getMimeType($this->findPathToUse($path)); + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + return $this->storage->hash($type, $this->findPathToUse($path), $raw); + } + + public function free_space(string $path): int|float|false { + return $this->storage->free_space($this->findPathToUse($path)); + } + + public function touch(string $path, ?int $mtime = null): bool { + return $this->storage->touch($this->findPathToUse($path), $mtime); + } + + public function getLocalFile(string $path): string|false { + return $this->storage->getLocalFile($this->findPathToUse($path)); + } + + public function hasUpdated(string $path, int $time): bool { + return $this->storage->hasUpdated($this->findPathToUse($path), $time); + } + + public function getCache(string $path = '', ?IStorage $storage = null): \OCP\Files\Cache\ICache { + if (!$storage) { + $storage = $this; + } + return $this->storage->getCache($this->findPathToUse($path), $storage); + } + + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { + if (!$storage) { + $storage = $this; + } + return $this->storage->getScanner($this->findPathToUse($path), $storage); + } + + public function getETag(string $path): string|false { + return $this->storage->getETag($this->findPathToUse($path)); + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + return $this->copy($sourceInternalPath, $this->findPathToUse($targetInternalPath)); + } + + $result = $this->storage->copyFromStorage($sourceStorage, $sourceInternalPath, $this->findPathToUse($targetInternalPath)); + if ($result) { + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + $result = $this->rename($sourceInternalPath, $this->findPathToUse($targetInternalPath)); + if ($result) { + unset($this->namesCache[$sourceInternalPath]); + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + $result = $this->storage->moveFromStorage($sourceStorage, $sourceInternalPath, $this->findPathToUse($targetInternalPath)); + if ($result) { + unset($this->namesCache[$sourceInternalPath]); + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + public function getMetaData(string $path): ?array { + $entry = $this->storage->getMetaData($this->findPathToUse($path)); + if ($entry !== null) { + $entry['name'] = trim(Filesystem::normalizePath($entry['name']), '/'); + } + return $entry; + } + + public function getDirectoryContent(string $directory): \Traversable { + $entries = $this->storage->getDirectoryContent($this->findPathToUse($directory)); + foreach ($entries as $entry) { + $entry['name'] = trim(Filesystem::normalizePath($entry['name']), '/'); + yield $entry; + } + } +} diff --git a/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php b/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php new file mode 100644 index 00000000000..0a90b49f0f1 --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php @@ -0,0 +1,34 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use Icewind\Streams\DirectoryWrapper; +use OC\Files\Filesystem; + +/** + * Normalize file names while reading directory entries + */ +class EncodingDirectoryWrapper extends DirectoryWrapper { + public function dir_readdir(): string|false { + $file = readdir($this->source); + if ($file !== false && $file !== '.' && $file !== '..') { + $file = trim(Filesystem::normalizePath($file), '/'); + } + + return $file; + } + + /** + * @param resource $source + * @return resource|false + */ + public static function wrap($source) { + return self::wrapSource($source, [ + 'source' => $source, + ]); + } +} diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php new file mode 100644 index 00000000000..58bd4dfddcf --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -0,0 +1,946 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Encryption\Exceptions\ModuleDoesNotExistsException; +use OC\Encryption\Util; +use OC\Files\Cache\CacheEntry; +use OC\Files\Filesystem; +use OC\Files\Mount\Manager; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\Storage\Common; +use OC\Files\Storage\LocalTempFileTrait; +use OC\Memcache\ArrayCache; +use OCP\Cache\CappedMemoryCache; +use OCP\Encryption\Exceptions\InvalidHeaderException; +use OCP\Encryption\IFile; +use OCP\Encryption\IManager; +use OCP\Encryption\Keys\IStorage; +use OCP\Files; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\GenericFileException; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\Storage; +use Psr\Log\LoggerInterface; + +class Encryption extends Wrapper { + use LocalTempFileTrait; + + private string $mountPoint; + protected array $unencryptedSize = []; + private IMountPoint $mount; + /** for which path we execute the repair step to avoid recursions */ + private array $fixUnencryptedSizeOf = []; + /** @var CappedMemoryCache<bool> */ + private CappedMemoryCache $encryptedPaths; + private bool $enabled = true; + + /** + * @param array $parameters + */ + public function __construct( + array $parameters, + private IManager $encryptionManager, + private Util $util, + private LoggerInterface $logger, + private IFile $fileHelper, + private ?string $uid, + private IStorage $keyStorage, + private Manager $mountManager, + private ArrayCache $arrayCache, + ) { + $this->mountPoint = $parameters['mountPoint']; + $this->mount = $parameters['mount']; + $this->encryptedPaths = new CappedMemoryCache(); + parent::__construct($parameters); + } + + public function filesize(string $path): int|float|false { + $fullPath = $this->getFullPath($path); + + $info = $this->getCache()->get($path); + if ($info === false) { + /* Pass call to wrapped storage, it may be a special file like a part file */ + return $this->storage->filesize($path); + } + if (isset($this->unencryptedSize[$fullPath])) { + $size = $this->unencryptedSize[$fullPath]; + + // Update file cache (only if file is already cached). + // Certain files are not cached (e.g. *.part). + if (isset($info['fileid'])) { + if ($info instanceof ICacheEntry) { + $info['encrypted'] = $info['encryptedVersion']; + } else { + /** + * @psalm-suppress RedundantCondition + */ + if (!is_array($info)) { + $info = []; + } + $info['encrypted'] = true; + $info = new CacheEntry($info); + } + + if ($size !== $info->getUnencryptedSize()) { + $this->getCache()->update($info->getId(), [ + 'unencrypted_size' => $size + ]); + } + } + + return $size; + } + + if (isset($info['fileid']) && $info['encrypted']) { + return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); + } + + return $this->storage->filesize($path); + } + + private function modifyMetaData(string $path, array $data): array { + $fullPath = $this->getFullPath($path); + $info = $this->getCache()->get($path); + + if (isset($this->unencryptedSize[$fullPath])) { + $data['encrypted'] = true; + $data['size'] = $this->unencryptedSize[$fullPath]; + $data['unencrypted_size'] = $data['size']; + } else { + if (isset($info['fileid']) && $info['encrypted']) { + $data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); + $data['encrypted'] = true; + $data['unencrypted_size'] = $data['size']; + } + } + + if (isset($info['encryptedVersion']) && $info['encryptedVersion'] > 1) { + $data['encryptedVersion'] = $info['encryptedVersion']; + } + + return $data; + } + + public function getMetaData(string $path): ?array { + $data = $this->storage->getMetaData($path); + if (is_null($data)) { + return null; + } + return $this->modifyMetaData($path, $data); + } + + public function getDirectoryContent(string $directory): \Traversable { + $parent = rtrim($directory, '/'); + foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) { + yield $this->modifyMetaData($parent . '/' . $data['name'], $data); + } + } + + public function file_get_contents(string $path): string|false { + $encryptionModule = $this->getEncryptionModule($path); + + if ($encryptionModule) { + $handle = $this->fopen($path, 'r'); + if (!$handle) { + return false; + } + $data = stream_get_contents($handle); + fclose($handle); + return $data; + } + return $this->storage->file_get_contents($path); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + // file put content will always be translated to a stream write + $handle = $this->fopen($path, 'w'); + if (is_resource($handle)) { + $written = fwrite($handle, $data); + fclose($handle); + return $written; + } + + return false; + } + + public function unlink(string $path): bool { + $fullPath = $this->getFullPath($path); + if ($this->util->isExcluded($fullPath)) { + return $this->storage->unlink($path); + } + + $encryptionModule = $this->getEncryptionModule($path); + if ($encryptionModule) { + $this->keyStorage->deleteAllFileKeys($fullPath); + } + + return $this->storage->unlink($path); + } + + public function rename(string $source, string $target): bool { + $result = $this->storage->rename($source, $target); + + if ($result + // versions always use the keys from the original file, so we can skip + // this step for versions + && $this->isVersion($target) === false + && $this->encryptionManager->isEnabled()) { + $sourcePath = $this->getFullPath($source); + if (!$this->util->isExcluded($sourcePath)) { + $targetPath = $this->getFullPath($target); + if (isset($this->unencryptedSize[$sourcePath])) { + $this->unencryptedSize[$targetPath] = $this->unencryptedSize[$sourcePath]; + } + $this->keyStorage->renameKeys($sourcePath, $targetPath); + $module = $this->getEncryptionModule($target); + if ($module) { + $module->update($targetPath, $this->uid, []); + } + } + } + + return $result; + } + + public function rmdir(string $path): bool { + $result = $this->storage->rmdir($path); + $fullPath = $this->getFullPath($path); + if ($result + && $this->util->isExcluded($fullPath) === false + && $this->encryptionManager->isEnabled() + ) { + $this->keyStorage->deleteAllFileKeys($fullPath); + } + + return $result; + } + + public function isReadable(string $path): bool { + $isReadable = true; + + $metaData = $this->getMetaData($path); + if ( + !$this->is_dir($path) + && isset($metaData['encrypted']) + && $metaData['encrypted'] === true + ) { + $fullPath = $this->getFullPath($path); + $module = $this->getEncryptionModule($path); + $isReadable = $module->isReadable($fullPath, $this->uid); + } + + return $this->storage->isReadable($path) && $isReadable; + } + + public function copy(string $source, string $target): bool { + $sourcePath = $this->getFullPath($source); + + if ($this->util->isExcluded($sourcePath)) { + return $this->storage->copy($source, $target); + } + + // need to stream copy file by file in case we copy between a encrypted + // and a unencrypted storage + $this->unlink($target); + return $this->copyFromStorage($this, $source, $target); + } + + public function fopen(string $path, string $mode) { + // check if the file is stored in the array cache, this means that we + // copy a file over to the versions folder, in this case we don't want to + // decrypt it + if ($this->arrayCache->hasKey('encryption_copy_version_' . $path)) { + $this->arrayCache->remove('encryption_copy_version_' . $path); + return $this->storage->fopen($path, $mode); + } + + if (!$this->enabled) { + return $this->storage->fopen($path, $mode); + } + + $encryptionEnabled = $this->encryptionManager->isEnabled(); + $shouldEncrypt = false; + $encryptionModule = null; + $header = $this->getHeader($path); + $signed = isset($header['signed']) && $header['signed'] === 'true'; + $fullPath = $this->getFullPath($path); + $encryptionModuleId = $this->util->getEncryptionModuleId($header); + + if ($this->util->isExcluded($fullPath) === false) { + $size = $unencryptedSize = 0; + $realFile = $this->util->stripPartialFileExtension($path); + $targetExists = $this->is_file($realFile) || $this->file_exists($path); + $targetIsEncrypted = false; + if ($targetExists) { + // in case the file exists we require the explicit module as + // specified in the file header - otherwise we need to fail hard to + // prevent data loss on client side + if (!empty($encryptionModuleId)) { + $targetIsEncrypted = true; + $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); + } + + if ($this->file_exists($path)) { + $size = $this->storage->filesize($path); + $unencryptedSize = $this->filesize($path); + } else { + $size = $unencryptedSize = 0; + } + } + + try { + if ( + $mode === 'w' + || $mode === 'w+' + || $mode === 'wb' + || $mode === 'wb+' + ) { + // if we update a encrypted file with a un-encrypted one we change the db flag + if ($targetIsEncrypted && $encryptionEnabled === false) { + $cache = $this->storage->getCache(); + $entry = $cache->get($path); + $cache->update($entry->getId(), ['encrypted' => 0]); + } + if ($encryptionEnabled) { + // if $encryptionModuleId is empty, the default module will be used + $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); + $shouldEncrypt = $encryptionModule->shouldEncrypt($fullPath); + $signed = true; + } + } else { + $info = $this->getCache()->get($path); + // only get encryption module if we found one in the header + // or if file should be encrypted according to the file cache + if (!empty($encryptionModuleId)) { + $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); + $shouldEncrypt = true; + } elseif ($info !== false && $info['encrypted'] === true) { + // we come from a old installation. No header and/or no module defined + // but the file is encrypted. In this case we need to use the + // OC_DEFAULT_MODULE to read the file + $encryptionModule = $this->encryptionManager->getEncryptionModule('OC_DEFAULT_MODULE'); + $shouldEncrypt = true; + $targetIsEncrypted = true; + } + } + } catch (ModuleDoesNotExistsException $e) { + $this->logger->warning('Encryption module "' . $encryptionModuleId . '" not found, file will be stored unencrypted', [ + 'exception' => $e, + 'app' => 'core', + ]); + } + + // encryption disabled on write of new file and write to existing unencrypted file -> don't encrypt + if (!$encryptionEnabled || !$this->shouldEncrypt($path)) { + if (!$targetExists || !$targetIsEncrypted) { + $shouldEncrypt = false; + } + } + + if ($shouldEncrypt === true && $encryptionModule !== null) { + $this->encryptedPaths->set($this->util->stripPartialFileExtension($path), true); + $headerSize = $this->getHeaderSize($path); + if ($mode === 'r' && $headerSize === 0) { + $firstBlock = $this->readFirstBlock($path); + if (!$firstBlock) { + throw new InvalidHeaderException("Unable to get header block for $path"); + } elseif (!str_starts_with($firstBlock, Util::HEADER_START)) { + throw new InvalidHeaderException("Unable to get header size for $path, file doesn't start with encryption header"); + } else { + throw new InvalidHeaderException("Unable to get header size for $path, even though file does start with encryption header"); + } + } + $source = $this->storage->fopen($path, $mode); + if (!is_resource($source)) { + return false; + } + $handle = \OC\Files\Stream\Encryption::wrap($source, $path, $fullPath, $header, + $this->uid, $encryptionModule, $this->storage, $this, $this->util, $this->fileHelper, $mode, + $size, $unencryptedSize, $headerSize, $signed); + + return $handle; + } + } + + return $this->storage->fopen($path, $mode); + } + + + /** + * perform some plausibility checks if the unencrypted size is correct. + * If not, we calculate the correct unencrypted size and return it + * + * @param string $path internal path relative to the storage root + * @param int $unencryptedSize size of the unencrypted file + * + * @return int unencrypted size + */ + protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int { + $size = $this->storage->filesize($path); + $result = $unencryptedSize; + + if ($unencryptedSize < 0 + || ($size > 0 && $unencryptedSize === $size) + || $unencryptedSize > $size + ) { + // check if we already calculate the unencrypted size for the + // given path to avoid recursions + if (isset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]) === false) { + $this->fixUnencryptedSizeOf[$this->getFullPath($path)] = true; + try { + $result = $this->fixUnencryptedSize($path, $size, $unencryptedSize); + } catch (\Exception $e) { + $this->logger->error('Couldn\'t re-calculate unencrypted size for ' . $path, ['exception' => $e]); + } + unset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]); + } + } + + return $result; + } + + /** + * calculate the unencrypted size + * + * @param string $path internal path relative to the storage root + * @param int $size size of the physical file + * @param int $unencryptedSize size of the unencrypted file + */ + protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int|float { + $headerSize = $this->getHeaderSize($path); + $header = $this->getHeader($path); + $encryptionModule = $this->getEncryptionModule($path); + + $stream = $this->storage->fopen($path, 'r'); + + // if we couldn't open the file we return the old unencrypted size + if (!is_resource($stream)) { + $this->logger->error('Could not open ' . $path . '. Recalculation of unencrypted size aborted.'); + return $unencryptedSize; + } + + $newUnencryptedSize = 0; + $size -= $headerSize; + $blockSize = $this->util->getBlockSize(); + + // if a header exists we skip it + if ($headerSize > 0) { + $this->fread_block($stream, $headerSize); + } + + // fast path, else the calculation for $lastChunkNr is bogus + if ($size === 0) { + return 0; + } + + $signed = isset($header['signed']) && $header['signed'] === 'true'; + $unencryptedBlockSize = $encryptionModule->getUnencryptedBlockSize($signed); + + // calculate last chunk nr + // next highest is end of chunks, one subtracted is last one + // we have to read the last chunk, we can't just calculate it (because of padding etc) + + $lastChunkNr = ceil($size / $blockSize) - 1; + // calculate last chunk position + $lastChunkPos = ($lastChunkNr * $blockSize); + // try to fseek to the last chunk, if it fails we have to read the whole file + if (@fseek($stream, $lastChunkPos, SEEK_CUR) === 0) { + $newUnencryptedSize += $lastChunkNr * $unencryptedBlockSize; + } + + $lastChunkContentEncrypted = ''; + $count = $blockSize; + + while ($count > 0) { + $data = $this->fread_block($stream, $blockSize); + $count = strlen($data); + $lastChunkContentEncrypted .= $data; + if (strlen($lastChunkContentEncrypted) > $blockSize) { + $newUnencryptedSize += $unencryptedBlockSize; + $lastChunkContentEncrypted = substr($lastChunkContentEncrypted, $blockSize); + } + } + + fclose($stream); + + // we have to decrypt the last chunk to get it actual size + $encryptionModule->begin($this->getFullPath($path), $this->uid, 'r', $header, []); + $decryptedLastChunk = $encryptionModule->decrypt($lastChunkContentEncrypted, $lastChunkNr . 'end'); + $decryptedLastChunk .= $encryptionModule->end($this->getFullPath($path), $lastChunkNr . 'end'); + + // calc the real file size with the size of the last chunk + $newUnencryptedSize += strlen($decryptedLastChunk); + + $this->updateUnencryptedSize($this->getFullPath($path), $newUnencryptedSize); + + // write to cache if applicable + $cache = $this->storage->getCache(); + $entry = $cache->get($path); + $cache->update($entry['fileid'], [ + 'unencrypted_size' => $newUnencryptedSize + ]); + + return $newUnencryptedSize; + } + + /** + * fread_block + * + * This function is a wrapper around the fread function. It is based on the + * stream_read_block function from lib/private/Files/Streams/Encryption.php + * It calls stream read until the requested $blockSize was received or no remaining data is present. + * This is required as stream_read only returns smaller chunks of data when the stream fetches from a + * remote storage over the internet and it does not care about the given $blockSize. + * + * @param resource $handle the stream to read from + * @param int $blockSize Length of requested data block in bytes + * @return string Data fetched from stream. + */ + private function fread_block($handle, int $blockSize): string { + $remaining = $blockSize; + $data = ''; + + do { + $chunk = fread($handle, $remaining); + $chunk_len = strlen($chunk); + $data .= $chunk; + $remaining -= $chunk_len; + } while (($remaining > 0) && ($chunk_len > 0)); + + return $data; + } + + public function moveFromStorage( + Storage\IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + $preserveMtime = true, + ): bool { + if ($sourceStorage === $this) { + return $this->rename($sourceInternalPath, $targetInternalPath); + } + + // TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed: + // - call $this->storage->moveFromStorage() instead of $this->copyBetweenStorage + // - copy the file cache update from $this->copyBetweenStorage to this method + // - copy the copyKeys() call from $this->copyBetweenStorage to this method + // - remove $this->copyBetweenStorage + + if (!$sourceStorage->isDeletable($sourceInternalPath)) { + return false; + } + + $result = $this->copyBetweenStorage($sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, true); + if ($result) { + $setPreserveCacheOnDelete = $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && !$this->instanceOfStorage(ObjectStoreStorage::class); + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + try { + if ($sourceStorage->is_dir($sourceInternalPath)) { + $result = $sourceStorage->rmdir($sourceInternalPath); + } else { + $result = $sourceStorage->unlink($sourceInternalPath); + } + } finally { + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + } + } + return $result; + } + + public function copyFromStorage( + Storage\IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + $preserveMtime = false, + $isRename = false, + ): bool { + // TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed: + // - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage + // - copy the file cache update from $this->copyBetweenStorage to this method + // - copy the copyKeys() call from $this->copyBetweenStorage to this method + // - remove $this->copyBetweenStorage + + return $this->copyBetweenStorage($sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename); + } + + /** + * Update the encrypted cache version in the database + */ + private function updateEncryptedVersion( + Storage\IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + bool $isRename, + bool $keepEncryptionVersion, + ): void { + $isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath); + $cacheInformation = [ + 'encrypted' => $isEncrypted, + ]; + if ($isEncrypted) { + $sourceCacheEntry = $sourceStorage->getCache()->get($sourceInternalPath); + $targetCacheEntry = $this->getCache()->get($targetInternalPath); + + // Rename of the cache already happened, so we do the cleanup on the target + if ($sourceCacheEntry === false && $targetCacheEntry !== false) { + $encryptedVersion = $targetCacheEntry['encryptedVersion']; + $isRename = false; + } else { + $encryptedVersion = $sourceCacheEntry['encryptedVersion']; + } + + // In case of a move operation from an unencrypted to an encrypted + // storage the old encrypted version would stay with "0" while the + // correct value would be "1". Thus we manually set the value to "1" + // for those cases. + // See also https://github.com/owncloud/core/issues/23078 + if ($encryptedVersion === 0 || !$keepEncryptionVersion) { + $encryptedVersion = 1; + } + + $cacheInformation['encryptedVersion'] = $encryptedVersion; + } + + // in case of a rename we need to manipulate the source cache because + // this information will be kept for the new target + if ($isRename) { + $sourceStorage->getCache()->put($sourceInternalPath, $cacheInformation); + } else { + $this->getCache()->put($targetInternalPath, $cacheInformation); + } + } + + /** + * copy file between two storages + * @throws \Exception + */ + private function copyBetweenStorage( + Storage\IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime, + bool $isRename, + ): bool { + // for versions we have nothing to do, because versions should always use the + // key from the original file. Just create a 1:1 copy and done + if ($this->isVersion($targetInternalPath) + || $this->isVersion($sourceInternalPath)) { + // remember that we try to create a version so that we can detect it during + // fopen($sourceInternalPath) and by-pass the encryption in order to + // create a 1:1 copy of the file + $this->arrayCache->set('encryption_copy_version_' . $sourceInternalPath, true); + $result = $this->storage->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + $this->arrayCache->remove('encryption_copy_version_' . $sourceInternalPath); + if ($result) { + $info = $this->getCache('', $sourceStorage)->get($sourceInternalPath); + // make sure that we update the unencrypted size for the version + if (isset($info['encrypted']) && $info['encrypted'] === true) { + $this->updateUnencryptedSize( + $this->getFullPath($targetInternalPath), + $info->getUnencryptedSize() + ); + } + $this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true); + } + return $result; + } + + // first copy the keys that we reuse the existing file key on the target location + // and don't create a new one which would break versions for example. + if ($sourceStorage->instanceOfStorage(Common::class) && $sourceStorage->getMountOption('mount_point')) { + $mountPoint = $sourceStorage->getMountOption('mount_point'); + $source = $mountPoint . '/' . $sourceInternalPath; + $target = $this->getFullPath($targetInternalPath); + $this->copyKeys($source, $target); + } else { + $this->logger->error('Could not find mount point, can\'t keep encryption keys'); + } + + if ($sourceStorage->is_dir($sourceInternalPath)) { + $dh = $sourceStorage->opendir($sourceInternalPath); + if (!$this->is_dir($targetInternalPath)) { + $result = $this->mkdir($targetInternalPath); + } else { + $result = true; + } + if (is_resource($dh)) { + while ($result && ($file = readdir($dh)) !== false) { + if (!Filesystem::isIgnoredDir($file)) { + $result = $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file, $preserveMtime, $isRename); + } + } + } + } else { + try { + $source = $sourceStorage->fopen($sourceInternalPath, 'r'); + $target = $this->fopen($targetInternalPath, 'w'); + if ($source === false || $target === false) { + $result = false; + } else { + [, $result] = Files::streamCopy($source, $target, true); + } + } finally { + if (isset($source) && $source !== false) { + fclose($source); + } + if (isset($target) && $target !== false) { + fclose($target); + } + } + if ($result) { + if ($preserveMtime) { + $this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath)); + } + $this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, false); + } else { + // delete partially written target file + $this->unlink($targetInternalPath); + // delete cache entry that was created by fopen + $this->getCache()->remove($targetInternalPath); + } + } + return (bool)$result; + } + + public function getLocalFile(string $path): string|false { + if ($this->encryptionManager->isEnabled()) { + $cachedFile = $this->getCachedFile($path); + if (is_string($cachedFile)) { + return $cachedFile; + } + } + return $this->storage->getLocalFile($path); + } + + public function isLocal(): bool { + if ($this->encryptionManager->isEnabled()) { + return false; + } + return $this->storage->isLocal(); + } + + public function stat(string $path): array|false { + $stat = $this->storage->stat($path); + if (!$stat) { + return false; + } + $fileSize = $this->filesize($path); + $stat['size'] = $fileSize; + $stat[7] = $fileSize; + $stat['hasHeader'] = $this->getHeaderSize($path) > 0; + return $stat; + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + $fh = $this->fopen($path, 'rb'); + if ($fh === false) { + return false; + } + $ctx = hash_init($type); + hash_update_stream($ctx, $fh); + fclose($fh); + return hash_final($ctx, $raw); + } + + /** + * return full path, including mount point + * + * @param string $path relative to mount point + * @return string full path including mount point + */ + protected function getFullPath(string $path): string { + return Filesystem::normalizePath($this->mountPoint . '/' . $path); + } + + /** + * read first block of encrypted file, typically this will contain the + * encryption header + */ + protected function readFirstBlock(string $path): string { + $firstBlock = ''; + if ($this->storage->is_file($path)) { + $handle = $this->storage->fopen($path, 'r'); + if ($handle === false) { + return ''; + } + $firstBlock = fread($handle, $this->util->getHeaderSize()); + fclose($handle); + } + return $firstBlock; + } + + /** + * return header size of given file + */ + protected function getHeaderSize(string $path): int { + $headerSize = 0; + $realFile = $this->util->stripPartialFileExtension($path); + if ($this->storage->is_file($realFile)) { + $path = $realFile; + } + $firstBlock = $this->readFirstBlock($path); + + if (str_starts_with($firstBlock, Util::HEADER_START)) { + $headerSize = $this->util->getHeaderSize(); + } + + return $headerSize; + } + + /** + * read header from file + */ + protected function getHeader(string $path): array { + $realFile = $this->util->stripPartialFileExtension($path); + $exists = $this->storage->is_file($realFile); + if ($exists) { + $path = $realFile; + } + + $result = []; + + $isEncrypted = $this->encryptedPaths->get($realFile); + if (is_null($isEncrypted)) { + $info = $this->getCache()->get($path); + $isEncrypted = isset($info['encrypted']) && $info['encrypted'] === true; + } + + if ($isEncrypted) { + $firstBlock = $this->readFirstBlock($path); + $result = $this->util->parseRawHeader($firstBlock); + + // if the header doesn't contain a encryption module we check if it is a + // legacy file. If true, we add the default encryption module + if (!isset($result[Util::HEADER_ENCRYPTION_MODULE_KEY]) && (!empty($result) || $exists)) { + $result[Util::HEADER_ENCRYPTION_MODULE_KEY] = 'OC_DEFAULT_MODULE'; + } + } + + return $result; + } + + /** + * read encryption module needed to read/write the file located at $path + * + * @throws ModuleDoesNotExistsException + * @throws \Exception + */ + protected function getEncryptionModule(string $path): ?\OCP\Encryption\IEncryptionModule { + $encryptionModule = null; + $header = $this->getHeader($path); + $encryptionModuleId = $this->util->getEncryptionModuleId($header); + if (!empty($encryptionModuleId)) { + try { + $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); + } catch (ModuleDoesNotExistsException $e) { + $this->logger->critical('Encryption module defined in "' . $path . '" not loaded!'); + throw $e; + } + } + + return $encryptionModule; + } + + public function updateUnencryptedSize(string $path, int|float $unencryptedSize): void { + $this->unencryptedSize[$path] = $unencryptedSize; + } + + /** + * copy keys to new location + * + * @param string $source path relative to data/ + * @param string $target path relative to data/ + */ + protected function copyKeys(string $source, string $target): bool { + if (!$this->util->isExcluded($source)) { + return $this->keyStorage->copyKeys($source, $target); + } + + return false; + } + + /** + * check if path points to a files version + */ + protected function isVersion(string $path): bool { + $normalized = Filesystem::normalizePath($path); + return substr($normalized, 0, strlen('/files_versions/')) === '/files_versions/'; + } + + /** + * check if the given storage should be encrypted or not + */ + protected function shouldEncrypt(string $path): bool { + $fullPath = $this->getFullPath($path); + $mountPointConfig = $this->mount->getOption('encrypt', true); + if ($mountPointConfig === false) { + return false; + } + + try { + $encryptionModule = $this->getEncryptionModule($fullPath); + } catch (ModuleDoesNotExistsException $e) { + return false; + } + + if ($encryptionModule === null) { + $encryptionModule = $this->encryptionManager->getEncryptionModule(); + } + + return $encryptionModule->shouldEncrypt($fullPath); + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + // always fall back to fopen + $target = $this->fopen($path, 'w'); + if ($target === false) { + throw new GenericFileException("Failed to open $path for writing"); + } + [$count, $result] = Files::streamCopy($stream, $target, true); + fclose($stream); + fclose($target); + + // object store, stores the size after write and doesn't update this during scan + // manually store the unencrypted size + if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class) && $this->shouldEncrypt($path)) { + $this->getCache()->put($path, ['unencrypted_size' => $count]); + } + + return $count; + } + + public function clearIsEncryptedCache(): void { + $this->encryptedPaths->clear(); + } + + /** + * Allow temporarily disabling the wrapper + */ + public function setEnabled(bool $enabled): void { + $this->enabled = $enabled; + } + + /** + * Check if the on-disk data for a file has a valid encrypted header + * + * @param string $path + * @return bool + */ + public function hasValidHeader(string $path): bool { + $firstBlock = $this->readFirstBlock($path); + $header = $this->util->parseRawHeader($firstBlock); + return (count($header) > 0); + } +} diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php new file mode 100644 index 00000000000..38b113cef88 --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Jail.php @@ -0,0 +1,267 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Cache\Wrapper\JailPropagator; +use OC\Files\Cache\Wrapper\JailWatcher; +use OC\Files\Filesystem; +use OCP\Files; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Cache\IWatcher; +use OCP\Files\Storage\IStorage; +use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Lock\ILockingProvider; + +/** + * Jail to a subdirectory of the wrapped storage + * + * This restricts access to a subfolder of the wrapped storage with the subfolder becoming the root folder new storage + */ +class Jail extends Wrapper { + /** + * @var string + */ + protected $rootPath; + + /** + * @param array $parameters ['storage' => $storage, 'root' => $root] + * + * $storage: The storage that will be wrapper + * $root: The folder in the wrapped storage that will become the root folder of the wrapped storage + */ + public function __construct(array $parameters) { + parent::__construct($parameters); + $this->rootPath = $parameters['root']; + } + + public function getUnjailedPath(string $path): string { + return trim(Filesystem::normalizePath($this->rootPath . '/' . $path), '/'); + } + + /** + * This is separate from Wrapper::getWrapperStorage so we can get the jailed storage consistently even if the jail is inside another wrapper + */ + public function getUnjailedStorage(): IStorage { + return $this->storage; + } + + + public function getJailedPath(string $path): ?string { + $root = rtrim($this->rootPath, '/') . '/'; + + if ($path !== $this->rootPath && !str_starts_with($path, $root)) { + return null; + } else { + $path = substr($path, strlen($this->rootPath)); + return trim($path, '/'); + } + } + + public function getId(): string { + return parent::getId(); + } + + public function mkdir(string $path): bool { + return $this->getWrapperStorage()->mkdir($this->getUnjailedPath($path)); + } + + public function rmdir(string $path): bool { + return $this->getWrapperStorage()->rmdir($this->getUnjailedPath($path)); + } + + public function opendir(string $path) { + return $this->getWrapperStorage()->opendir($this->getUnjailedPath($path)); + } + + public function is_dir(string $path): bool { + return $this->getWrapperStorage()->is_dir($this->getUnjailedPath($path)); + } + + public function is_file(string $path): bool { + return $this->getWrapperStorage()->is_file($this->getUnjailedPath($path)); + } + + public function stat(string $path): array|false { + return $this->getWrapperStorage()->stat($this->getUnjailedPath($path)); + } + + public function filetype(string $path): string|false { + return $this->getWrapperStorage()->filetype($this->getUnjailedPath($path)); + } + + public function filesize(string $path): int|float|false { + return $this->getWrapperStorage()->filesize($this->getUnjailedPath($path)); + } + + public function isCreatable(string $path): bool { + return $this->getWrapperStorage()->isCreatable($this->getUnjailedPath($path)); + } + + public function isReadable(string $path): bool { + return $this->getWrapperStorage()->isReadable($this->getUnjailedPath($path)); + } + + public function isUpdatable(string $path): bool { + return $this->getWrapperStorage()->isUpdatable($this->getUnjailedPath($path)); + } + + public function isDeletable(string $path): bool { + return $this->getWrapperStorage()->isDeletable($this->getUnjailedPath($path)); + } + + public function isSharable(string $path): bool { + return $this->getWrapperStorage()->isSharable($this->getUnjailedPath($path)); + } + + public function getPermissions(string $path): int { + return $this->getWrapperStorage()->getPermissions($this->getUnjailedPath($path)); + } + + public function file_exists(string $path): bool { + return $this->getWrapperStorage()->file_exists($this->getUnjailedPath($path)); + } + + public function filemtime(string $path): int|false { + return $this->getWrapperStorage()->filemtime($this->getUnjailedPath($path)); + } + + public function file_get_contents(string $path): string|false { + return $this->getWrapperStorage()->file_get_contents($this->getUnjailedPath($path)); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + return $this->getWrapperStorage()->file_put_contents($this->getUnjailedPath($path), $data); + } + + public function unlink(string $path): bool { + return $this->getWrapperStorage()->unlink($this->getUnjailedPath($path)); + } + + public function rename(string $source, string $target): bool { + return $this->getWrapperStorage()->rename($this->getUnjailedPath($source), $this->getUnjailedPath($target)); + } + + public function copy(string $source, string $target): bool { + return $this->getWrapperStorage()->copy($this->getUnjailedPath($source), $this->getUnjailedPath($target)); + } + + public function fopen(string $path, string $mode) { + return $this->getWrapperStorage()->fopen($this->getUnjailedPath($path), $mode); + } + + public function getMimeType(string $path): string|false { + return $this->getWrapperStorage()->getMimeType($this->getUnjailedPath($path)); + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + return $this->getWrapperStorage()->hash($type, $this->getUnjailedPath($path), $raw); + } + + public function free_space(string $path): int|float|false { + return $this->getWrapperStorage()->free_space($this->getUnjailedPath($path)); + } + + public function touch(string $path, ?int $mtime = null): bool { + return $this->getWrapperStorage()->touch($this->getUnjailedPath($path), $mtime); + } + + public function getLocalFile(string $path): string|false { + return $this->getWrapperStorage()->getLocalFile($this->getUnjailedPath($path)); + } + + public function hasUpdated(string $path, int $time): bool { + return $this->getWrapperStorage()->hasUpdated($this->getUnjailedPath($path), $time); + } + + public function getCache(string $path = '', ?IStorage $storage = null): ICache { + $sourceCache = $this->getWrapperStorage()->getCache($this->getUnjailedPath($path)); + return new CacheJail($sourceCache, $this->rootPath); + } + + public function getOwner(string $path): string|false { + return $this->getWrapperStorage()->getOwner($this->getUnjailedPath($path)); + } + + public function getWatcher(string $path = '', ?IStorage $storage = null): IWatcher { + $sourceWatcher = $this->getWrapperStorage()->getWatcher($this->getUnjailedPath($path), $this->getWrapperStorage()); + return new JailWatcher($sourceWatcher, $this->rootPath); + } + + public function getETag(string $path): string|false { + return $this->getWrapperStorage()->getETag($this->getUnjailedPath($path)); + } + + public function getMetaData(string $path): ?array { + return $this->getWrapperStorage()->getMetaData($this->getUnjailedPath($path)); + } + + public function acquireLock(string $path, int $type, ILockingProvider $provider): void { + $this->getWrapperStorage()->acquireLock($this->getUnjailedPath($path), $type, $provider); + } + + public function releaseLock(string $path, int $type, ILockingProvider $provider): void { + $this->getWrapperStorage()->releaseLock($this->getUnjailedPath($path), $type, $provider); + } + + public function changeLock(string $path, int $type, ILockingProvider $provider): void { + $this->getWrapperStorage()->changeLock($this->getUnjailedPath($path), $type, $provider); + } + + /** + * Resolve the path for the source of the share + */ + public function resolvePath(string $path): array { + return [$this->getWrapperStorage(), $this->getUnjailedPath($path)]; + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + return $this->copy($sourceInternalPath, $targetInternalPath); + } + return $this->getWrapperStorage()->copyFromStorage($sourceStorage, $sourceInternalPath, $this->getUnjailedPath($targetInternalPath)); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + return $this->rename($sourceInternalPath, $targetInternalPath); + } + return $this->getWrapperStorage()->moveFromStorage($sourceStorage, $sourceInternalPath, $this->getUnjailedPath($targetInternalPath)); + } + + public function getPropagator(?IStorage $storage = null): IPropagator { + if (isset($this->propagator)) { + return $this->propagator; + } + + if (!$storage) { + $storage = $this; + } + $this->propagator = new JailPropagator($storage, \OC::$server->getDatabaseConnection()); + return $this->propagator; + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + $storage = $this->getWrapperStorage(); + if ($storage->instanceOfStorage(IWriteStreamStorage::class)) { + /** @var IWriteStreamStorage $storage */ + return $storage->writeStream($this->getUnjailedPath($path), $stream, $size); + } else { + $target = $this->fopen($path, 'w'); + $count = Files::streamCopy($stream, $target); + fclose($stream); + fclose($target); + return $count; + } + } + + public function getDirectoryContent(string $directory): \Traversable { + return $this->getWrapperStorage()->getDirectoryContent($this->getUnjailedPath($directory)); + } +} diff --git a/lib/private/Files/Storage/Wrapper/KnownMtime.php b/lib/private/Files/Storage/Wrapper/KnownMtime.php new file mode 100644 index 00000000000..657c6c9250c --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/KnownMtime.php @@ -0,0 +1,146 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Storage\Wrapper; + +use OCP\Cache\CappedMemoryCache; +use OCP\Files\Storage\IStorage; +use Psr\Clock\ClockInterface; + +/** + * Wrapper that overwrites the mtime return by stat/getMetaData if the returned value + * is lower than when we last modified the file. + * + * This is useful because some storage servers can return an outdated mtime right after writes + */ +class KnownMtime extends Wrapper { + private CappedMemoryCache $knowMtimes; + private ClockInterface $clock; + + public function __construct(array $parameters) { + parent::__construct($parameters); + $this->knowMtimes = new CappedMemoryCache(); + $this->clock = $parameters['clock']; + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + $result = parent::file_put_contents($path, $data); + if ($result) { + $now = $this->clock->now()->getTimestamp(); + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function stat(string $path): array|false { + $stat = parent::stat($path); + if ($stat) { + $this->applyKnownMtime($path, $stat); + } + return $stat; + } + + public function getMetaData(string $path): ?array { + $stat = parent::getMetaData($path); + if ($stat) { + $this->applyKnownMtime($path, $stat); + } + return $stat; + } + + private function applyKnownMtime(string $path, array &$stat): void { + if (isset($stat['mtime'])) { + $knownMtime = $this->knowMtimes->get($path) ?? 0; + $stat['mtime'] = max($stat['mtime'], $knownMtime); + } + } + + public function filemtime(string $path): int|false { + $knownMtime = $this->knowMtimes->get($path) ?? 0; + return max(parent::filemtime($path), $knownMtime); + } + + public function mkdir(string $path): bool { + $result = parent::mkdir($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function rmdir(string $path): bool { + $result = parent::rmdir($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function unlink(string $path): bool { + $result = parent::unlink($path); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function rename(string $source, string $target): bool { + $result = parent::rename($source, $target); + if ($result) { + $this->knowMtimes->set($target, $this->clock->now()->getTimestamp()); + $this->knowMtimes->set($source, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function copy(string $source, string $target): bool { + $result = parent::copy($source, $target); + if ($result) { + $this->knowMtimes->set($target, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function fopen(string $path, string $mode) { + $result = parent::fopen($path, $mode); + if ($result && $mode === 'w') { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function touch(string $path, ?int $mtime = null): bool { + $result = parent::touch($path, $mtime); + if ($result) { + $this->knowMtimes->set($path, $mtime ?? $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + $result = parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + if ($result) { + $this->knowMtimes->set($targetInternalPath, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + $result = parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + if ($result) { + $this->knowMtimes->set($targetInternalPath, $this->clock->now()->getTimestamp()); + } + return $result; + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + $result = parent::writeStream($path, $stream, $size); + if ($result) { + $this->knowMtimes->set($path, $this->clock->now()->getTimestamp()); + } + return $result; + } +} diff --git a/lib/private/Files/Storage/Wrapper/PermissionsMask.php b/lib/private/Files/Storage/Wrapper/PermissionsMask.php new file mode 100644 index 00000000000..684040146ba --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/PermissionsMask.php @@ -0,0 +1,138 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Files\Cache\Wrapper\CachePermissionsMask; +use OCP\Constants; +use OCP\Files\Storage\IStorage; + +/** + * Mask the permissions of a storage + * + * This can be used to restrict update, create, delete and/or share permissions of a storage + * + * Note that the read permissions can't be masked + */ +class PermissionsMask extends Wrapper { + /** + * @var int the permissions bits we want to keep + */ + private $mask; + + /** + * @param array $parameters ['storage' => $storage, 'mask' => $mask] + * + * $storage: The storage the permissions mask should be applied on + * $mask: The permission bits that should be kept, a combination of the \OCP\Constant::PERMISSION_ constants + */ + public function __construct(array $parameters) { + parent::__construct($parameters); + $this->mask = $parameters['mask']; + } + + private function checkMask(int $permissions): bool { + return ($this->mask & $permissions) === $permissions; + } + + public function isUpdatable(string $path): bool { + return $this->checkMask(Constants::PERMISSION_UPDATE) and parent::isUpdatable($path); + } + + public function isCreatable(string $path): bool { + return $this->checkMask(Constants::PERMISSION_CREATE) and parent::isCreatable($path); + } + + public function isDeletable(string $path): bool { + return $this->checkMask(Constants::PERMISSION_DELETE) and parent::isDeletable($path); + } + + public function isSharable(string $path): bool { + return $this->checkMask(Constants::PERMISSION_SHARE) and parent::isSharable($path); + } + + public function getPermissions(string $path): int { + return $this->storage->getPermissions($path) & $this->mask; + } + + public function rename(string $source, string $target): bool { + //This is a rename of the transfer file to the original file + if (dirname($source) === dirname($target) && strpos($source, '.ocTransferId') > 0) { + return $this->checkMask(Constants::PERMISSION_CREATE) and parent::rename($source, $target); + } + return $this->checkMask(Constants::PERMISSION_UPDATE) and parent::rename($source, $target); + } + + public function copy(string $source, string $target): bool { + return $this->checkMask(Constants::PERMISSION_CREATE) and parent::copy($source, $target); + } + + public function touch(string $path, ?int $mtime = null): bool { + $permissions = $this->file_exists($path) ? Constants::PERMISSION_UPDATE : Constants::PERMISSION_CREATE; + return $this->checkMask($permissions) and parent::touch($path, $mtime); + } + + public function mkdir(string $path): bool { + return $this->checkMask(Constants::PERMISSION_CREATE) and parent::mkdir($path); + } + + public function rmdir(string $path): bool { + return $this->checkMask(Constants::PERMISSION_DELETE) and parent::rmdir($path); + } + + public function unlink(string $path): bool { + return $this->checkMask(Constants::PERMISSION_DELETE) and parent::unlink($path); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + $permissions = $this->file_exists($path) ? Constants::PERMISSION_UPDATE : Constants::PERMISSION_CREATE; + return $this->checkMask($permissions) ? parent::file_put_contents($path, $data) : false; + } + + public function fopen(string $path, string $mode) { + if ($mode === 'r' or $mode === 'rb') { + return parent::fopen($path, $mode); + } else { + $permissions = $this->file_exists($path) ? Constants::PERMISSION_UPDATE : Constants::PERMISSION_CREATE; + return $this->checkMask($permissions) ? parent::fopen($path, $mode) : false; + } + } + + public function getCache(string $path = '', ?IStorage $storage = null): \OCP\Files\Cache\ICache { + if (!$storage) { + $storage = $this; + } + $sourceCache = parent::getCache($path, $storage); + return new CachePermissionsMask($sourceCache, $this->mask); + } + + public function getMetaData(string $path): ?array { + $data = parent::getMetaData($path); + + if ($data && isset($data['permissions'])) { + $data['scan_permissions'] = $data['scan_permissions'] ?? $data['permissions']; + $data['permissions'] &= $this->mask; + } + return $data; + } + + public function getScanner(string $path = '', ?IStorage $storage = null): \OCP\Files\Cache\IScanner { + if (!$storage) { + $storage = $this->storage; + } + return parent::getScanner($path, $storage); + } + + public function getDirectoryContent(string $directory): \Traversable { + foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) { + $data['scan_permissions'] = $data['scan_permissions'] ?? $data['permissions']; + $data['permissions'] &= $this->mask; + + yield $data; + } + } +} diff --git a/lib/private/Files/Storage/Wrapper/Quota.php b/lib/private/Files/Storage/Wrapper/Quota.php new file mode 100644 index 00000000000..35a265f8c8e --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Quota.php @@ -0,0 +1,208 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Files\Filesystem; +use OC\SystemConfig; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\FileInfo; +use OCP\Files\Storage\IStorage; + +class Quota extends Wrapper { + /** @var callable|null */ + protected $quotaCallback; + /** @var int|float|null int on 64bits, float on 32bits for bigint */ + protected int|float|null $quota; + protected string $sizeRoot; + private SystemConfig $config; + private bool $quotaIncludeExternalStorage; + private bool $enabled = true; + + /** + * @param array $parameters + */ + public function __construct(array $parameters) { + parent::__construct($parameters); + $this->quota = $parameters['quota'] ?? null; + $this->quotaCallback = $parameters['quotaCallback'] ?? null; + $this->sizeRoot = $parameters['root'] ?? ''; + $this->quotaIncludeExternalStorage = $parameters['include_external_storage'] ?? false; + } + + public function getQuota(): int|float { + if ($this->quota === null) { + $quotaCallback = $this->quotaCallback; + if ($quotaCallback === null) { + throw new \Exception('No quota or quota callback provider'); + } + $this->quota = $quotaCallback(); + } + + return $this->quota; + } + + private function hasQuota(): bool { + if (!$this->enabled) { + return false; + } + return $this->getQuota() !== FileInfo::SPACE_UNLIMITED; + } + + protected function getSize(string $path, ?IStorage $storage = null): int|float { + if ($this->quotaIncludeExternalStorage) { + $rootInfo = Filesystem::getFileInfo('', 'ext'); + if ($rootInfo) { + return $rootInfo->getSize(true); + } + return FileInfo::SPACE_NOT_COMPUTED; + } else { + $cache = is_null($storage) ? $this->getCache() : $storage->getCache(); + $data = $cache->get($path); + if ($data instanceof ICacheEntry && isset($data['size'])) { + return $data['size']; + } else { + return FileInfo::SPACE_NOT_COMPUTED; + } + } + } + + public function free_space(string $path): int|float|false { + if (!$this->hasQuota()) { + return $this->storage->free_space($path); + } + if ($this->getQuota() < 0 || str_starts_with($path, 'cache') || str_starts_with($path, 'uploads')) { + return $this->storage->free_space($path); + } else { + $used = $this->getSize($this->sizeRoot); + if ($used < 0) { + return FileInfo::SPACE_NOT_COMPUTED; + } else { + $free = $this->storage->free_space($path); + $quotaFree = max($this->getQuota() - $used, 0); + // if free space is known + $free = $free >= 0 ? min($free, $quotaFree) : $quotaFree; + return $free; + } + } + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + if (!$this->hasQuota()) { + return $this->storage->file_put_contents($path, $data); + } + $free = $this->free_space($path); + if ($free < 0 || strlen($data) < $free) { + return $this->storage->file_put_contents($path, $data); + } else { + return false; + } + } + + public function copy(string $source, string $target): bool { + if (!$this->hasQuota()) { + return $this->storage->copy($source, $target); + } + $free = $this->free_space($target); + if ($free < 0 || $this->getSize($source) < $free) { + return $this->storage->copy($source, $target); + } else { + return false; + } + } + + public function fopen(string $path, string $mode) { + if (!$this->hasQuota()) { + return $this->storage->fopen($path, $mode); + } + $source = $this->storage->fopen($path, $mode); + + // don't apply quota for part files + if (!$this->isPartFile($path)) { + $free = $this->free_space($path); + if ($source && (is_int($free) || is_float($free)) && $free >= 0 && $mode !== 'r' && $mode !== 'rb') { + // only apply quota for files, not metadata, trash or others + if ($this->shouldApplyQuota($path)) { + return \OC\Files\Stream\Quota::wrap($source, $free); + } + } + } + + return $source; + } + + /** + * Checks whether the given path is a part file + * + * @param string $path Path that may identify a .part file + * @note this is needed for reusing keys + */ + private function isPartFile(string $path): bool { + $extension = pathinfo($path, PATHINFO_EXTENSION); + + return ($extension === 'part'); + } + + /** + * Only apply quota for files, not metadata, trash or others + */ + protected function shouldApplyQuota(string $path): bool { + return str_starts_with(ltrim($path, '/'), 'files/'); + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if (!$this->hasQuota()) { + return $this->storage->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + $free = $this->free_space($targetInternalPath); + if ($free < 0 || $this->getSize($sourceInternalPath, $sourceStorage) < $free) { + return $this->storage->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } else { + return false; + } + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if (!$this->hasQuota()) { + return $this->storage->moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + $free = $this->free_space($targetInternalPath); + if ($free < 0 || $this->getSize($sourceInternalPath, $sourceStorage) < $free) { + return $this->storage->moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } else { + return false; + } + } + + public function mkdir(string $path): bool { + if (!$this->hasQuota()) { + return $this->storage->mkdir($path); + } + $free = $this->free_space($path); + if ($this->shouldApplyQuota($path) && $free == 0) { + return false; + } + + return parent::mkdir($path); + } + + public function touch(string $path, ?int $mtime = null): bool { + if (!$this->hasQuota()) { + return $this->storage->touch($path, $mtime); + } + $free = $this->free_space($path); + if ($free == 0) { + return false; + } + + return parent::touch($path, $mtime); + } + + public function enableQuota(bool $enabled): void { + $this->enabled = $enabled; + } +} diff --git a/lib/private/Files/Storage/Wrapper/Wrapper.php b/lib/private/Files/Storage/Wrapper/Wrapper.php new file mode 100644 index 00000000000..7af11dd5ef7 --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Wrapper.php @@ -0,0 +1,351 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Storage\Wrapper; + +use OC\Files\Storage\FailedStorage; +use OC\Files\Storage\Storage; +use OCP\Files; +use OCP\Files\Cache\ICache; +use OCP\Files\Cache\IPropagator; +use OCP\Files\Cache\IScanner; +use OCP\Files\Cache\IUpdater; +use OCP\Files\Cache\IWatcher; +use OCP\Files\Storage\ILockingStorage; +use OCP\Files\Storage\IStorage; +use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Lock\ILockingProvider; +use OCP\Server; +use Psr\Log\LoggerInterface; + +class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStreamStorage { + /** + * @var \OC\Files\Storage\Storage $storage + */ + protected $storage; + + public $cache; + public $scanner; + public $watcher; + public $propagator; + public $updater; + + /** + * @param array $parameters + */ + public function __construct(array $parameters) { + $this->storage = $parameters['storage']; + } + + public function getWrapperStorage(): Storage { + if (!$this->storage) { + $message = 'storage wrapper ' . get_class($this) . " doesn't have a wrapped storage set"; + $logger = Server::get(LoggerInterface::class); + $logger->error($message); + $this->storage = new FailedStorage(['exception' => new \Exception($message)]); + } + return $this->storage; + } + + public function getId(): string { + return $this->getWrapperStorage()->getId(); + } + + public function mkdir(string $path): bool { + return $this->getWrapperStorage()->mkdir($path); + } + + public function rmdir(string $path): bool { + return $this->getWrapperStorage()->rmdir($path); + } + + public function opendir(string $path) { + return $this->getWrapperStorage()->opendir($path); + } + + public function is_dir(string $path): bool { + return $this->getWrapperStorage()->is_dir($path); + } + + public function is_file(string $path): bool { + return $this->getWrapperStorage()->is_file($path); + } + + public function stat(string $path): array|false { + return $this->getWrapperStorage()->stat($path); + } + + public function filetype(string $path): string|false { + return $this->getWrapperStorage()->filetype($path); + } + + public function filesize(string $path): int|float|false { + return $this->getWrapperStorage()->filesize($path); + } + + public function isCreatable(string $path): bool { + return $this->getWrapperStorage()->isCreatable($path); + } + + public function isReadable(string $path): bool { + return $this->getWrapperStorage()->isReadable($path); + } + + public function isUpdatable(string $path): bool { + return $this->getWrapperStorage()->isUpdatable($path); + } + + public function isDeletable(string $path): bool { + return $this->getWrapperStorage()->isDeletable($path); + } + + public function isSharable(string $path): bool { + return $this->getWrapperStorage()->isSharable($path); + } + + public function getPermissions(string $path): int { + return $this->getWrapperStorage()->getPermissions($path); + } + + public function file_exists(string $path): bool { + return $this->getWrapperStorage()->file_exists($path); + } + + public function filemtime(string $path): int|false { + return $this->getWrapperStorage()->filemtime($path); + } + + public function file_get_contents(string $path): string|false { + return $this->getWrapperStorage()->file_get_contents($path); + } + + public function file_put_contents(string $path, mixed $data): int|float|false { + return $this->getWrapperStorage()->file_put_contents($path, $data); + } + + public function unlink(string $path): bool { + return $this->getWrapperStorage()->unlink($path); + } + + public function rename(string $source, string $target): bool { + return $this->getWrapperStorage()->rename($source, $target); + } + + public function copy(string $source, string $target): bool { + return $this->getWrapperStorage()->copy($source, $target); + } + + public function fopen(string $path, string $mode) { + return $this->getWrapperStorage()->fopen($path, $mode); + } + + public function getMimeType(string $path): string|false { + return $this->getWrapperStorage()->getMimeType($path); + } + + public function hash(string $type, string $path, bool $raw = false): string|false { + return $this->getWrapperStorage()->hash($type, $path, $raw); + } + + public function free_space(string $path): int|float|false { + return $this->getWrapperStorage()->free_space($path); + } + + public function touch(string $path, ?int $mtime = null): bool { + return $this->getWrapperStorage()->touch($path, $mtime); + } + + public function getLocalFile(string $path): string|false { + return $this->getWrapperStorage()->getLocalFile($path); + } + + public function hasUpdated(string $path, int $time): bool { + return $this->getWrapperStorage()->hasUpdated($path, $time); + } + + public function getCache(string $path = '', ?IStorage $storage = null): ICache { + if (!$storage) { + $storage = $this; + } + return $this->getWrapperStorage()->getCache($path, $storage); + } + + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { + if (!$storage) { + $storage = $this; + } + return $this->getWrapperStorage()->getScanner($path, $storage); + } + + public function getOwner(string $path): string|false { + return $this->getWrapperStorage()->getOwner($path); + } + + public function getWatcher(string $path = '', ?IStorage $storage = null): IWatcher { + if (!$storage) { + $storage = $this; + } + return $this->getWrapperStorage()->getWatcher($path, $storage); + } + + public function getPropagator(?IStorage $storage = null): IPropagator { + if (!$storage) { + $storage = $this; + } + return $this->getWrapperStorage()->getPropagator($storage); + } + + public function getUpdater(?IStorage $storage = null): IUpdater { + if (!$storage) { + $storage = $this; + } + return $this->getWrapperStorage()->getUpdater($storage); + } + + public function getStorageCache(): \OC\Files\Cache\Storage { + return $this->getWrapperStorage()->getStorageCache(); + } + + public function getETag(string $path): string|false { + return $this->getWrapperStorage()->getETag($path); + } + + public function test(): bool { + return $this->getWrapperStorage()->test(); + } + + public function isLocal(): bool { + return $this->getWrapperStorage()->isLocal(); + } + + public function instanceOfStorage(string $class): bool { + if (ltrim($class, '\\') === 'OC\Files\Storage\Shared') { + // FIXME Temporary fix to keep existing checks working + $class = '\OCA\Files_Sharing\SharedStorage'; + } + return is_a($this, $class) or $this->getWrapperStorage()->instanceOfStorage($class); + } + + /** + * @psalm-template T of IStorage + * @psalm-param class-string<T> $class + * @psalm-return T|null + */ + public function getInstanceOfStorage(string $class): ?IStorage { + $storage = $this; + while ($storage instanceof Wrapper) { + if ($storage instanceof $class) { + break; + } + $storage = $storage->getWrapperStorage(); + } + if (!($storage instanceof $class)) { + return null; + } + return $storage; + } + + /** + * Pass any methods custom to specific storage implementations to the wrapped storage + * + * @return mixed + */ + public function __call(string $method, array $args) { + return call_user_func_array([$this->getWrapperStorage(), $method], $args); + } + + public function getDirectDownload(string $path): array|false { + return $this->getWrapperStorage()->getDirectDownload($path); + } + + public function getAvailability(): array { + return $this->getWrapperStorage()->getAvailability(); + } + + public function setAvailability(bool $isAvailable): void { + $this->getWrapperStorage()->setAvailability($isAvailable); + } + + public function verifyPath(string $path, string $fileName): void { + $this->getWrapperStorage()->verifyPath($path, $fileName); + } + + public function copyFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + return $this->copy($sourceInternalPath, $targetInternalPath); + } + + return $this->getWrapperStorage()->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath): bool { + if ($sourceStorage === $this) { + return $this->rename($sourceInternalPath, $targetInternalPath); + } + + return $this->getWrapperStorage()->moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function getMetaData(string $path): ?array { + return $this->getWrapperStorage()->getMetaData($path); + } + + public function acquireLock(string $path, int $type, ILockingProvider $provider): void { + if ($this->getWrapperStorage()->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->getWrapperStorage()->acquireLock($path, $type, $provider); + } + } + + public function releaseLock(string $path, int $type, ILockingProvider $provider): void { + if ($this->getWrapperStorage()->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->getWrapperStorage()->releaseLock($path, $type, $provider); + } + } + + public function changeLock(string $path, int $type, ILockingProvider $provider): void { + if ($this->getWrapperStorage()->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $this->getWrapperStorage()->changeLock($path, $type, $provider); + } + } + + public function needsPartFile(): bool { + return $this->getWrapperStorage()->needsPartFile(); + } + + public function writeStream(string $path, $stream, ?int $size = null): int { + $storage = $this->getWrapperStorage(); + if ($storage->instanceOfStorage(IWriteStreamStorage::class)) { + /** @var IWriteStreamStorage $storage */ + return $storage->writeStream($path, $stream, $size); + } else { + $target = $this->fopen($path, 'w'); + $count = Files::streamCopy($stream, $target); + fclose($stream); + fclose($target); + return $count; + } + } + + public function getDirectoryContent(string $directory): \Traversable { + return $this->getWrapperStorage()->getDirectoryContent($directory); + } + + public function isWrapperOf(IStorage $storage): bool { + $wrapped = $this->getWrapperStorage(); + if ($wrapped === $storage) { + return true; + } + if ($wrapped instanceof Wrapper) { + return $wrapped->isWrapperOf($storage); + } + return false; + } + + public function setOwner(?string $user): void { + $this->getWrapperStorage()->setOwner($user); + } +} diff --git a/lib/private/Files/Stream/Encryption.php b/lib/private/Files/Stream/Encryption.php new file mode 100644 index 00000000000..ef147ec421f --- /dev/null +++ b/lib/private/Files/Stream/Encryption.php @@ -0,0 +1,502 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Stream; + +use Icewind\Streams\Wrapper; +use OC\Encryption\Exceptions\EncryptionHeaderKeyExistsException; +use OC\Encryption\File; +use OC\Encryption\Util; +use OC\Files\Storage\Storage; +use OCP\Encryption\IEncryptionModule; +use function is_array; +use function stream_context_create; + +class Encryption extends Wrapper { + protected Util $util; + protected File $file; + protected IEncryptionModule $encryptionModule; + protected Storage $storage; + protected \OC\Files\Storage\Wrapper\Encryption $encryptionStorage; + protected string $internalPath; + protected string $cache; + protected ?int $size = null; + protected int $position; + protected ?int $unencryptedSize = null; + protected int $headerSize; + protected int $unencryptedBlockSize; + protected array $header; + protected string $fullPath; + protected bool $signed; + /** + * header data returned by the encryption module, will be written to the file + * in case of a write operation + */ + protected array $newHeader; + /** + * user who perform the read/write operation null for public access + */ + protected ?string $uid; + protected bool $readOnly; + protected bool $writeFlag; + protected array $expectedContextProperties; + protected bool $fileUpdated; + + public function __construct() { + $this->expectedContextProperties = [ + 'source', + 'storage', + 'internalPath', + 'fullPath', + 'encryptionModule', + 'header', + 'uid', + 'file', + 'util', + 'size', + 'unencryptedSize', + 'encryptionStorage', + 'headerSize', + 'signed' + ]; + } + + + /** + * Wraps a stream with the provided callbacks + * + * @param resource $source + * @param string $internalPath relative to mount point + * @param string $fullPath relative to data/ + * @param array $header + * @param string $uid + * @param IEncryptionModule $encryptionModule + * @param Storage $storage + * @param \OC\Files\Storage\Wrapper\Encryption $encStorage + * @param Util $util + * @param File $file + * @param string $mode + * @param int|float $size + * @param int|float $unencryptedSize + * @param int $headerSize + * @param bool $signed + * @param string $wrapper stream wrapper class + * @return resource + * + * @throws \BadMethodCallException + */ + public static function wrap( + $source, + $internalPath, + $fullPath, + array $header, + $uid, + IEncryptionModule $encryptionModule, + Storage $storage, + \OC\Files\Storage\Wrapper\Encryption $encStorage, + Util $util, + File $file, + $mode, + $size, + $unencryptedSize, + $headerSize, + $signed, + $wrapper = Encryption::class, + ) { + $context = stream_context_create([ + 'ocencryption' => [ + 'source' => $source, + 'storage' => $storage, + 'internalPath' => $internalPath, + 'fullPath' => $fullPath, + 'encryptionModule' => $encryptionModule, + 'header' => $header, + 'uid' => $uid, + 'util' => $util, + 'file' => $file, + 'size' => $size, + 'unencryptedSize' => $unencryptedSize, + 'encryptionStorage' => $encStorage, + 'headerSize' => $headerSize, + 'signed' => $signed + ] + ]); + + return self::wrapSource($source, $context, 'ocencryption', $wrapper, $mode); + } + + /** + * add stream wrapper + * + * @param resource|int $source + * @param resource|array $context + * @param string|null $protocol + * @param string|null $class + * @param string $mode + * @return resource + * @throws \BadMethodCallException + */ + protected static function wrapSource($source, $context = [], $protocol = null, $class = null, $mode = 'r+') { + try { + if ($protocol === null) { + $protocol = self::getProtocol($class); + } + + stream_wrapper_register($protocol, $class); + $context = self::buildContext($protocol, $context, $source); + if (self::isDirectoryHandle($source)) { + $wrapped = opendir($protocol . '://', $context); + } else { + $wrapped = fopen($protocol . '://', $mode, false, $context); + } + } catch (\Exception $e) { + stream_wrapper_unregister($protocol); + throw $e; + } + stream_wrapper_unregister($protocol); + return $wrapped; + } + + /** + * @todo this is a copy of \Icewind\Streams\WrapperHandler::buildContext -> combine to one shared method? + */ + private static function buildContext($protocol, $context, $source) { + if (is_array($context)) { + $context['source'] = $source; + return stream_context_create([$protocol => $context]); + } + + return $context; + } + + /** + * Load the source from the stream context and return the context options + * + * @param string|null $name + * @return array + * @throws \BadMethodCallException + */ + protected function loadContext($name = null) { + $context = parent::loadContext($name); + + foreach ($this->expectedContextProperties as $property) { + if (array_key_exists($property, $context)) { + $this->{$property} = $context[$property]; + } else { + throw new \BadMethodCallException('Invalid context, "' . $property . '" options not set'); + } + } + return $context; + } + + public function stream_open($path, $mode, $options, &$opened_path) { + $this->loadContext('ocencryption'); + + $this->position = 0; + $this->cache = ''; + $this->writeFlag = false; + $this->fileUpdated = false; + + if ( + $mode === 'w' + || $mode === 'w+' + || $mode === 'wb' + || $mode === 'wb+' + || $mode === 'r+' + || $mode === 'rb+' + ) { + $this->readOnly = false; + } else { + $this->readOnly = true; + } + + $sharePath = $this->fullPath; + if (!$this->storage->file_exists($this->internalPath)) { + $sharePath = dirname($sharePath); + } + + $accessList = []; + if ($this->encryptionModule->needDetailedAccessList()) { + $accessList = $this->file->getAccessList($sharePath); + } + $this->newHeader = $this->encryptionModule->begin($this->fullPath, $this->uid, $mode, $this->header, $accessList); + $this->unencryptedBlockSize = $this->encryptionModule->getUnencryptedBlockSize($this->signed); + + if ( + $mode === 'w' + || $mode === 'w+' + || $mode === 'wb' + || $mode === 'wb+' + ) { + // We're writing a new file so start write counter with 0 bytes + $this->unencryptedSize = 0; + $this->writeHeader(); + $this->headerSize = $this->util->getHeaderSize(); + $this->size = $this->headerSize; + } else { + $this->skipHeader(); + } + + return true; + } + + public function stream_eof() { + return $this->position >= $this->unencryptedSize; + } + + public function stream_read($count) { + $result = ''; + + $count = min($count, $this->unencryptedSize - $this->position); + while ($count > 0) { + $remainingLength = $count; + // update the cache of the current block + $this->readCache(); + // determine the relative position in the current block + $blockPosition = ($this->position % $this->unencryptedBlockSize); + // if entire read inside current block then only position needs to be updated + if ($remainingLength < ($this->unencryptedBlockSize - $blockPosition)) { + $result .= substr($this->cache, $blockPosition, $remainingLength); + $this->position += $remainingLength; + $count = 0; + // otherwise remainder of current block is fetched, the block is flushed and the position updated + } else { + $result .= substr($this->cache, $blockPosition); + $this->flush(); + $this->position += ($this->unencryptedBlockSize - $blockPosition); + $count -= ($this->unencryptedBlockSize - $blockPosition); + } + } + return $result; + } + + /** + * stream_read_block + * + * This function is a wrapper for function stream_read. + * It calls stream read until the requested $blockSize was received or no remaining data is present. + * This is required as stream_read only returns smaller chunks of data when the stream fetches from a + * remote storage over the internet and it does not care about the given $blockSize. + * + * @param int $blockSize Length of requested data block in bytes + * @return string Data fetched from stream. + */ + private function stream_read_block(int $blockSize): string { + $remaining = $blockSize; + $data = ''; + + do { + $chunk = parent::stream_read($remaining); + $chunk_len = strlen($chunk); + $data .= $chunk; + $remaining -= $chunk_len; + } while (($remaining > 0) && ($chunk_len > 0)); + + return $data; + } + + public function stream_write($data) { + $length = 0; + // loop over $data to fit it in 6126 sized unencrypted blocks + while (isset($data[0])) { + $remainingLength = strlen($data); + + // set the cache to the current 6126 block + $this->readCache(); + + // for seekable streams the pointer is moved back to the beginning of the encrypted block + // flush will start writing there when the position moves to another block + $positionInFile = (int)floor($this->position / $this->unencryptedBlockSize) + * $this->util->getBlockSize() + $this->headerSize; + $resultFseek = $this->parentStreamSeek($positionInFile); + + // only allow writes on seekable streams, or at the end of the encrypted stream + if (!$this->readOnly && ($resultFseek || $positionInFile === $this->size)) { + // switch the writeFlag so flush() will write the block + $this->writeFlag = true; + $this->fileUpdated = true; + + // determine the relative position in the current block + $blockPosition = ($this->position % $this->unencryptedBlockSize); + // check if $data fits in current block + // if so, overwrite existing data (if any) + // update position and liberate $data + if ($remainingLength < ($this->unencryptedBlockSize - $blockPosition)) { + $this->cache = substr($this->cache, 0, $blockPosition) + . $data . substr($this->cache, $blockPosition + $remainingLength); + $this->position += $remainingLength; + $length += $remainingLength; + $data = ''; + // if $data doesn't fit the current block, the fill the current block and reiterate + // after the block is filled, it is flushed and $data is updatedxxx + } else { + $this->cache = substr($this->cache, 0, $blockPosition) + . substr($data, 0, $this->unencryptedBlockSize - $blockPosition); + $this->flush(); + $this->position += ($this->unencryptedBlockSize - $blockPosition); + $length += ($this->unencryptedBlockSize - $blockPosition); + $data = substr($data, $this->unencryptedBlockSize - $blockPosition); + } + } else { + $data = ''; + } + $this->unencryptedSize = max($this->unencryptedSize, $this->position); + } + return $length; + } + + public function stream_tell() { + return $this->position; + } + + public function stream_seek($offset, $whence = SEEK_SET) { + $return = false; + + switch ($whence) { + case SEEK_SET: + $newPosition = $offset; + break; + case SEEK_CUR: + $newPosition = $this->position + $offset; + break; + case SEEK_END: + $newPosition = $this->unencryptedSize + $offset; + break; + default: + return $return; + } + + if ($newPosition > $this->unencryptedSize || $newPosition < 0) { + return $return; + } + + $newFilePosition = (int)floor($newPosition / $this->unencryptedBlockSize) + * $this->util->getBlockSize() + $this->headerSize; + + $oldFilePosition = parent::stream_tell(); + if ($this->parentStreamSeek($newFilePosition)) { + $this->parentStreamSeek($oldFilePosition); + $this->flush(); + $this->parentStreamSeek($newFilePosition); + $this->position = $newPosition; + $return = true; + } + return $return; + } + + public function stream_close() { + $this->flush('end'); + $position = (int)floor($this->position / $this->unencryptedBlockSize); + $remainingData = $this->encryptionModule->end($this->fullPath, $position . 'end'); + if ($this->readOnly === false) { + if (!empty($remainingData)) { + parent::stream_write($remainingData); + } + $this->encryptionStorage->updateUnencryptedSize($this->fullPath, $this->unencryptedSize); + } + $result = parent::stream_close(); + + if ($this->fileUpdated) { + $cache = $this->storage->getCache(); + $cacheEntry = $cache->get($this->internalPath); + if ($cacheEntry) { + $version = $cacheEntry['encryptedVersion'] + 1; + $cache->update($cacheEntry->getId(), ['encrypted' => $version, 'encryptedVersion' => $version, 'unencrypted_size' => $this->unencryptedSize]); + } + } + + return $result; + } + + /** + * write block to file + * @param string $positionPrefix + */ + protected function flush($positionPrefix = '') { + // write to disk only when writeFlag was set to 1 + if ($this->writeFlag) { + // Disable the file proxies so that encryption is not + // automatically attempted when the file is written to disk - + // we are handling that separately here and we don't want to + // get into an infinite loop + $position = (int)floor($this->position / $this->unencryptedBlockSize); + $encrypted = $this->encryptionModule->encrypt($this->cache, $position . $positionPrefix); + $bytesWritten = parent::stream_write($encrypted); + $this->writeFlag = false; + // Check whether the write concerns the last block + // If so then update the encrypted filesize + // Note that the unencrypted pointer and filesize are NOT yet updated when flush() is called + // We recalculate the encrypted filesize as we do not know the context of calling flush() + $completeBlocksInFile = (int)floor($this->unencryptedSize / $this->unencryptedBlockSize); + if ($completeBlocksInFile === (int)floor($this->position / $this->unencryptedBlockSize)) { + $this->size = $this->util->getBlockSize() * $completeBlocksInFile; + $this->size += $bytesWritten; + $this->size += $this->headerSize; + } + } + // always empty the cache (otherwise readCache() will not fill it with the new block) + $this->cache = ''; + } + + /** + * read block to file + */ + protected function readCache() { + // cache should always be empty string when this function is called + // don't try to fill the cache when trying to write at the end of the unencrypted file when it coincides with new block + if ($this->cache === '' && !($this->position === $this->unencryptedSize && ($this->position % $this->unencryptedBlockSize) === 0)) { + // Get the data from the file handle + $data = $this->stream_read_block($this->util->getBlockSize()); + $position = (int)floor($this->position / $this->unencryptedBlockSize); + $numberOfChunks = (int)($this->unencryptedSize / $this->unencryptedBlockSize); + if ($numberOfChunks === $position) { + $position .= 'end'; + } + $this->cache = $this->encryptionModule->decrypt($data, $position); + } + } + + /** + * write header at beginning of encrypted file + * + * @return int|false + * @throws EncryptionHeaderKeyExistsException if header key is already in use + */ + protected function writeHeader() { + $header = $this->util->createHeader($this->newHeader, $this->encryptionModule); + $this->fileUpdated = true; + return parent::stream_write($header); + } + + /** + * read first block to skip the header + */ + protected function skipHeader() { + $this->stream_read_block($this->headerSize); + } + + /** + * call stream_seek() from parent class + * + * @param integer $position + * @return bool + */ + protected function parentStreamSeek($position) { + return parent::stream_seek($position); + } + + /** + * @param string $path + * @param array $options + * @return bool + */ + public function dir_opendir($path, $options) { + return false; + } +} diff --git a/lib/private/Files/Stream/HashWrapper.php b/lib/private/Files/Stream/HashWrapper.php new file mode 100644 index 00000000000..5956ad92549 --- /dev/null +++ b/lib/private/Files/Stream/HashWrapper.php @@ -0,0 +1,68 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Stream; + +use Icewind\Streams\Wrapper; + +class HashWrapper extends Wrapper { + protected $callback; + protected $hash; + + public static function wrap($source, string $algo, callable $callback) { + $hash = hash_init($algo); + $context = stream_context_create([ + 'hash' => [ + 'source' => $source, + 'callback' => $callback, + 'hash' => $hash, + ], + ]); + return Wrapper::wrapSource($source, $context, 'hash', self::class); + } + + protected function open() { + $context = $this->loadContext('hash'); + + $this->callback = $context['callback']; + $this->hash = $context['hash']; + return true; + } + + public function dir_opendir($path, $options) { + return $this->open(); + } + + public function stream_open($path, $mode, $options, &$opened_path) { + return $this->open(); + } + + public function stream_read($count) { + $result = parent::stream_read($count); + hash_update($this->hash, $result); + return $result; + } + + public function stream_close() { + if (is_callable($this->callback)) { + // if the stream is closed as a result of the end-of-request GC, the hash context might be cleaned up before this stream + if ($this->hash instanceof \HashContext) { + try { + $hash = @hash_final($this->hash); + if ($hash) { + call_user_func($this->callback, $hash); + } + } catch (\Throwable $e) { + } + } + // prevent further calls by potential PHP 7 GC ghosts + $this->callback = null; + } + return parent::stream_close(); + } +} diff --git a/lib/private/Files/Stream/Quota.php b/lib/private/Files/Stream/Quota.php new file mode 100644 index 00000000000..cc737910fd8 --- /dev/null +++ b/lib/private/Files/Stream/Quota.php @@ -0,0 +1,84 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Stream; + +use Icewind\Streams\Wrapper; + +/** + * stream wrapper limits the amount of data that can be written to a stream + * + * usage: resource \OC\Files\Stream\Quota::wrap($stream, $limit) + */ +class Quota extends Wrapper { + /** + * @var int $limit + */ + private $limit; + + /** + * @param resource $stream + * @param int $limit + * @return resource|false + */ + public static function wrap($stream, $limit) { + $context = stream_context_create([ + 'quota' => [ + 'source' => $stream, + 'limit' => $limit + ] + ]); + return Wrapper::wrapSource($stream, $context, 'quota', self::class); + } + + public function stream_open($path, $mode, $options, &$opened_path) { + $context = $this->loadContext('quota'); + $this->source = $context['source']; + $this->limit = $context['limit']; + + return true; + } + + public function dir_opendir($path, $options) { + return false; + } + + public function stream_seek($offset, $whence = SEEK_SET) { + if ($whence === SEEK_END) { + // go to the end to find out last position's offset + $oldOffset = $this->stream_tell(); + if (fseek($this->source, 0, $whence) !== 0) { + return false; + } + $whence = SEEK_SET; + $offset = $this->stream_tell() + $offset; + $this->limit += $oldOffset - $offset; + } elseif ($whence === SEEK_SET) { + $this->limit += $this->stream_tell() - $offset; + } else { + $this->limit -= $offset; + } + // this wrapper needs to return "true" for success. + // the fseek call itself returns 0 on succeess + return fseek($this->source, $offset, $whence) === 0; + } + + public function stream_read($count) { + $this->limit -= $count; + return fread($this->source, $count); + } + + public function stream_write($data) { + $size = strlen($data); + if ($size > $this->limit) { + $data = substr($data, 0, $this->limit); + $size = $this->limit; + } + $this->limit -= $size; + return fwrite($this->source, $data); + } +} diff --git a/lib/private/Files/Stream/SeekableHttpStream.php b/lib/private/Files/Stream/SeekableHttpStream.php new file mode 100644 index 00000000000..6ce0a880e8d --- /dev/null +++ b/lib/private/Files/Stream/SeekableHttpStream.php @@ -0,0 +1,248 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Stream; + +use Icewind\Streams\File; +use Icewind\Streams\Wrapper; + +/** + * A stream wrapper that uses http range requests to provide a seekable stream for http reading + */ +class SeekableHttpStream implements File { + private const PROTOCOL = 'httpseek'; + + private static bool $registered = false; + + /** + * Registers the stream wrapper using the `httpseek://` url scheme + * $return void + */ + private static function registerIfNeeded() { + if (!self::$registered) { + stream_wrapper_register( + self::PROTOCOL, + self::class + ); + self::$registered = true; + } + } + + /** + * Open a readonly-seekable http stream + * + * The provided callback will be called with byte range and should return an http stream for the requested range + * + * @param callable $callback + * @return false|resource + */ + public static function open(callable $callback) { + $context = stream_context_create([ + SeekableHttpStream::PROTOCOL => [ + 'callback' => $callback + ], + ]); + + SeekableHttpStream::registerIfNeeded(); + return fopen(SeekableHttpStream::PROTOCOL . '://', 'r', false, $context); + } + + /** @var resource */ + public $context; + + /** @var callable */ + private $openCallback; + + /** @var ?resource|closed-resource */ + private $current; + /** @var int $offset offset of the current chunk */ + private int $offset = 0; + /** @var int $length length of the current chunk */ + private int $length = 0; + /** @var int $totalSize size of the full stream */ + private int $totalSize = 0; + private bool $needReconnect = false; + + private function reconnect(int $start): bool { + $this->needReconnect = false; + $range = $start . '-'; + if ($this->hasOpenStream()) { + fclose($this->current); + } + + $stream = ($this->openCallback)($range); + + if ($stream === false) { + $this->current = null; + return false; + } + $this->current = $stream; + + $responseHead = stream_get_meta_data($this->current)['wrapper_data']; + + while ($responseHead instanceof Wrapper) { + $wrapperOptions = stream_context_get_options($responseHead->context); + foreach ($wrapperOptions as $options) { + if (isset($options['source']) && is_resource($options['source'])) { + $responseHead = stream_get_meta_data($options['source'])['wrapper_data']; + continue 2; + } + } + throw new \Exception('Failed to get source stream from stream wrapper of ' . get_class($responseHead)); + } + + $rangeHeaders = array_values(array_filter($responseHead, function ($v) { + return preg_match('#^content-range:#i', $v) === 1; + })); + if (!$rangeHeaders) { + $this->current = null; + return false; + } + $contentRange = $rangeHeaders[0]; + + $content = trim(explode(':', $contentRange)[1]); + $range = trim(explode(' ', $content)[1]); + $begin = intval(explode('-', $range)[0]); + $length = intval(explode('/', $range)[1]); + + if ($begin !== $start) { + $this->current = null; + return false; + } + + $this->offset = $begin; + $this->length = $length; + if ($start === 0) { + $this->totalSize = $length; + } + + return true; + } + + /** + * @return ?resource + */ + private function getCurrent() { + if ($this->needReconnect) { + $this->reconnect($this->offset); + } + if (is_resource($this->current)) { + return $this->current; + } else { + return null; + } + } + + /** + * @return bool + * @psalm-assert-if-true resource $this->current + */ + private function hasOpenStream(): bool { + return is_resource($this->current); + } + + public function stream_open($path, $mode, $options, &$opened_path) { + $options = stream_context_get_options($this->context)[self::PROTOCOL]; + $this->openCallback = $options['callback']; + + return $this->reconnect(0); + } + + public function stream_read($count) { + if (!$this->getCurrent()) { + return false; + } + $ret = fread($this->getCurrent(), $count); + $this->offset += strlen($ret); + return $ret; + } + + public function stream_seek($offset, $whence = SEEK_SET) { + switch ($whence) { + case SEEK_SET: + if ($offset === $this->offset) { + return true; + } else { + $this->offset = $offset; + } + break; + case SEEK_CUR: + if ($offset === 0) { + return true; + } else { + $this->offset += $offset; + } + break; + case SEEK_END: + if ($this->length === 0) { + return false; + } elseif ($this->length + $offset === $this->offset) { + return true; + } else { + $this->offset = $this->length + $offset; + } + break; + } + + if ($this->hasOpenStream()) { + fclose($this->current); + } + $this->current = null; + $this->needReconnect = true; + return true; + } + + public function stream_tell() { + return $this->offset; + } + + public function stream_stat() { + if ($this->getCurrent()) { + $stat = fstat($this->getCurrent()); + if ($stat) { + $stat['size'] = $this->totalSize; + } + return $stat; + } else { + return false; + } + } + + public function stream_eof() { + if ($this->getCurrent()) { + return feof($this->getCurrent()); + } else { + return true; + } + } + + public function stream_close() { + if ($this->hasOpenStream()) { + fclose($this->current); + } + $this->current = null; + } + + public function stream_write($data) { + return false; + } + + public function stream_set_option($option, $arg1, $arg2) { + return false; + } + + public function stream_truncate($size) { + return false; + } + + public function stream_lock($operation) { + return false; + } + + public function stream_flush() { + return; //noop because readonly stream + } +} diff --git a/lib/private/Files/Template/TemplateManager.php b/lib/private/Files/Template/TemplateManager.php new file mode 100644 index 00000000000..80ef5a14786 --- /dev/null +++ b/lib/private/Files/Template/TemplateManager.php @@ -0,0 +1,421 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Template; + +use OC\AppFramework\Bootstrap\Coordinator; +use OC\Files\Cache\Scanner; +use OC\Files\Filesystem; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\GenericFileException; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\Template\BeforeGetTemplatesEvent; +use OCP\Files\Template\Field; +use OCP\Files\Template\FileCreatedFromTemplateEvent; +use OCP\Files\Template\ICustomTemplateProvider; +use OCP\Files\Template\ITemplateManager; +use OCP\Files\Template\RegisterTemplateCreatorEvent; +use OCP\Files\Template\Template; +use OCP\Files\Template\TemplateFileCreator; +use OCP\IConfig; +use OCP\IPreview; +use OCP\IServerContainer; +use OCP\IUserManager; +use OCP\IUserSession; +use OCP\L10N\IFactory; +use Psr\Log\LoggerInterface; + +class TemplateManager implements ITemplateManager { + private $registeredTypes = []; + private $types = []; + + /** @var array|null */ + private $providers = null; + + private $serverContainer; + private $eventDispatcher; + private $rootFolder; + private $userManager; + private $previewManager; + private $config; + private $l10n; + private $logger; + private $userId; + private $l10nFactory; + /** @var Coordinator */ + private $bootstrapCoordinator; + + public function __construct( + IServerContainer $serverContainer, + IEventDispatcher $eventDispatcher, + Coordinator $coordinator, + IRootFolder $rootFolder, + IUserSession $userSession, + IUserManager $userManager, + IPreview $previewManager, + IConfig $config, + IFactory $l10nFactory, + LoggerInterface $logger, + ) { + $this->serverContainer = $serverContainer; + $this->eventDispatcher = $eventDispatcher; + $this->bootstrapCoordinator = $coordinator; + $this->rootFolder = $rootFolder; + $this->userManager = $userManager; + $this->previewManager = $previewManager; + $this->config = $config; + $this->l10nFactory = $l10nFactory; + $this->l10n = $l10nFactory->get('lib'); + $this->logger = $logger; + $user = $userSession->getUser(); + $this->userId = $user ? $user->getUID() : null; + } + + public function registerTemplateFileCreator(callable $callback): void { + $this->registeredTypes[] = $callback; + } + + public function getRegisteredProviders(): array { + if ($this->providers !== null) { + return $this->providers; + } + + $context = $this->bootstrapCoordinator->getRegistrationContext(); + + $this->providers = []; + foreach ($context->getTemplateProviders() as $provider) { + $class = $provider->getService(); + $this->providers[$class] = $this->serverContainer->get($class); + } + return $this->providers; + } + + public function getTypes(): array { + if (!empty($this->types)) { + return $this->types; + } + $this->eventDispatcher->dispatchTyped(new RegisterTemplateCreatorEvent($this)); + foreach ($this->registeredTypes as $registeredType) { + $this->types[] = $registeredType(); + } + return $this->types; + } + + public function listCreators(): array { + $types = $this->getTypes(); + usort($types, function (TemplateFileCreator $a, TemplateFileCreator $b) { + return $a->getOrder() - $b->getOrder(); + }); + return $types; + } + + public function listTemplates(): array { + return array_values(array_map(function (TemplateFileCreator $entry) { + return array_merge($entry->jsonSerialize(), [ + 'templates' => $this->getTemplateFiles($entry) + ]); + }, $this->listCreators())); + } + + public function listTemplateFields(int $fileId): array { + foreach ($this->listCreators() as $creator) { + $fields = $this->getTemplateFields($creator, $fileId); + if (empty($fields)) { + continue; + } + + return $fields; + } + + return []; + } + + /** + * @param string $filePath + * @param string $templateId + * @param array $templateFields + * @return array + * @throws GenericFileException + */ + public function createFromTemplate(string $filePath, string $templateId = '', string $templateType = 'user', array $templateFields = []): array { + $userFolder = $this->rootFolder->getUserFolder($this->userId); + try { + $userFolder->get($filePath); + throw new GenericFileException($this->l10n->t('File already exists')); + } catch (NotFoundException $e) { + } + try { + if (!$userFolder->nodeExists(dirname($filePath))) { + throw new GenericFileException($this->l10n->t('Invalid path')); + } + $folder = $userFolder->get(dirname($filePath)); + $template = null; + if ($templateType === 'user' && $templateId !== '') { + $template = $userFolder->get($templateId); + } else { + $matchingProvider = array_filter($this->getRegisteredProviders(), function (ICustomTemplateProvider $provider) use ($templateType) { + return $templateType === get_class($provider); + }); + $provider = array_shift($matchingProvider); + if ($provider) { + $template = $provider->getCustomTemplate($templateId); + } + } + + $targetFile = $folder->newFile(basename($filePath), ($template instanceof File ? $template->fopen('rb') : null)); + + $this->eventDispatcher->dispatchTyped(new FileCreatedFromTemplateEvent($template, $targetFile, $templateFields)); + return $this->formatFile($userFolder->get($filePath)); + } catch (\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new GenericFileException($this->l10n->t('Failed to create file from template')); + } + } + + /** + * @return Folder + * @throws \OCP\Files\NotFoundException + * @throws \OCP\Files\NotPermittedException + * @throws \OC\User\NoUserException + */ + private function getTemplateFolder(): Folder { + if ($this->getTemplatePath() !== '') { + $path = $this->rootFolder->getUserFolder($this->userId)->get($this->getTemplatePath()); + if ($path instanceof Folder) { + return $path; + } + } + throw new NotFoundException(); + } + + /** + * @return list<Template> + */ + private function getTemplateFiles(TemplateFileCreator $type): array { + $templates = array_merge( + $this->getProviderTemplates($type), + $this->getUserTemplates($type) + ); + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($templates, false)); + + return $templates; + } + + /** + * @return list<Template> + */ + private function getProviderTemplates(TemplateFileCreator $type): array { + $templates = []; + foreach ($this->getRegisteredProviders() as $provider) { + foreach ($type->getMimetypes() as $mimetype) { + foreach ($provider->getCustomTemplates($mimetype) as $template) { + $templates[] = $template; + } + } + } + + return $templates; + } + + /** + * @return list<Template> + */ + private function getUserTemplates(TemplateFileCreator $type): array { + $templates = []; + + try { + $userTemplateFolder = $this->getTemplateFolder(); + } catch (\Exception $e) { + return $templates; + } + + foreach ($type->getMimetypes() as $mimetype) { + foreach ($userTemplateFolder->searchByMime($mimetype) as $templateFile) { + $template = new Template( + 'user', + $this->rootFolder->getUserFolder($this->userId)->getRelativePath($templateFile->getPath()), + $templateFile + ); + $template->setHasPreview($this->previewManager->isAvailable($templateFile)); + $templates[] = $template; + } + } + + return $templates; + } + + /* + * @return list<Field> + */ + private function getTemplateFields(TemplateFileCreator $type, int $fileId): array { + $providerTemplates = $this->getProviderTemplates($type); + $userTemplates = $this->getUserTemplates($type); + + $matchedTemplates = array_filter( + array_merge($providerTemplates, $userTemplates), + function (Template $template) use ($fileId) { + return $template->jsonSerialize()['fileid'] === $fileId; + }); + + if (empty($matchedTemplates)) { + return []; + } + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($matchedTemplates, true)); + + return array_values(array_map(function (Template $template) { + return $template->jsonSerialize()['fields'] ?? []; + }, $matchedTemplates)); + } + + /** + * @param Node|File $file + * @return array + * @throws NotFoundException + * @throws \OCP\Files\InvalidPathException + */ + private function formatFile(Node $file): array { + return [ + 'basename' => $file->getName(), + 'etag' => $file->getEtag(), + 'fileid' => $file->getId(), + 'filename' => $this->rootFolder->getUserFolder($this->userId)->getRelativePath($file->getPath()), + 'lastmod' => $file->getMTime(), + 'mime' => $file->getMimetype(), + 'size' => $file->getSize(), + 'type' => $file->getType(), + 'hasPreview' => $this->previewManager->isAvailable($file), + 'permissions' => $file->getPermissions(), + ]; + } + + public function hasTemplateDirectory(): bool { + try { + $this->getTemplateFolder(); + return true; + } catch (\Exception $e) { + } + return false; + } + + public function setTemplatePath(string $path): void { + $this->config->setUserValue($this->userId, 'core', 'templateDirectory', $path); + } + + public function getTemplatePath(): string { + return $this->config->getUserValue($this->userId, 'core', 'templateDirectory', ''); + } + + public function initializeTemplateDirectory(?string $path = null, ?string $userId = null, $copyTemplates = true): string { + if ($userId !== null) { + $this->userId = $userId; + } + + $defaultSkeletonDirectory = \OC::$SERVERROOT . '/core/skeleton'; + $defaultTemplateDirectory = \OC::$SERVERROOT . '/core/skeleton/Templates'; + $skeletonPath = $this->config->getSystemValueString('skeletondirectory', $defaultSkeletonDirectory); + $skeletonTemplatePath = $this->config->getSystemValueString('templatedirectory', $defaultTemplateDirectory); + $isDefaultSkeleton = $skeletonPath === $defaultSkeletonDirectory; + $isDefaultTemplates = $skeletonTemplatePath === $defaultTemplateDirectory; + $userLang = $this->l10nFactory->getUserLanguage($this->userManager->get($this->userId)); + + if ($skeletonTemplatePath === '') { + $this->setTemplatePath(''); + return ''; + } + + try { + $l10n = $this->l10nFactory->get('lib', $userLang); + $userFolder = $this->rootFolder->getUserFolder($this->userId); + $userTemplatePath = $path ?? $this->config->getAppValue('core', 'defaultTemplateDirectory', $l10n->t('Templates')) . '/'; + + // Initial user setup without a provided path + if ($path === null) { + // All locations are default so we just need to rename the directory to the users language + if ($isDefaultSkeleton && $isDefaultTemplates) { + if (!$userFolder->nodeExists('Templates')) { + return ''; + } + $newPath = Filesystem::normalizePath($userFolder->getPath() . '/' . $userTemplatePath); + if ($newPath !== $userFolder->get('Templates')->getPath()) { + $userFolder->get('Templates')->move($newPath); + } + $this->setTemplatePath($userTemplatePath); + return $userTemplatePath; + } + + if ($isDefaultSkeleton && !empty($skeletonTemplatePath) && !$isDefaultTemplates && $userFolder->nodeExists('Templates')) { + $shippedSkeletonTemplates = $userFolder->get('Templates'); + $shippedSkeletonTemplates->delete(); + } + } + + try { + $folder = $userFolder->get($userTemplatePath); + } catch (NotFoundException $e) { + $folder = $userFolder->get(dirname($userTemplatePath)); + $folder = $folder->newFolder(basename($userTemplatePath)); + } + + $folderIsEmpty = count($folder->getDirectoryListing()) === 0; + + if (!$copyTemplates) { + $this->setTemplatePath($userTemplatePath); + return $userTemplatePath; + } + + if (!$isDefaultTemplates && $folderIsEmpty) { + $localizedSkeletonTemplatePath = $this->getLocalizedTemplatePath($skeletonTemplatePath, $userLang); + if (!empty($localizedSkeletonTemplatePath) && file_exists($localizedSkeletonTemplatePath)) { + \OC_Util::copyr($localizedSkeletonTemplatePath, $folder); + $userFolder->getStorage()->getScanner()->scan($folder->getInternalPath(), Scanner::SCAN_RECURSIVE); + $this->setTemplatePath($userTemplatePath); + return $userTemplatePath; + } + } + + if ($path !== null && $isDefaultSkeleton && $isDefaultTemplates && $folderIsEmpty) { + $localizedSkeletonPath = $this->getLocalizedTemplatePath($skeletonPath . '/Templates', $userLang); + if (!empty($localizedSkeletonPath) && file_exists($localizedSkeletonPath)) { + \OC_Util::copyr($localizedSkeletonPath, $folder); + $userFolder->getStorage()->getScanner()->scan($folder->getInternalPath(), Scanner::SCAN_RECURSIVE); + $this->setTemplatePath($userTemplatePath); + return $userTemplatePath; + } + } + + $this->setTemplatePath($path ?? ''); + return $this->getTemplatePath(); + } catch (\Throwable $e) { + $this->logger->error('Failed to initialize templates directory to user language ' . $userLang . ' for ' . $userId, ['app' => 'files_templates', 'exception' => $e]); + } + $this->setTemplatePath(''); + return $this->getTemplatePath(); + } + + private function getLocalizedTemplatePath(string $skeletonTemplatePath, string $userLang) { + $localizedSkeletonTemplatePath = str_replace('{lang}', $userLang, $skeletonTemplatePath); + + if (!file_exists($localizedSkeletonTemplatePath)) { + $dialectStart = strpos($userLang, '_'); + if ($dialectStart !== false) { + $localizedSkeletonTemplatePath = str_replace('{lang}', substr($userLang, 0, $dialectStart), $skeletonTemplatePath); + } + if ($dialectStart === false || !file_exists($localizedSkeletonTemplatePath)) { + $localizedSkeletonTemplatePath = str_replace('{lang}', 'default', $skeletonTemplatePath); + } + } + + return $localizedSkeletonTemplatePath; + } +} diff --git a/lib/private/Files/Type/Detection.php b/lib/private/Files/Type/Detection.php new file mode 100644 index 00000000000..6af6ce1a0b1 --- /dev/null +++ b/lib/private/Files/Type/Detection.php @@ -0,0 +1,376 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Type; + +use OCP\Files\IMimeTypeDetector; +use OCP\IBinaryFinder; +use OCP\ITempManager; +use OCP\IURLGenerator; +use Psr\Log\LoggerInterface; + +/** + * Class Detection + * + * Mimetype detection + * + * @package OC\Files\Type + */ +class Detection implements IMimeTypeDetector { + private const CUSTOM_MIMETYPEMAPPING = 'mimetypemapping.json'; + private const CUSTOM_MIMETYPEALIASES = 'mimetypealiases.json'; + private const CUSTOM_MIMETYPENAMES = 'mimetypenames.json'; + + /** @var array<list{string, string|null}> */ + protected array $mimeTypes = []; + protected array $secureMimeTypes = []; + + protected array $mimeTypeIcons = []; + /** @var array<string,string> */ + protected array $mimeTypeAlias = []; + /** @var array<string,string> */ + protected array $mimeTypesNames = []; + + public function __construct( + private IURLGenerator $urlGenerator, + private LoggerInterface $logger, + private string $customConfigDir, + private string $defaultConfigDir, + ) { + } + + /** + * Add an extension -> MIME type mapping + * + * $mimeType is the assumed correct mime type + * The optional $secureMimeType is an alternative to send to send + * to avoid potential XSS. + * + * @param string $extension + * @param string $mimeType + * @param string|null $secureMimeType + */ + public function registerType( + string $extension, + string $mimeType, + ?string $secureMimeType = null): void { + // Make sure the extension is a string + // https://github.com/nextcloud/server/issues/42902 + $this->mimeTypes[$extension] = [$mimeType, $secureMimeType]; + $this->secureMimeTypes[$mimeType] = $secureMimeType ?? $mimeType; + } + + /** + * Add an array of extension -> MIME type mappings + * + * The mimeType value is in itself an array where the first index is + * the assumed correct mimeType and the second is either a secure alternative + * or null if the correct is considered secure. + * + * @param array $types + */ + public function registerTypeArray(array $types): void { + // Register the types, + foreach ($types as $extension => $mimeType) { + $this->registerType((string)$extension, $mimeType[0], $mimeType[1] ?? null); + } + + // Update the alternative mimeTypes to avoid having to look them up each time. + foreach ($this->mimeTypes as $extension => $mimeType) { + if (str_starts_with((string)$extension, '_comment')) { + continue; + } + + $this->secureMimeTypes[$mimeType[0]] = $mimeType[1] ?? $mimeType[0]; + if (isset($mimeType[1])) { + $this->secureMimeTypes[$mimeType[1]] = $mimeType[1]; + } + } + } + + private function loadCustomDefinitions(string $fileName, array $definitions): array { + if (file_exists($this->customConfigDir . '/' . $fileName)) { + $custom = json_decode(file_get_contents($this->customConfigDir . '/' . $fileName), true); + if (json_last_error() === JSON_ERROR_NONE) { + $definitions = array_merge($definitions, $custom); + } else { + $this->logger->warning('Failed to parse ' . $fileName . ': ' . json_last_error_msg()); + } + } + return $definitions; + } + + /** + * Add the MIME type aliases if they are not yet present + */ + private function loadAliases(): void { + if (!empty($this->mimeTypeAlias)) { + return; + } + + $this->mimeTypeAlias = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypealiases.dist.json'), true); + $this->mimeTypeAlias = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPEALIASES, $this->mimeTypeAlias); + } + + /** + * @return array<string,string> + */ + public function getAllAliases(): array { + $this->loadAliases(); + return $this->mimeTypeAlias; + } + + public function getOnlyDefaultAliases(): array { + $this->loadMappings(); + $this->mimeTypeAlias = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypealiases.dist.json'), true); + return $this->mimeTypeAlias; + } + + /** + * Add MIME type mappings if they are not yet present + */ + private function loadMappings(): void { + if (!empty($this->mimeTypes)) { + return; + } + + $mimeTypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypemapping.dist.json'), true); + $mimeTypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPEMAPPING, $mimeTypeMapping); + + $this->registerTypeArray($mimeTypeMapping); + } + + /** + * @return array<list{string, string|null}> + */ + public function getAllMappings(): array { + $this->loadMappings(); + return $this->mimeTypes; + } + + private function loadNamings(): void { + if (!empty($this->mimeTypesNames)) { + return; + } + + $mimeTypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypenames.dist.json'), true); + $mimeTypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPENAMES, $mimeTypeMapping); + + $this->mimeTypesNames = $mimeTypeMapping; + } + + /** + * @return array<string,string> + */ + public function getAllNamings(): array { + $this->loadNamings(); + return $this->mimeTypesNames; + } + + /** + * detect MIME type only based on filename, content of file is not used + * + * @param string $path + * @return string + */ + public function detectPath($path): string { + $this->loadMappings(); + + $fileName = basename($path); + + // remove leading dot on hidden files with a file extension + $fileName = ltrim($fileName, '.'); + + // note: leading dot doesn't qualify as extension + if (strpos($fileName, '.') > 0) { + // remove versioning extension: name.v1508946057 and transfer extension: name.ocTransferId2057600214.part + $fileName = preg_replace('!((\.v\d+)|((\.ocTransferId\d+)?\.part))$!', '', $fileName); + + //try to guess the type by the file extension + $extension = strrchr($fileName, '.'); + if ($extension !== false) { + $extension = strtolower($extension); + $extension = substr($extension, 1); // remove leading . + return $this->mimeTypes[$extension][0] ?? 'application/octet-stream'; + } + } + + return 'application/octet-stream'; + } + + /** + * Detect MIME type only based on the content of file. + * + * @param string $path + * @return string + * @since 18.0.0 + */ + public function detectContent(string $path): string { + $this->loadMappings(); + + if (@is_dir($path)) { + // directories are easy + return 'httpd/unix-directory'; + } + + if (class_exists(finfo::class)) { + $finfo = new finfo(FILEINFO_MIME_TYPE); + $mimeType = @$finfo->file($path); + if ($mimeType) { + $mimeType = $this->getSecureMimeType($mimeType); + if ($mimeType !== 'application/octet-stream') { + return $mimeType; + } + } + } + + if (str_starts_with($path, 'file://')) { + // Is the file wrapped in a stream? + return 'application/octet-stream'; + } + + if (function_exists('mime_content_type')) { + // use mime magic extension if available + $mimeType = mime_content_type($path); + if ($mimeType) { + $mimeType = $this->getSecureMimeType($mimeType); + if ($mimeType !== 'application/octet-stream') { + return $mimeType; + } + } + } + + $binaryFinder = \OCP\Server::get(IBinaryFinder::class); + $program = $binaryFinder->findBinaryPath('file'); + if ($program !== false) { + // it looks like we have a 'file' command, + // lets see if it does have mime support + $path = escapeshellarg($path); + $fp = popen("test -f $path && $program -b --mime-type $path", 'r'); + if ($fp !== false) { + $mimeType = fgets($fp); + pclose($fp); + if ($mimeType) { + //trim the newline + $mimeType = trim($mimeType); + $mimeType = $this->getSecureMimeType($mimeType); + return $mimeType; + } + } + } + + return 'application/octet-stream'; + } + + /** + * Detect MIME type based on both filename and content + * + * @param string $path + * @return string + */ + public function detect($path): string { + $mimeType = $this->detectPath($path); + + if ($mimeType !== 'application/octet-stream') { + return $mimeType; + } + + return $this->detectContent($path); + } + + /** + * Detect MIME type based on the content of a string + * + * @param string $data + * @return string + */ + public function detectString($data): string { + if (class_exists(finfo::class)) { + $finfo = new finfo(FILEINFO_MIME_TYPE); + $mimeType = $finfo->buffer($data); + if ($mimeType) { + return $mimeType; + } + } + + $tmpFile = \OCP\Server::get(ITempManager::class)->getTemporaryFile(); + $fh = fopen($tmpFile, 'wb'); + fwrite($fh, $data, 8024); + fclose($fh); + $mimeType = $this->detect($tmpFile); + unset($tmpFile); + return $mimeType; + } + + /** + * Get a secure MIME type that won't expose potential XSS. + * + * @param string $mimeType + * @return string + */ + public function getSecureMimeType($mimeType): string { + $this->loadMappings(); + + return $this->secureMimeTypes[$mimeType] ?? 'application/octet-stream'; + } + + /** + * Get path to the icon of a file type + * @param string $mimeType the MIME type + * @return string the url + */ + public function mimeTypeIcon($mimeType): string { + $this->loadAliases(); + + while (isset($this->mimeTypeAlias[$mimeType])) { + $mimeType = $this->mimeTypeAlias[$mimeType]; + } + if (isset($this->mimeTypeIcons[$mimeType])) { + return $this->mimeTypeIcons[$mimeType]; + } + + // Replace slash and backslash with a minus + $icon = str_replace(['/', '\\'], '-', $mimeType); + + // Is it a dir? + if ($mimeType === 'dir') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder.svg'); + return $this->mimeTypeIcons[$mimeType]; + } + if ($mimeType === 'dir-shared') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder-shared.svg'); + return $this->mimeTypeIcons[$mimeType]; + } + if ($mimeType === 'dir-external') { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/folder-external.svg'); + return $this->mimeTypeIcons[$mimeType]; + } + + // Icon exists? + try { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/' . $icon . '.svg'); + return $this->mimeTypeIcons[$mimeType]; + } catch (\RuntimeException $e) { + // Specified image not found + } + + // Try only the first part of the filetype + if (strpos($icon, '-')) { + $mimePart = substr($icon, 0, strpos($icon, '-')); + try { + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/' . $mimePart . '.svg'); + return $this->mimeTypeIcons[$mimeType]; + } catch (\RuntimeException $e) { + // Image for the first part of the MIME type not found + } + } + + $this->mimeTypeIcons[$mimeType] = $this->urlGenerator->imagePath('core', 'filetypes/file.svg'); + return $this->mimeTypeIcons[$mimeType]; + } +} diff --git a/lib/private/Files/Type/Loader.php b/lib/private/Files/Type/Loader.php new file mode 100644 index 00000000000..5fbe4139759 --- /dev/null +++ b/lib/private/Files/Type/Loader.php @@ -0,0 +1,165 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Type; + +use OC\DB\Exceptions\DbalException; +use OCP\AppFramework\Db\TTransactional; +use OCP\DB\Exception as DBException; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; + +/** + * Mimetype database loader + * + * @package OC\Files\Type + */ +class Loader implements IMimeTypeLoader { + use TTransactional; + + /** @psalm-var array<int, string> */ + protected array $mimetypes; + + /** @psalm-var array<string, int> */ + protected array $mimetypeIds; + + /** + * @param IDBConnection $dbConnection + */ + public function __construct( + private IDBConnection $dbConnection, + ) { + $this->mimetypes = []; + $this->mimetypeIds = []; + } + + /** + * Get a mimetype from its ID + */ + public function getMimetypeById(int $id): ?string { + if (!$this->mimetypes) { + $this->loadMimetypes(); + } + if (isset($this->mimetypes[$id])) { + return $this->mimetypes[$id]; + } + return null; + } + + /** + * Get a mimetype ID, adding the mimetype to the DB if it does not exist + */ + public function getId(string $mimetype): int { + if (!$this->mimetypeIds) { + $this->loadMimetypes(); + } + if (isset($this->mimetypeIds[$mimetype])) { + return $this->mimetypeIds[$mimetype]; + } + return $this->store($mimetype); + } + + /** + * Test if a mimetype exists in the database + */ + public function exists(string $mimetype): bool { + if (!$this->mimetypeIds) { + $this->loadMimetypes(); + } + return isset($this->mimetypeIds[$mimetype]); + } + + /** + * Clear all loaded mimetypes, allow for re-loading + */ + public function reset(): void { + $this->mimetypes = []; + $this->mimetypeIds = []; + } + + /** + * Store a mimetype in the DB + * + * @param string $mimetype + * @return int inserted ID + */ + protected function store(string $mimetype): int { + try { + $mimetypeId = $this->atomic(function () use ($mimetype) { + $insert = $this->dbConnection->getQueryBuilder(); + $insert->insert('mimetypes') + ->values([ + 'mimetype' => $insert->createNamedParameter($mimetype) + ]) + ->executeStatement(); + return $insert->getLastInsertId(); + }, $this->dbConnection); + } catch (DbalException $e) { + if ($e->getReason() !== DBException::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('id') + ->from('mimetypes') + ->where($qb->expr()->eq('mimetype', $qb->createNamedParameter($mimetype))); + $result = $qb->executeQuery(); + $id = $result->fetchOne(); + $result->closeCursor(); + if ($id === false) { + throw new \Exception("Database threw an unique constraint on inserting a new mimetype, but couldn't return the ID for this very mimetype"); + } + + $mimetypeId = (int)$id; + } + + $this->mimetypes[$mimetypeId] = $mimetype; + $this->mimetypeIds[$mimetype] = $mimetypeId; + return $mimetypeId; + } + + /** + * Load all mimetypes from DB + */ + private function loadMimetypes(): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('id', 'mimetype') + ->from('mimetypes'); + + $result = $qb->executeQuery(); + $results = $result->fetchAll(); + $result->closeCursor(); + + foreach ($results as $row) { + $this->mimetypes[(int)$row['id']] = $row['mimetype']; + $this->mimetypeIds[$row['mimetype']] = (int)$row['id']; + } + } + + /** + * Update filecache mimetype based on file extension + * + * @return int number of changed rows + */ + public function updateFilecache(string $ext, int $mimeTypeId): int { + $folderMimeTypeId = $this->getId('httpd/unix-directory'); + $update = $this->dbConnection->getQueryBuilder(); + $update->update('filecache') + ->set('mimetype', $update->createNamedParameter($mimeTypeId)) + ->where($update->expr()->neq( + 'mimetype', $update->createNamedParameter($mimeTypeId) + )) + ->andWhere($update->expr()->neq( + 'mimetype', $update->createNamedParameter($folderMimeTypeId) + )) + ->andWhere($update->expr()->like( + $update->func()->lower('name'), + $update->createNamedParameter('%' . $this->dbConnection->escapeLikeParameter('.' . $ext)) + )); + return $update->executeStatement(); + } +} diff --git a/lib/private/Files/Type/TemplateManager.php b/lib/private/Files/Type/TemplateManager.php new file mode 100644 index 00000000000..a0c0e9fac9f --- /dev/null +++ b/lib/private/Files/Type/TemplateManager.php @@ -0,0 +1,50 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Type; + +/** + * @deprecated 18.0.0 + */ +class TemplateManager { + protected $templates = []; + + public function registerTemplate($mimetype, $path) { + $this->templates[$mimetype] = $path; + } + + /** + * get the path of the template for a mimetype + * + * @deprecated 18.0.0 + * @param string $mimetype + * @return string|null + */ + public function getTemplatePath($mimetype) { + if (isset($this->templates[$mimetype])) { + return $this->templates[$mimetype]; + } else { + return null; + } + } + + /** + * get the template content for a mimetype + * + * @deprecated 18.0.0 + * @param string $mimetype + * @return string + */ + public function getTemplate($mimetype) { + $path = $this->getTemplatePath($mimetype); + if ($path) { + return file_get_contents($path); + } else { + return ''; + } + } +} diff --git a/lib/private/Files/Utils/PathHelper.php b/lib/private/Files/Utils/PathHelper.php new file mode 100644 index 00000000000..db1294bcc10 --- /dev/null +++ b/lib/private/Files/Utils/PathHelper.php @@ -0,0 +1,57 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\Utils; + +class PathHelper { + /** + * Make a path relative to a root path, or return null if the path is outside the root + * + * @param string $root + * @param string $path + * @return ?string + */ + public static function getRelativePath(string $root, string $path) { + if ($root === '' or $root === '/') { + return self::normalizePath($path); + } + if ($path === $root) { + return '/'; + } elseif (!str_starts_with($path, $root . '/')) { + return null; + } else { + $path = substr($path, strlen($root)); + return self::normalizePath($path); + } + } + + /** + * @param string $path + * @return string + */ + public static function normalizePath(string $path): string { + if ($path === '' or $path === '/') { + return '/'; + } + // No null bytes + $path = str_replace(chr(0), '', $path); + //no windows style slashes + $path = str_replace('\\', '/', $path); + //add leading slash + if ($path[0] !== '/') { + $path = '/' . $path; + } + //remove duplicate slashes + while (str_contains($path, '//')) { + $path = str_replace('//', '/', $path); + } + //remove trailing slash + $path = rtrim($path, '/'); + + return $path; + } +} diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php new file mode 100644 index 00000000000..576cb66b3cf --- /dev/null +++ b/lib/private/Files/Utils/Scanner.php @@ -0,0 +1,307 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\Utils; + +use OC\Files\Cache\Cache; +use OC\Files\Filesystem; +use OC\Files\Storage\FailedStorage; +use OC\Files\Storage\Home; +use OC\ForbiddenException; +use OC\Hooks\PublicEmitter; +use OC\Lock\DBLockingProvider; +use OCA\Files_Sharing\SharedStorage; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\BeforeFileScannedEvent; +use OCP\Files\Events\BeforeFolderScannedEvent; +use OCP\Files\Events\FileCacheUpdated; +use OCP\Files\Events\FileScannedEvent; +use OCP\Files\Events\FolderScannedEvent; +use OCP\Files\Events\NodeAddedToCache; +use OCP\Files\Events\NodeRemovedFromCache; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\IDBConnection; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; +use Psr\Log\LoggerInterface; + +/** + * Class Scanner + * + * Hooks available in scope \OC\Utils\Scanner + * - scanFile(string $absolutePath) + * - scanFolder(string $absolutePath) + * + * @package OC\Files\Utils + */ +class Scanner extends PublicEmitter { + public const MAX_ENTRIES_TO_COMMIT = 10000; + + /** @var string $user */ + private $user; + + /** @var IDBConnection */ + protected $db; + + /** @var IEventDispatcher */ + private $dispatcher; + + protected LoggerInterface $logger; + + /** + * Whether to use a DB transaction + * + * @var bool + */ + protected $useTransaction; + + /** + * Number of entries scanned to commit + * + * @var int + */ + protected $entriesToCommit; + + /** + * @param string $user + * @param IDBConnection|null $db + * @param IEventDispatcher $dispatcher + */ + public function __construct($user, $db, IEventDispatcher $dispatcher, LoggerInterface $logger) { + $this->user = $user; + $this->db = $db; + $this->dispatcher = $dispatcher; + $this->logger = $logger; + // when DB locking is used, no DB transactions will be used + $this->useTransaction = !(\OC::$server->get(ILockingProvider::class) instanceof DBLockingProvider); + } + + /** + * get all storages for $dir + * + * @param string $dir + * @return array<string, IMountPoint> + */ + protected function getMounts($dir) { + //TODO: move to the node based fileapi once that's done + \OC_Util::tearDownFS(); + \OC_Util::setupFS($this->user); + + $mountManager = Filesystem::getMountManager(); + $mounts = $mountManager->findIn($dir); + $mounts[] = $mountManager->find($dir); + $mounts = array_reverse($mounts); //start with the mount of $dir + $mountPoints = array_map(fn ($mount) => $mount->getMountPoint(), $mounts); + + return array_combine($mountPoints, $mounts); + } + + /** + * attach listeners to the scanner + * + * @param \OC\Files\Mount\MountPoint $mount + */ + protected function attachListener($mount) { + /** @var \OC\Files\Cache\Scanner $scanner */ + $scanner = $mount->getStorage()->getScanner(); + $scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'scanFile', [$mount->getMountPoint() . $path]); + $this->dispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'scanFolder', [$mount->getMountPoint() . $path]); + $this->dispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'postScanFile', [$mount->getMountPoint() . $path]); + $this->dispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', [$mount->getMountPoint() . $path]); + $this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', [$path]); + }); + } + + /** + * @param string $dir + */ + public function backgroundScan($dir) { + $mounts = $this->getMounts($dir); + foreach ($mounts as $mount) { + try { + $storage = $mount->getStorage(); + if (is_null($storage)) { + continue; + } + + // don't bother scanning failed storages (shortcut for same result) + if ($storage->instanceOfStorage(FailedStorage::class)) { + continue; + } + + /** @var \OC\Files\Cache\Scanner $scanner */ + $scanner = $storage->getScanner(); + $this->attachListener($mount); + + $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) { + $this->triggerPropagator($storage, $path); + }); + + $propagator = $storage->getPropagator(); + $propagator->beginBatch(); + $scanner->backgroundScan(); + $propagator->commitBatch(); + } catch (\Exception $e) { + $this->logger->error("Error while trying to scan mount as {$mount->getMountPoint()}:" . $e->getMessage(), ['exception' => $e, 'app' => 'files']); + } + } + } + + /** + * @param string $dir + * @param $recursive + * @param callable|null $mountFilter + * @throws ForbiddenException + * @throws NotFoundException + */ + public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, ?callable $mountFilter = null) { + if (!Filesystem::isValidPath($dir)) { + throw new \InvalidArgumentException('Invalid path to scan'); + } + $mounts = $this->getMounts($dir); + foreach ($mounts as $mount) { + if ($mountFilter && !$mountFilter($mount)) { + continue; + } + $storage = $mount->getStorage(); + if (is_null($storage)) { + continue; + } + + // don't bother scanning failed storages (shortcut for same result) + if ($storage->instanceOfStorage(FailedStorage::class)) { + continue; + } + + // if the home storage isn't writable then the scanner is run as the wrong user + if ($storage->instanceOfStorage(Home::class)) { + /** @var Home $storage */ + foreach (['', 'files'] as $path) { + if (!$storage->isCreatable($path)) { + $fullPath = $storage->getSourcePath($path); + if (isset($mounts[$mount->getMountPoint() . $path . '/'])) { + // /<user>/files is overwritten by a mountpoint, so this check is irrelevant + break; + } elseif (!$storage->is_dir($path) && $storage->getCache()->inCache($path)) { + throw new NotFoundException("User folder $fullPath exists in cache but not on disk"); + } elseif ($storage->is_dir($path)) { + $ownerUid = fileowner($fullPath); + $owner = posix_getpwuid($ownerUid); + $owner = $owner['name'] ?? $ownerUid; + $permissions = decoct(fileperms($fullPath)); + throw new ForbiddenException("User folder $fullPath is not writable, folders is owned by $owner and has mode $permissions"); + } else { + // if the root exists in neither the cache nor the storage the user isn't setup yet + break 2; + } + } + } + } + + // don't scan received local shares, these can be scanned when scanning the owner's storage + if ($storage->instanceOfStorage(SharedStorage::class)) { + continue; + } + $relativePath = $mount->getInternalPath($dir); + /** @var \OC\Files\Cache\Scanner $scanner */ + $scanner = $storage->getScanner(); + $scanner->setUseTransactions(false); + $this->attachListener($mount); + + $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) { + $this->postProcessEntry($storage, $path); + $this->dispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) { + $this->postProcessEntry($storage, $path); + $this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); + }); + $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path, $storageId, $data, $fileId) use ($storage) { + $this->postProcessEntry($storage, $path); + if ($fileId) { + $this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); + } else { + $this->dispatcher->dispatchTyped(new NodeAddedToCache($storage, $path)); + } + }); + + if (!$storage->file_exists($relativePath)) { + throw new NotFoundException($dir); + } + + if ($this->useTransaction) { + $this->db->beginTransaction(); + } + try { + $propagator = $storage->getPropagator(); + $propagator->beginBatch(); + try { + $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + } catch (LockedException $e) { + if (is_string($e->getReadablePath()) && str_starts_with($e->getReadablePath(), 'scanner::')) { + throw new LockedException("scanner::$dir", $e, $e->getExistingLock()); + } else { + throw $e; + } + } + $cache = $storage->getCache(); + if ($cache instanceof Cache) { + // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner + $cache->correctFolderSize($relativePath); + } + $propagator->commitBatch(); + } catch (StorageNotAvailableException $e) { + $this->logger->error('Storage ' . $storage->getId() . ' not available', ['exception' => $e]); + $this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]); + } + if ($this->useTransaction) { + $this->db->commit(); + } + } + } + + private function triggerPropagator(IStorage $storage, $internalPath) { + $storage->getPropagator()->propagateChange($internalPath, time()); + } + + private function postProcessEntry(IStorage $storage, $internalPath) { + $this->triggerPropagator($storage, $internalPath); + if ($this->useTransaction) { + $this->entriesToCommit++; + if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) { + $propagator = $storage->getPropagator(); + $this->entriesToCommit = 0; + $this->db->commit(); + $propagator->commitBatch(); + $this->db->beginTransaction(); + $propagator->beginBatch(); + } + } + } +} diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php new file mode 100644 index 00000000000..a852f453963 --- /dev/null +++ b/lib/private/Files/View.php @@ -0,0 +1,2286 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files; + +use Icewind\Streams\CallbackWrapper; +use OC\Files\Mount\MoveableMount; +use OC\Files\Storage\Storage; +use OC\Files\Storage\Wrapper\Quota; +use OC\Share\Share; +use OC\User\LazyUser; +use OC\User\Manager as UserManager; +use OC\User\User; +use OCA\Files_Sharing\SharedMount; +use OCP\Constants; +use OCP\Files; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\ConnectionLostException; +use OCP\Files\EmptyFileNameException; +use OCP\Files\FileNameTooLongException; +use OCP\Files\ForbiddenException; +use OCP\Files\InvalidCharacterInPathException; +use OCP\Files\InvalidDirectoryException; +use OCP\Files\InvalidPathException; +use OCP\Files\Mount\IMountManager; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotFoundException; +use OCP\Files\ReservedWordException; +use OCP\IUser; +use OCP\IUserManager; +use OCP\L10N\IFactory; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; +use OCP\Server; +use OCP\Share\IManager; +use OCP\Share\IShare; +use Psr\Log\LoggerInterface; + +/** + * Class to provide access to ownCloud filesystem via a "view", and methods for + * working with files within that view (e.g. read, write, delete, etc.). Each + * view is restricted to a set of directories via a virtual root. The default view + * uses the currently logged in user's data directory as root (parts of + * OC_Filesystem are merely a wrapper for OC\Files\View). + * + * Apps that need to access files outside of the user data folders (to modify files + * belonging to a user other than the one currently logged in, for example) should + * use this class directly rather than using OC_Filesystem, or making use of PHP's + * built-in file manipulation functions. This will ensure all hooks and proxies + * are triggered correctly. + * + * Filesystem functions are not called directly; they are passed to the correct + * \OC\Files\Storage\Storage object + */ +class View { + private string $fakeRoot = ''; + private ILockingProvider $lockingProvider; + private bool $lockingEnabled; + private bool $updaterEnabled = true; + private UserManager $userManager; + private LoggerInterface $logger; + + /** + * @throws \Exception If $root contains an invalid path + */ + public function __construct(string $root = '') { + if (!Filesystem::isValidPath($root)) { + throw new \Exception(); + } + + $this->fakeRoot = $root; + $this->lockingProvider = \OC::$server->get(ILockingProvider::class); + $this->lockingEnabled = !($this->lockingProvider instanceof \OC\Lock\NoopLockingProvider); + $this->userManager = \OC::$server->getUserManager(); + $this->logger = \OC::$server->get(LoggerInterface::class); + } + + /** + * @param ?string $path + * @psalm-template S as string|null + * @psalm-param S $path + * @psalm-return (S is string ? string : null) + */ + public function getAbsolutePath($path = '/'): ?string { + if ($path === null) { + return null; + } + $this->assertPathLength($path); + if ($path === '') { + $path = '/'; + } + if ($path[0] !== '/') { + $path = '/' . $path; + } + return $this->fakeRoot . $path; + } + + /** + * Change the root to a fake root + * + * @param string $fakeRoot + */ + public function chroot($fakeRoot): void { + if (!$fakeRoot == '') { + if ($fakeRoot[0] !== '/') { + $fakeRoot = '/' . $fakeRoot; + } + } + $this->fakeRoot = $fakeRoot; + } + + /** + * Get the fake root + */ + public function getRoot(): string { + return $this->fakeRoot; + } + + /** + * get path relative to the root of the view + * + * @param string $path + */ + public function getRelativePath($path): ?string { + $this->assertPathLength($path); + if ($this->fakeRoot == '') { + return $path; + } + + if (rtrim($path, '/') === rtrim($this->fakeRoot, '/')) { + return '/'; + } + + // missing slashes can cause wrong matches! + $root = rtrim($this->fakeRoot, '/') . '/'; + + if (!str_starts_with($path, $root)) { + return null; + } else { + $path = substr($path, strlen($this->fakeRoot)); + if (strlen($path) === 0) { + return '/'; + } else { + return $path; + } + } + } + + /** + * Get the mountpoint of the storage object for a path + * ( note: because a storage is not always mounted inside the fakeroot, the + * returned mountpoint is relative to the absolute root of the filesystem + * and does not take the chroot into account ) + * + * @param string $path + */ + public function getMountPoint($path): string { + return Filesystem::getMountPoint($this->getAbsolutePath($path)); + } + + /** + * Get the mountpoint of the storage object for a path + * ( note: because a storage is not always mounted inside the fakeroot, the + * returned mountpoint is relative to the absolute root of the filesystem + * and does not take the chroot into account ) + * + * @param string $path + */ + public function getMount($path): IMountPoint { + return Filesystem::getMountManager()->find($this->getAbsolutePath($path)); + } + + /** + * Resolve a path to a storage and internal path + * + * @param string $path + * @return array{?\OCP\Files\Storage\IStorage, string} an array consisting of the storage and the internal path + */ + public function resolvePath($path): array { + $a = $this->getAbsolutePath($path); + $p = Filesystem::normalizePath($a); + return Filesystem::resolvePath($p); + } + + /** + * Return the path to a local version of the file + * we need this because we can't know if a file is stored local or not from + * outside the filestorage and for some purposes a local file is needed + * + * @param string $path + */ + public function getLocalFile($path): string|false { + $parent = substr($path, 0, strrpos($path, '/') ?: 0); + $path = $this->getAbsolutePath($path); + [$storage, $internalPath] = Filesystem::resolvePath($path); + if (Filesystem::isValidPath($parent) && $storage) { + return $storage->getLocalFile($internalPath); + } else { + return false; + } + } + + /** + * the following functions operate with arguments and return values identical + * to those of their PHP built-in equivalents. Mostly they are merely wrappers + * for \OC\Files\Storage\Storage via basicOperation(). + */ + public function mkdir($path) { + return $this->basicOperation('mkdir', $path, ['create', 'write']); + } + + /** + * remove mount point + * + * @param IMountPoint $mount + * @param string $path relative to data/ + */ + protected function removeMount($mount, $path): bool { + if ($mount instanceof MoveableMount) { + // cut of /user/files to get the relative path to data/user/files + $pathParts = explode('/', $path, 4); + $relPath = '/' . $pathParts[3]; + $this->lockFile($relPath, ILockingProvider::LOCK_SHARED, true); + \OC_Hook::emit( + Filesystem::CLASSNAME, 'umount', + [Filesystem::signal_param_path => $relPath] + ); + $this->changeLock($relPath, ILockingProvider::LOCK_EXCLUSIVE, true); + $result = $mount->removeMount(); + $this->changeLock($relPath, ILockingProvider::LOCK_SHARED, true); + if ($result) { + \OC_Hook::emit( + Filesystem::CLASSNAME, 'post_umount', + [Filesystem::signal_param_path => $relPath] + ); + } + $this->unlockFile($relPath, ILockingProvider::LOCK_SHARED, true); + return $result; + } else { + // do not allow deleting the storage's root / the mount point + // because for some storages it might delete the whole contents + // but isn't supposed to work that way + return false; + } + } + + public function disableCacheUpdate(): void { + $this->updaterEnabled = false; + } + + public function enableCacheUpdate(): void { + $this->updaterEnabled = true; + } + + protected function writeUpdate(Storage $storage, string $internalPath, ?int $time = null, ?int $sizeDifference = null): void { + if ($this->updaterEnabled) { + if (is_null($time)) { + $time = time(); + } + $storage->getUpdater()->update($internalPath, $time, $sizeDifference); + } + } + + protected function removeUpdate(Storage $storage, string $internalPath): void { + if ($this->updaterEnabled) { + $storage->getUpdater()->remove($internalPath); + } + } + + protected function renameUpdate(Storage $sourceStorage, Storage $targetStorage, string $sourceInternalPath, string $targetInternalPath): void { + if ($this->updaterEnabled) { + $targetStorage->getUpdater()->renameFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + } + + protected function copyUpdate(Storage $sourceStorage, Storage $targetStorage, string $sourceInternalPath, string $targetInternalPath): void { + if ($this->updaterEnabled) { + $targetStorage->getUpdater()->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + } + + /** + * @param string $path + * @return bool|mixed + */ + public function rmdir($path) { + $absolutePath = $this->getAbsolutePath($path); + $mount = Filesystem::getMountManager()->find($absolutePath); + if ($mount->getInternalPath($absolutePath) === '') { + return $this->removeMount($mount, $absolutePath); + } + if ($this->is_dir($path)) { + $result = $this->basicOperation('rmdir', $path, ['delete']); + } else { + $result = false; + } + + if (!$result && !$this->file_exists($path)) { //clear ghost files from the cache on delete + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($absolutePath); + $storage->getUpdater()->remove($internalPath); + } + return $result; + } + + /** + * @param string $path + * @return resource|false + */ + public function opendir($path) { + return $this->basicOperation('opendir', $path, ['read']); + } + + /** + * @param string $path + * @return bool|mixed + */ + public function is_dir($path) { + if ($path == '/') { + return true; + } + return $this->basicOperation('is_dir', $path); + } + + /** + * @param string $path + * @return bool|mixed + */ + public function is_file($path) { + if ($path == '/') { + return false; + } + return $this->basicOperation('is_file', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function stat($path) { + return $this->basicOperation('stat', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function filetype($path) { + return $this->basicOperation('filetype', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function filesize(string $path) { + return $this->basicOperation('filesize', $path); + } + + /** + * @param string $path + * @return bool|mixed + * @throws InvalidPathException + */ + public function readfile($path) { + $this->assertPathLength($path); + if (ob_get_level()) { + ob_end_clean(); + } + $handle = $this->fopen($path, 'rb'); + if ($handle) { + $chunkSize = 524288; // 512 kiB chunks + while (!feof($handle)) { + echo fread($handle, $chunkSize); + flush(); + $this->checkConnectionStatus(); + } + fclose($handle); + return $this->filesize($path); + } + return false; + } + + /** + * @param string $path + * @param int $from + * @param int $to + * @return bool|mixed + * @throws InvalidPathException + * @throws \OCP\Files\UnseekableException + */ + public function readfilePart($path, $from, $to) { + $this->assertPathLength($path); + if (ob_get_level()) { + ob_end_clean(); + } + $handle = $this->fopen($path, 'rb'); + if ($handle) { + $chunkSize = 524288; // 512 kiB chunks + $startReading = true; + + if ($from !== 0 && $from !== '0' && fseek($handle, $from) !== 0) { + // forward file handle via chunked fread because fseek seem to have failed + + $end = $from + 1; + while (!feof($handle) && ftell($handle) < $end && ftell($handle) !== $from) { + $len = $from - ftell($handle); + if ($len > $chunkSize) { + $len = $chunkSize; + } + $result = fread($handle, $len); + + if ($result === false) { + $startReading = false; + break; + } + } + } + + if ($startReading) { + $end = $to + 1; + while (!feof($handle) && ftell($handle) < $end) { + $len = $end - ftell($handle); + if ($len > $chunkSize) { + $len = $chunkSize; + } + echo fread($handle, $len); + flush(); + $this->checkConnectionStatus(); + } + return ftell($handle) - $from; + } + + throw new \OCP\Files\UnseekableException('fseek error'); + } + return false; + } + + private function checkConnectionStatus(): void { + $connectionStatus = \connection_status(); + if ($connectionStatus !== CONNECTION_NORMAL) { + throw new ConnectionLostException("Connection lost. Status: $connectionStatus"); + } + } + + /** + * @param string $path + * @return mixed + */ + public function isCreatable($path) { + return $this->basicOperation('isCreatable', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function isReadable($path) { + return $this->basicOperation('isReadable', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function isUpdatable($path) { + return $this->basicOperation('isUpdatable', $path); + } + + /** + * @param string $path + * @return bool|mixed + */ + public function isDeletable($path) { + $absolutePath = $this->getAbsolutePath($path); + $mount = Filesystem::getMountManager()->find($absolutePath); + if ($mount->getInternalPath($absolutePath) === '') { + return $mount instanceof MoveableMount; + } + return $this->basicOperation('isDeletable', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function isSharable($path) { + return $this->basicOperation('isSharable', $path); + } + + /** + * @param string $path + * @return bool|mixed + */ + public function file_exists($path) { + if ($path == '/') { + return true; + } + return $this->basicOperation('file_exists', $path); + } + + /** + * @param string $path + * @return mixed + */ + public function filemtime($path) { + return $this->basicOperation('filemtime', $path); + } + + /** + * @param string $path + * @param int|string $mtime + */ + public function touch($path, $mtime = null): bool { + if (!is_null($mtime) && !is_numeric($mtime)) { + $mtime = strtotime($mtime); + } + + $hooks = ['touch']; + + if (!$this->file_exists($path)) { + $hooks[] = 'create'; + $hooks[] = 'write'; + } + try { + $result = $this->basicOperation('touch', $path, $hooks, $mtime); + } catch (\Exception $e) { + $this->logger->info('Error while setting modified time', ['app' => 'core', 'exception' => $e]); + $result = false; + } + if (!$result) { + // If create file fails because of permissions on external storage like SMB folders, + // check file exists and return false if not. + if (!$this->file_exists($path)) { + return false; + } + if (is_null($mtime)) { + $mtime = time(); + } + //if native touch fails, we emulate it by changing the mtime in the cache + $this->putFileInfo($path, ['mtime' => floor($mtime)]); + } + return true; + } + + /** + * @param string $path + * @return string|false + * @throws LockedException + */ + public function file_get_contents($path) { + return $this->basicOperation('file_get_contents', $path, ['read']); + } + + protected function emit_file_hooks_pre(bool $exists, string $path, bool &$run): void { + if (!$exists) { + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_create, [ + Filesystem::signal_param_path => $this->getHookPath($path), + Filesystem::signal_param_run => &$run, + ]); + } else { + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_update, [ + Filesystem::signal_param_path => $this->getHookPath($path), + Filesystem::signal_param_run => &$run, + ]); + } + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_write, [ + Filesystem::signal_param_path => $this->getHookPath($path), + Filesystem::signal_param_run => &$run, + ]); + } + + protected function emit_file_hooks_post(bool $exists, string $path): void { + if (!$exists) { + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_create, [ + Filesystem::signal_param_path => $this->getHookPath($path), + ]); + } else { + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_update, [ + Filesystem::signal_param_path => $this->getHookPath($path), + ]); + } + \OC_Hook::emit(Filesystem::CLASSNAME, Filesystem::signal_post_write, [ + Filesystem::signal_param_path => $this->getHookPath($path), + ]); + } + + /** + * @param string $path + * @param string|resource $data + * @return bool|mixed + * @throws LockedException + */ + public function file_put_contents($path, $data) { + if (is_resource($data)) { //not having to deal with streams in file_put_contents makes life easier + $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); + if (Filesystem::isValidPath($path) + && !Filesystem::isFileBlacklisted($path) + ) { + $path = $this->getRelativePath($absolutePath); + if ($path === null) { + throw new InvalidPathException("Path $absolutePath is not in the expected root"); + } + + $this->lockFile($path, ILockingProvider::LOCK_SHARED); + + $exists = $this->file_exists($path); + if ($this->shouldEmitHooks($path)) { + $run = true; + $this->emit_file_hooks_pre($exists, $path, $run); + if (!$run) { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + return false; + } + } + + try { + $this->changeLock($path, ILockingProvider::LOCK_EXCLUSIVE); + } catch (\Exception $e) { + // Release the shared lock before throwing. + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + throw $e; + } + + /** @var Storage $storage */ + [$storage, $internalPath] = $this->resolvePath($path); + $target = $storage->fopen($internalPath, 'w'); + if ($target) { + [, $result] = Files::streamCopy($data, $target, true); + fclose($target); + fclose($data); + + $this->writeUpdate($storage, $internalPath); + + $this->changeLock($path, ILockingProvider::LOCK_SHARED); + + if ($this->shouldEmitHooks($path) && $result !== false) { + $this->emit_file_hooks_post($exists, $path); + } + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + return $result; + } else { + $this->unlockFile($path, ILockingProvider::LOCK_EXCLUSIVE); + return false; + } + } else { + return false; + } + } else { + $hooks = $this->file_exists($path) ? ['update', 'write'] : ['create', 'write']; + return $this->basicOperation('file_put_contents', $path, $hooks, $data); + } + } + + /** + * @param string $path + * @return bool|mixed + */ + public function unlink($path) { + if ($path === '' || $path === '/') { + // do not allow deleting the root + return false; + } + $postFix = (substr($path, -1) === '/') ? '/' : ''; + $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); + $mount = Filesystem::getMountManager()->find($absolutePath . $postFix); + if ($mount->getInternalPath($absolutePath) === '') { + return $this->removeMount($mount, $absolutePath); + } + if ($this->is_dir($path)) { + $result = $this->basicOperation('rmdir', $path, ['delete']); + } else { + $result = $this->basicOperation('unlink', $path, ['delete']); + } + if (!$result && !$this->file_exists($path)) { //clear ghost files from the cache on delete + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($absolutePath); + $storage->getUpdater()->remove($internalPath); + return true; + } else { + return $result; + } + } + + /** + * @param string $directory + * @return bool|mixed + */ + public function deleteAll($directory) { + return $this->rmdir($directory); + } + + /** + * Rename/move a file or folder from the source path to target path. + * + * @param string $source source path + * @param string $target target path + * @param array $options + * + * @return bool|mixed + * @throws LockedException + */ + public function rename($source, $target, array $options = []) { + $checkSubMounts = $options['checkSubMounts'] ?? true; + + $absolutePath1 = Filesystem::normalizePath($this->getAbsolutePath($source)); + $absolutePath2 = Filesystem::normalizePath($this->getAbsolutePath($target)); + + if (str_starts_with($absolutePath2, $absolutePath1 . '/')) { + throw new ForbiddenException('Moving a folder into a child folder is forbidden', false); + } + + /** @var IMountManager $mountManager */ + $mountManager = \OC::$server->get(IMountManager::class); + + $targetParts = explode('/', $absolutePath2); + $targetUser = $targetParts[1] ?? null; + $result = false; + if ( + Filesystem::isValidPath($target) + && Filesystem::isValidPath($source) + && !Filesystem::isFileBlacklisted($target) + ) { + $source = $this->getRelativePath($absolutePath1); + $target = $this->getRelativePath($absolutePath2); + $exists = $this->file_exists($target); + + if ($source == null || $target == null) { + return false; + } + + try { + $this->verifyPath(dirname($target), basename($target)); + } catch (InvalidPathException) { + return false; + } + + $this->lockFile($source, ILockingProvider::LOCK_SHARED, true); + try { + $this->lockFile($target, ILockingProvider::LOCK_SHARED, true); + + $run = true; + if ($this->shouldEmitHooks($source) && (Cache\Scanner::isPartialFile($source) && !Cache\Scanner::isPartialFile($target))) { + // if it was a rename from a part file to a regular file it was a write and not a rename operation + $this->emit_file_hooks_pre($exists, $target, $run); + } elseif ($this->shouldEmitHooks($source)) { + $sourcePath = $this->getHookPath($source); + $targetPath = $this->getHookPath($target); + if ($sourcePath !== null && $targetPath !== null) { + \OC_Hook::emit( + Filesystem::CLASSNAME, Filesystem::signal_rename, + [ + Filesystem::signal_param_oldpath => $sourcePath, + Filesystem::signal_param_newpath => $targetPath, + Filesystem::signal_param_run => &$run + ] + ); + } + } + if ($run) { + $manager = Filesystem::getMountManager(); + $mount1 = $this->getMount($source); + $mount2 = $this->getMount($target); + $storage1 = $mount1->getStorage(); + $storage2 = $mount2->getStorage(); + $internalPath1 = $mount1->getInternalPath($absolutePath1); + $internalPath2 = $mount2->getInternalPath($absolutePath2); + + $this->changeLock($source, ILockingProvider::LOCK_EXCLUSIVE, true); + try { + $this->changeLock($target, ILockingProvider::LOCK_EXCLUSIVE, true); + + if ($checkSubMounts) { + $movedMounts = $mountManager->findIn($this->getAbsolutePath($source)); + } else { + $movedMounts = []; + } + + if ($internalPath1 === '') { + $sourceParentMount = $this->getMount(dirname($source)); + $movedMounts[] = $mount1; + $this->validateMountMove($movedMounts, $sourceParentMount, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + /** + * @var \OC\Files\Mount\MountPoint | \OC\Files\Mount\MoveableMount $mount1 + */ + $sourceMountPoint = $mount1->getMountPoint(); + $result = $mount1->moveMount($absolutePath2); + $manager->moveMount($sourceMountPoint, $mount1->getMountPoint()); + + // moving a file/folder within the same mount point + } elseif ($storage1 === $storage2) { + if (count($movedMounts) > 0) { + $this->validateMountMove($movedMounts, $mount1, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + } + if ($storage1) { + $result = $storage1->rename($internalPath1, $internalPath2); + } else { + $result = false; + } + // moving a file/folder between storages (from $storage1 to $storage2) + } else { + if (count($movedMounts) > 0) { + $this->validateMountMove($movedMounts, $mount1, $mount2, !$this->targetIsNotShared($targetUser, $absolutePath2)); + } + $result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2); + } + + if ((Cache\Scanner::isPartialFile($source) && !Cache\Scanner::isPartialFile($target)) && $result !== false) { + // if it was a rename from a part file to a regular file it was a write and not a rename operation + $this->writeUpdate($storage2, $internalPath2); + } elseif ($result) { + if ($internalPath1 !== '') { // don't do a cache update for moved mounts + $this->renameUpdate($storage1, $storage2, $internalPath1, $internalPath2); + } + } + } catch (\Exception $e) { + throw $e; + } finally { + $this->changeLock($source, ILockingProvider::LOCK_SHARED, true); + $this->changeLock($target, ILockingProvider::LOCK_SHARED, true); + } + + if ((Cache\Scanner::isPartialFile($source) && !Cache\Scanner::isPartialFile($target)) && $result !== false) { + if ($this->shouldEmitHooks()) { + $this->emit_file_hooks_post($exists, $target); + } + } elseif ($result) { + if ($this->shouldEmitHooks($source) && $this->shouldEmitHooks($target)) { + $sourcePath = $this->getHookPath($source); + $targetPath = $this->getHookPath($target); + if ($sourcePath !== null && $targetPath !== null) { + \OC_Hook::emit( + Filesystem::CLASSNAME, + Filesystem::signal_post_rename, + [ + Filesystem::signal_param_oldpath => $sourcePath, + Filesystem::signal_param_newpath => $targetPath, + ] + ); + } + } + } + } + } catch (\Exception $e) { + throw $e; + } finally { + $this->unlockFile($source, ILockingProvider::LOCK_SHARED, true); + $this->unlockFile($target, ILockingProvider::LOCK_SHARED, true); + } + } + return $result; + } + + /** + * @throws ForbiddenException + */ + private function validateMountMove(array $mounts, IMountPoint $sourceMount, IMountPoint $targetMount, bool $targetIsShared): void { + $targetPath = $this->getRelativePath($targetMount->getMountPoint()); + if ($targetPath) { + $targetPath = trim($targetPath, '/'); + } else { + $targetPath = $targetMount->getMountPoint(); + } + + $l = \OC::$server->get(IFactory::class)->get('files'); + foreach ($mounts as $mount) { + $sourcePath = $this->getRelativePath($mount->getMountPoint()); + if ($sourcePath) { + $sourcePath = trim($sourcePath, '/'); + } else { + $sourcePath = $mount->getMountPoint(); + } + + if (!$mount instanceof MoveableMount) { + throw new ForbiddenException($l->t('Storage %s cannot be moved', [$sourcePath]), false); + } + + if ($targetIsShared) { + if ($sourceMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a share (%s) into a shared folder is not allowed', [$sourcePath]), false); + } else { + throw new ForbiddenException($l->t('Moving a storage (%s) into a shared folder is not allowed', [$sourcePath]), false); + } + } + + if ($sourceMount !== $targetMount) { + if ($sourceMount instanceof SharedMount) { + if ($targetMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a share (%s) into another share (%s) is not allowed', [$sourcePath, $targetPath]), false); + } else { + throw new ForbiddenException($l->t('Moving a share (%s) into another storage (%s) is not allowed', [$sourcePath, $targetPath]), false); + } + } else { + if ($targetMount instanceof SharedMount) { + throw new ForbiddenException($l->t('Moving a storage (%s) into a share (%s) is not allowed', [$sourcePath, $targetPath]), false); + } else { + throw new ForbiddenException($l->t('Moving a storage (%s) into another storage (%s) is not allowed', [$sourcePath, $targetPath]), false); + } + } + } + } + } + + /** + * Copy a file/folder from the source path to target path + * + * @param string $source source path + * @param string $target target path + * @param bool $preserveMtime whether to preserve mtime on the copy + * + * @return bool|mixed + */ + public function copy($source, $target, $preserveMtime = false) { + $absolutePath1 = Filesystem::normalizePath($this->getAbsolutePath($source)); + $absolutePath2 = Filesystem::normalizePath($this->getAbsolutePath($target)); + $result = false; + if ( + Filesystem::isValidPath($target) + && Filesystem::isValidPath($source) + && !Filesystem::isFileBlacklisted($target) + ) { + $source = $this->getRelativePath($absolutePath1); + $target = $this->getRelativePath($absolutePath2); + + if ($source == null || $target == null) { + return false; + } + $run = true; + + $this->lockFile($target, ILockingProvider::LOCK_SHARED); + $this->lockFile($source, ILockingProvider::LOCK_SHARED); + $lockTypePath1 = ILockingProvider::LOCK_SHARED; + $lockTypePath2 = ILockingProvider::LOCK_SHARED; + + try { + $exists = $this->file_exists($target); + if ($this->shouldEmitHooks($target)) { + \OC_Hook::emit( + Filesystem::CLASSNAME, + Filesystem::signal_copy, + [ + Filesystem::signal_param_oldpath => $this->getHookPath($source), + Filesystem::signal_param_newpath => $this->getHookPath($target), + Filesystem::signal_param_run => &$run + ] + ); + $this->emit_file_hooks_pre($exists, $target, $run); + } + if ($run) { + $mount1 = $this->getMount($source); + $mount2 = $this->getMount($target); + $storage1 = $mount1->getStorage(); + $internalPath1 = $mount1->getInternalPath($absolutePath1); + $storage2 = $mount2->getStorage(); + $internalPath2 = $mount2->getInternalPath($absolutePath2); + + $this->changeLock($target, ILockingProvider::LOCK_EXCLUSIVE); + $lockTypePath2 = ILockingProvider::LOCK_EXCLUSIVE; + + if ($mount1->getMountPoint() == $mount2->getMountPoint()) { + if ($storage1) { + $result = $storage1->copy($internalPath1, $internalPath2); + } else { + $result = false; + } + } else { + $result = $storage2->copyFromStorage($storage1, $internalPath1, $internalPath2); + } + + if ($result) { + $this->copyUpdate($storage1, $storage2, $internalPath1, $internalPath2); + } + + $this->changeLock($target, ILockingProvider::LOCK_SHARED); + $lockTypePath2 = ILockingProvider::LOCK_SHARED; + + if ($this->shouldEmitHooks($target) && $result !== false) { + \OC_Hook::emit( + Filesystem::CLASSNAME, + Filesystem::signal_post_copy, + [ + Filesystem::signal_param_oldpath => $this->getHookPath($source), + Filesystem::signal_param_newpath => $this->getHookPath($target) + ] + ); + $this->emit_file_hooks_post($exists, $target); + } + } + } catch (\Exception $e) { + $this->unlockFile($target, $lockTypePath2); + $this->unlockFile($source, $lockTypePath1); + throw $e; + } + + $this->unlockFile($target, $lockTypePath2); + $this->unlockFile($source, $lockTypePath1); + } + return $result; + } + + /** + * @param string $path + * @param string $mode 'r' or 'w' + * @return resource|false + * @throws LockedException + */ + public function fopen($path, $mode) { + $mode = str_replace('b', '', $mode); // the binary flag is a windows only feature which we do not support + $hooks = []; + switch ($mode) { + case 'r': + $hooks[] = 'read'; + break; + case 'r+': + case 'w+': + case 'x+': + case 'a+': + $hooks[] = 'read'; + $hooks[] = 'write'; + break; + case 'w': + case 'x': + case 'a': + $hooks[] = 'write'; + break; + default: + $this->logger->error('invalid mode (' . $mode . ') for ' . $path, ['app' => 'core']); + } + + if ($mode !== 'r' && $mode !== 'w') { + $this->logger->info('Trying to open a file with a mode other than "r" or "w" can cause severe performance issues with some backends', ['app' => 'core']); + } + + $handle = $this->basicOperation('fopen', $path, $hooks, $mode); + if (!is_resource($handle) && $mode === 'r') { + // trying to read a file that isn't on disk, check if the cache is out of sync and rescan if needed + $mount = $this->getMount($path); + $internalPath = $mount->getInternalPath($this->getAbsolutePath($path)); + $storage = $mount->getStorage(); + if ($storage->getCache()->inCache($internalPath) && !$storage->file_exists($path)) { + $this->writeUpdate($storage, $internalPath); + } + } + return $handle; + } + + /** + * @param string $path + * @throws InvalidPathException + */ + public function toTmpFile($path): string|false { + $this->assertPathLength($path); + if (Filesystem::isValidPath($path)) { + $source = $this->fopen($path, 'r'); + if ($source) { + $extension = pathinfo($path, PATHINFO_EXTENSION); + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($extension); + file_put_contents($tmpFile, $source); + return $tmpFile; + } else { + return false; + } + } else { + return false; + } + } + + /** + * @param string $tmpFile + * @param string $path + * @return bool|mixed + * @throws InvalidPathException + */ + public function fromTmpFile($tmpFile, $path) { + $this->assertPathLength($path); + if (Filesystem::isValidPath($path)) { + // Get directory that the file is going into + $filePath = dirname($path); + + // Create the directories if any + if (!$this->file_exists($filePath)) { + $result = $this->createParentDirectories($filePath); + if ($result === false) { + return false; + } + } + + $source = fopen($tmpFile, 'r'); + if ($source) { + $result = $this->file_put_contents($path, $source); + /** + * $this->file_put_contents() might have already closed + * the resource, so we check it, before trying to close it + * to avoid messages in the error log. + * @psalm-suppress RedundantCondition false-positive + */ + if (is_resource($source)) { + fclose($source); + } + unlink($tmpFile); + return $result; + } else { + return false; + } + } else { + return false; + } + } + + + /** + * @param string $path + * @return mixed + * @throws InvalidPathException + */ + public function getMimeType($path) { + $this->assertPathLength($path); + return $this->basicOperation('getMimeType', $path); + } + + /** + * @param string $type + * @param string $path + * @param bool $raw + */ + public function hash($type, $path, $raw = false): string|bool { + $postFix = (substr($path, -1) === '/') ? '/' : ''; + $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); + if (Filesystem::isValidPath($path)) { + $path = $this->getRelativePath($absolutePath); + if ($path == null) { + return false; + } + if ($this->shouldEmitHooks($path)) { + \OC_Hook::emit( + Filesystem::CLASSNAME, + Filesystem::signal_read, + [Filesystem::signal_param_path => $this->getHookPath($path)] + ); + } + /** @var Storage|null $storage */ + [$storage, $internalPath] = Filesystem::resolvePath($absolutePath . $postFix); + if ($storage) { + return $storage->hash($type, $internalPath, $raw); + } + } + return false; + } + + /** + * @param string $path + * @return mixed + * @throws InvalidPathException + */ + public function free_space($path = '/') { + $this->assertPathLength($path); + $result = $this->basicOperation('free_space', $path); + if ($result === null) { + throw new InvalidPathException(); + } + return $result; + } + + /** + * abstraction layer for basic filesystem functions: wrapper for \OC\Files\Storage\Storage + * + * @param mixed $extraParam (optional) + * @return mixed + * @throws LockedException + * + * This method takes requests for basic filesystem functions (e.g. reading & writing + * files), processes hooks and proxies, sanitises paths, and finally passes them on to + * \OC\Files\Storage\Storage for delegation to a storage backend for execution + */ + private function basicOperation(string $operation, string $path, array $hooks = [], $extraParam = null) { + $postFix = (substr($path, -1) === '/') ? '/' : ''; + $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); + if (Filesystem::isValidPath($path) + && !Filesystem::isFileBlacklisted($path) + ) { + $path = $this->getRelativePath($absolutePath); + if ($path == null) { + return false; + } + + if (in_array('write', $hooks) || in_array('delete', $hooks) || in_array('read', $hooks)) { + // always a shared lock during pre-hooks so the hook can read the file + $this->lockFile($path, ILockingProvider::LOCK_SHARED); + } + + $run = $this->runHooks($hooks, $path); + [$storage, $internalPath] = Filesystem::resolvePath($absolutePath . $postFix); + if ($run && $storage) { + /** @var Storage $storage */ + if (in_array('write', $hooks) || in_array('delete', $hooks)) { + try { + $this->changeLock($path, ILockingProvider::LOCK_EXCLUSIVE); + } catch (LockedException $e) { + // release the shared lock we acquired before quitting + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + throw $e; + } + } + try { + if (!is_null($extraParam)) { + $result = $storage->$operation($internalPath, $extraParam); + } else { + $result = $storage->$operation($internalPath); + } + } catch (\Exception $e) { + if (in_array('write', $hooks) || in_array('delete', $hooks)) { + $this->unlockFile($path, ILockingProvider::LOCK_EXCLUSIVE); + } elseif (in_array('read', $hooks)) { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + } + throw $e; + } + + if ($result !== false && in_array('delete', $hooks)) { + $this->removeUpdate($storage, $internalPath); + } + if ($result !== false && in_array('write', $hooks, true) && $operation !== 'fopen' && $operation !== 'touch') { + $isCreateOperation = $operation === 'mkdir' || ($operation === 'file_put_contents' && in_array('create', $hooks, true)); + $sizeDifference = $operation === 'mkdir' ? 0 : $result; + $this->writeUpdate($storage, $internalPath, null, $isCreateOperation ? $sizeDifference : null); + } + if ($result !== false && in_array('touch', $hooks)) { + $this->writeUpdate($storage, $internalPath, $extraParam, 0); + } + + if ((in_array('write', $hooks) || in_array('delete', $hooks)) && ($operation !== 'fopen' || $result === false)) { + $this->changeLock($path, ILockingProvider::LOCK_SHARED); + } + + $unlockLater = false; + if ($this->lockingEnabled && $operation === 'fopen' && is_resource($result)) { + $unlockLater = true; + // make sure our unlocking callback will still be called if connection is aborted + ignore_user_abort(true); + $result = CallbackWrapper::wrap($result, null, null, function () use ($hooks, $path) { + if (in_array('write', $hooks)) { + $this->unlockFile($path, ILockingProvider::LOCK_EXCLUSIVE); + } elseif (in_array('read', $hooks)) { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + } + }); + } + + if ($this->shouldEmitHooks($path) && $result !== false) { + if ($operation != 'fopen') { //no post hooks for fopen, the file stream is still open + $this->runHooks($hooks, $path, true); + } + } + + if (!$unlockLater + && (in_array('write', $hooks) || in_array('delete', $hooks) || in_array('read', $hooks)) + ) { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + } + return $result; + } else { + $this->unlockFile($path, ILockingProvider::LOCK_SHARED); + } + } + return null; + } + + /** + * get the path relative to the default root for hook usage + * + * @param string $path + * @return ?string + */ + private function getHookPath($path): ?string { + $view = Filesystem::getView(); + if (!$view) { + return $path; + } + return $view->getRelativePath($this->getAbsolutePath($path)); + } + + private function shouldEmitHooks(string $path = ''): bool { + if ($path && Cache\Scanner::isPartialFile($path)) { + return false; + } + if (!Filesystem::$loaded) { + return false; + } + $defaultRoot = Filesystem::getRoot(); + if ($defaultRoot === null) { + return false; + } + if ($this->fakeRoot === $defaultRoot) { + return true; + } + $fullPath = $this->getAbsolutePath($path); + + if ($fullPath === $defaultRoot) { + return true; + } + + return (strlen($fullPath) > strlen($defaultRoot)) && (substr($fullPath, 0, strlen($defaultRoot) + 1) === $defaultRoot . '/'); + } + + /** + * @param string[] $hooks + * @param string $path + * @param bool $post + * @return bool + */ + private function runHooks($hooks, $path, $post = false) { + $relativePath = $path; + $path = $this->getHookPath($path); + $prefix = $post ? 'post_' : ''; + $run = true; + if ($this->shouldEmitHooks($relativePath)) { + foreach ($hooks as $hook) { + if ($hook != 'read') { + \OC_Hook::emit( + Filesystem::CLASSNAME, + $prefix . $hook, + [ + Filesystem::signal_param_run => &$run, + Filesystem::signal_param_path => $path + ] + ); + } elseif (!$post) { + \OC_Hook::emit( + Filesystem::CLASSNAME, + $prefix . $hook, + [ + Filesystem::signal_param_path => $path + ] + ); + } + } + } + return $run; + } + + /** + * check if a file or folder has been updated since $time + * + * @param string $path + * @param int $time + * @return bool + */ + public function hasUpdated($path, $time) { + return $this->basicOperation('hasUpdated', $path, [], $time); + } + + /** + * @param string $ownerId + * @return IUser + */ + private function getUserObjectForOwner(string $ownerId) { + return new LazyUser($ownerId, $this->userManager); + } + + /** + * Get file info from cache + * + * If the file is not in cached it will be scanned + * If the file has changed on storage the cache will be updated + * + * @param Storage $storage + * @param string $internalPath + * @param string $relativePath + * @return ICacheEntry|bool + */ + private function getCacheEntry($storage, $internalPath, $relativePath) { + $cache = $storage->getCache($internalPath); + $data = $cache->get($internalPath); + $watcher = $storage->getWatcher($internalPath); + + try { + // if the file is not in the cache or needs to be updated, trigger the scanner and reload the data + if (!$data || (isset($data['size']) && $data['size'] === -1)) { + if (!$storage->file_exists($internalPath)) { + return false; + } + // don't need to get a lock here since the scanner does it's own locking + $scanner = $storage->getScanner($internalPath); + $scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW); + $data = $cache->get($internalPath); + } elseif (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) { + $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); + $watcher->update($internalPath, $data); + $storage->getPropagator()->propagateChange($internalPath, time()); + $data = $cache->get($internalPath); + $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); + } + } catch (LockedException $e) { + // if the file is locked we just use the old cache info + } + + return $data; + } + + /** + * get the filesystem info + * + * @param string $path + * @param bool|string $includeMountPoints true to add mountpoint sizes, + * 'ext' to add only ext storage mount point sizes. Defaults to true. + * @return \OC\Files\FileInfo|false False if file does not exist + */ + public function getFileInfo($path, $includeMountPoints = true) { + $this->assertPathLength($path); + if (!Filesystem::isValidPath($path)) { + return false; + } + $relativePath = $path; + $path = Filesystem::normalizePath($this->fakeRoot . '/' . $path); + + $mount = Filesystem::getMountManager()->find($path); + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($path); + if ($storage) { + $data = $this->getCacheEntry($storage, $internalPath, $relativePath); + + if (!$data instanceof ICacheEntry) { + if (Cache\Scanner::isPartialFile($relativePath)) { + return $this->getPartFileInfo($relativePath); + } + + return false; + } + + if ($mount instanceof MoveableMount && $internalPath === '') { + $data['permissions'] |= \OCP\Constants::PERMISSION_DELETE; + } + if ($internalPath === '' && $data['name']) { + $data['name'] = basename($path); + } + + $ownerId = $storage->getOwner($internalPath); + $owner = null; + if ($ownerId !== false) { + // ownerId might be null if files are accessed with an access token without file system access + $owner = $this->getUserObjectForOwner($ownerId); + } + $info = new FileInfo($path, $storage, $internalPath, $data, $mount, $owner); + + if (isset($data['fileid'])) { + if ($includeMountPoints && $data['mimetype'] === 'httpd/unix-directory') { + //add the sizes of other mount points to the folder + $extOnly = ($includeMountPoints === 'ext'); + $this->addSubMounts($info, $extOnly); + } + } + + return $info; + } else { + $this->logger->warning('Storage not valid for mountpoint: ' . $mount->getMountPoint(), ['app' => 'core']); + } + + return false; + } + + /** + * Extend a FileInfo that was previously requested with `$includeMountPoints = false` to include the sub mounts + */ + public function addSubMounts(FileInfo $info, $extOnly = false): void { + $mounts = Filesystem::getMountManager()->findIn($info->getPath()); + $info->setSubMounts(array_filter($mounts, function (IMountPoint $mount) use ($extOnly) { + return !($extOnly && $mount instanceof SharedMount); + })); + } + + /** + * get the content of a directory + * + * @param string $directory path under datadirectory + * @param string $mimetype_filter limit returned content to this mimetype or mimepart + * @return FileInfo[] + */ + public function getDirectoryContent($directory, $mimetype_filter = '', ?\OCP\Files\FileInfo $directoryInfo = null) { + $this->assertPathLength($directory); + if (!Filesystem::isValidPath($directory)) { + return []; + } + + $path = $this->getAbsolutePath($directory); + $path = Filesystem::normalizePath($path); + $mount = $this->getMount($directory); + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($path); + if (!$storage) { + return []; + } + + $cache = $storage->getCache($internalPath); + $user = \OC_User::getUser(); + + if (!$directoryInfo) { + $data = $this->getCacheEntry($storage, $internalPath, $directory); + if (!$data instanceof ICacheEntry || !isset($data['fileid'])) { + return []; + } + } else { + $data = $directoryInfo; + } + + if (!($data->getPermissions() & Constants::PERMISSION_READ)) { + return []; + } + + $folderId = $data->getId(); + $contents = $cache->getFolderContentsById($folderId); //TODO: mimetype_filter + + $sharingDisabled = \OCP\Util::isSharingDisabledForUser(); + + $fileNames = array_map(function (ICacheEntry $content) { + return $content->getName(); + }, $contents); + /** + * @var \OC\Files\FileInfo[] $fileInfos + */ + $fileInfos = array_map(function (ICacheEntry $content) use ($path, $storage, $mount, $sharingDisabled) { + if ($sharingDisabled) { + $content['permissions'] = $content['permissions'] & ~\OCP\Constants::PERMISSION_SHARE; + } + $ownerId = $storage->getOwner($content['path']); + if ($ownerId !== false) { + $owner = $this->getUserObjectForOwner($ownerId); + } else { + $owner = null; + } + return new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner); + }, $contents); + $files = array_combine($fileNames, $fileInfos); + + //add a folder for any mountpoint in this directory and add the sizes of other mountpoints to the folders + $mounts = Filesystem::getMountManager()->findIn($path); + + // make sure nested mounts are sorted after their parent mounts + // otherwise doesn't propagate the etag across storage boundaries correctly + usort($mounts, function (IMountPoint $a, IMountPoint $b) { + return $a->getMountPoint() <=> $b->getMountPoint(); + }); + + $dirLength = strlen($path); + foreach ($mounts as $mount) { + $mountPoint = $mount->getMountPoint(); + $subStorage = $mount->getStorage(); + if ($subStorage) { + $subCache = $subStorage->getCache(''); + + $rootEntry = $subCache->get(''); + if (!$rootEntry) { + $subScanner = $subStorage->getScanner(); + try { + $subScanner->scanFile(''); + } catch (\OCP\Files\StorageNotAvailableException $e) { + continue; + } catch (\OCP\Files\StorageInvalidException $e) { + continue; + } catch (\Exception $e) { + // sometimes when the storage is not available it can be any exception + $this->logger->error('Exception while scanning storage "' . $subStorage->getId() . '"', [ + 'exception' => $e, + 'app' => 'core', + ]); + continue; + } + $rootEntry = $subCache->get(''); + } + + if ($rootEntry && ($rootEntry->getPermissions() & Constants::PERMISSION_READ)) { + $relativePath = trim(substr($mountPoint, $dirLength), '/'); + if ($pos = strpos($relativePath, '/')) { + //mountpoint inside subfolder add size to the correct folder + $entryName = substr($relativePath, 0, $pos); + + // Create parent folders if the mountpoint is inside a subfolder that doesn't exist yet + if (!isset($files[$entryName])) { + try { + [$storage, ] = $this->resolvePath($path . '/' . $entryName); + // make sure we can create the mountpoint folder, even if the user has a quota of 0 + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(false); + } + + if ($this->mkdir($path . '/' . $entryName) !== false) { + $info = $this->getFileInfo($path . '/' . $entryName); + if ($info !== false) { + $files[$entryName] = $info; + } + } + + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(true); + } + } catch (\Exception $e) { + // Creating the parent folder might not be possible, for example due to a lack of permissions. + $this->logger->debug('Failed to create non-existent parent', ['exception' => $e, 'path' => $path . '/' . $entryName]); + } + } + + if (isset($files[$entryName])) { + $files[$entryName]->addSubEntry($rootEntry, $mountPoint); + } + } else { //mountpoint in this folder, add an entry for it + $rootEntry['name'] = $relativePath; + $rootEntry['type'] = $rootEntry['mimetype'] === 'httpd/unix-directory' ? 'dir' : 'file'; + $permissions = $rootEntry['permissions']; + // do not allow renaming/deleting the mount point if they are not shared files/folders + // for shared files/folders we use the permissions given by the owner + if ($mount instanceof MoveableMount) { + $rootEntry['permissions'] = $permissions | \OCP\Constants::PERMISSION_UPDATE | \OCP\Constants::PERMISSION_DELETE; + } else { + $rootEntry['permissions'] = $permissions & (\OCP\Constants::PERMISSION_ALL - (\OCP\Constants::PERMISSION_UPDATE | \OCP\Constants::PERMISSION_DELETE)); + } + + $rootEntry['path'] = substr(Filesystem::normalizePath($path . '/' . $rootEntry['name']), strlen($user) + 2); // full path without /$user/ + + // if sharing was disabled for the user we remove the share permissions + if ($sharingDisabled) { + $rootEntry['permissions'] = $rootEntry['permissions'] & ~\OCP\Constants::PERMISSION_SHARE; + } + + $ownerId = $subStorage->getOwner(''); + if ($ownerId !== false) { + $owner = $this->getUserObjectForOwner($ownerId); + } else { + $owner = null; + } + $files[$rootEntry->getName()] = new FileInfo($path . '/' . $rootEntry['name'], $subStorage, '', $rootEntry, $mount, $owner); + } + } + } + } + + if ($mimetype_filter) { + $files = array_filter($files, function (FileInfo $file) use ($mimetype_filter) { + if (strpos($mimetype_filter, '/')) { + return $file->getMimetype() === $mimetype_filter; + } else { + return $file->getMimePart() === $mimetype_filter; + } + }); + } + + return array_values($files); + } + + /** + * change file metadata + * + * @param string $path + * @param array|\OCP\Files\FileInfo $data + * @return int + * + * returns the fileid of the updated file + */ + public function putFileInfo($path, $data) { + $this->assertPathLength($path); + if ($data instanceof FileInfo) { + $data = $data->getData(); + } + $path = Filesystem::normalizePath($this->fakeRoot . '/' . $path); + /** + * @var Storage $storage + * @var string $internalPath + */ + [$storage, $internalPath] = Filesystem::resolvePath($path); + if ($storage) { + $cache = $storage->getCache($path); + + if (!$cache->inCache($internalPath)) { + $scanner = $storage->getScanner($internalPath); + $scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW); + } + + return $cache->put($internalPath, $data); + } else { + return -1; + } + } + + /** + * search for files with the name matching $query + * + * @param string $query + * @return FileInfo[] + */ + public function search($query) { + return $this->searchCommon('search', ['%' . $query . '%']); + } + + /** + * search for files with the name matching $query + * + * @param string $query + * @return FileInfo[] + */ + public function searchRaw($query) { + return $this->searchCommon('search', [$query]); + } + + /** + * search for files by mimetype + * + * @param string $mimetype + * @return FileInfo[] + */ + public function searchByMime($mimetype) { + return $this->searchCommon('searchByMime', [$mimetype]); + } + + /** + * search for files by tag + * + * @param string|int $tag name or tag id + * @param string $userId owner of the tags + * @return FileInfo[] + */ + public function searchByTag($tag, $userId) { + return $this->searchCommon('searchByTag', [$tag, $userId]); + } + + /** + * @param string $method cache method + * @param array $args + * @return FileInfo[] + */ + private function searchCommon($method, $args) { + $files = []; + $rootLength = strlen($this->fakeRoot); + + $mount = $this->getMount(''); + $mountPoint = $mount->getMountPoint(); + $storage = $mount->getStorage(); + $userManager = \OC::$server->getUserManager(); + if ($storage) { + $cache = $storage->getCache(''); + + $results = call_user_func_array([$cache, $method], $args); + foreach ($results as $result) { + if (substr($mountPoint . $result['path'], 0, $rootLength + 1) === $this->fakeRoot . '/') { + $internalPath = $result['path']; + $path = $mountPoint . $result['path']; + $result['path'] = substr($mountPoint . $result['path'], $rootLength); + $ownerId = $storage->getOwner($internalPath); + if ($ownerId !== false) { + $owner = $userManager->get($ownerId); + } else { + $owner = null; + } + $files[] = new FileInfo($path, $storage, $internalPath, $result, $mount, $owner); + } + } + + $mounts = Filesystem::getMountManager()->findIn($this->fakeRoot); + foreach ($mounts as $mount) { + $mountPoint = $mount->getMountPoint(); + $storage = $mount->getStorage(); + if ($storage) { + $cache = $storage->getCache(''); + + $relativeMountPoint = substr($mountPoint, $rootLength); + $results = call_user_func_array([$cache, $method], $args); + if ($results) { + foreach ($results as $result) { + $internalPath = $result['path']; + $result['path'] = rtrim($relativeMountPoint . $result['path'], '/'); + $path = rtrim($mountPoint . $internalPath, '/'); + $ownerId = $storage->getOwner($internalPath); + if ($ownerId !== false) { + $owner = $userManager->get($ownerId); + } else { + $owner = null; + } + $files[] = new FileInfo($path, $storage, $internalPath, $result, $mount, $owner); + } + } + } + } + } + return $files; + } + + /** + * Get the owner for a file or folder + * + * @throws NotFoundException + */ + public function getOwner(string $path): string { + $info = $this->getFileInfo($path); + if (!$info) { + throw new NotFoundException($path . ' not found while trying to get owner'); + } + + if ($info->getOwner() === null) { + throw new NotFoundException($path . ' has no owner'); + } + + return $info->getOwner()->getUID(); + } + + /** + * get the ETag for a file or folder + * + * @param string $path + * @return string|false + */ + public function getETag($path) { + [$storage, $internalPath] = $this->resolvePath($path); + if ($storage) { + return $storage->getETag($internalPath); + } else { + return false; + } + } + + /** + * Get the path of a file by id, relative to the view + * + * Note that the resulting path is not guaranteed to be unique for the id, multiple paths can point to the same file + * + * @param int $id + * @param int|null $storageId + * @return string + * @throws NotFoundException + */ + public function getPath($id, ?int $storageId = null): string { + $id = (int)$id; + $rootFolder = Server::get(Files\IRootFolder::class); + + $node = $rootFolder->getFirstNodeByIdInPath($id, $this->getRoot()); + if ($node) { + if ($storageId === null || $storageId === $node->getStorage()->getCache()->getNumericStorageId()) { + return $this->getRelativePath($node->getPath()) ?? ''; + } + } else { + throw new NotFoundException(sprintf('File with id "%s" has not been found.', $id)); + } + + foreach ($rootFolder->getByIdInPath($id, $this->getRoot()) as $node) { + if ($storageId === $node->getStorage()->getCache()->getNumericStorageId()) { + return $this->getRelativePath($node->getPath()) ?? ''; + } + } + + throw new NotFoundException(sprintf('File with id "%s" has not been found.', $id)); + } + + /** + * @param string $path + * @throws InvalidPathException + */ + private function assertPathLength($path): void { + $maxLen = min(PHP_MAXPATHLEN, 4000); + // Check for the string length - performed using isset() instead of strlen() + // because isset() is about 5x-40x faster. + if (isset($path[$maxLen])) { + $pathLen = strlen($path); + throw new InvalidPathException("Path length($pathLen) exceeds max path length($maxLen): $path"); + } + } + + /** + * check if it is allowed to move a mount point to a given target. + * It is not allowed to move a mount point into a different mount point or + * into an already shared folder + */ + private function targetIsNotShared(string $user, string $targetPath): bool { + $providers = [ + IShare::TYPE_USER, + IShare::TYPE_GROUP, + IShare::TYPE_EMAIL, + IShare::TYPE_CIRCLE, + IShare::TYPE_ROOM, + IShare::TYPE_DECK, + IShare::TYPE_SCIENCEMESH + ]; + $shareManager = Server::get(IManager::class); + /** @var IShare[] $shares */ + $shares = array_merge(...array_map(function (int $type) use ($shareManager, $user) { + return $shareManager->getSharesBy($user, $type); + }, $providers)); + + foreach ($shares as $share) { + $sharedPath = $share->getNode()->getPath(); + if ($targetPath === $sharedPath || str_starts_with($targetPath, $sharedPath . '/')) { + $this->logger->debug( + 'It is not allowed to move one mount point into a shared folder', + ['app' => 'files']); + return false; + } + } + + return true; + } + + /** + * Get a fileinfo object for files that are ignored in the cache (part files) + */ + private function getPartFileInfo(string $path): \OC\Files\FileInfo { + $mount = $this->getMount($path); + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($this->getAbsolutePath($path)); + $ownerId = $storage->getOwner($internalPath); + if ($ownerId !== false) { + $owner = Server::get(IUserManager::class)->get($ownerId); + } else { + $owner = null; + } + return new FileInfo( + $this->getAbsolutePath($path), + $storage, + $internalPath, + [ + 'fileid' => null, + 'mimetype' => $storage->getMimeType($internalPath), + 'name' => basename($path), + 'etag' => null, + 'size' => $storage->filesize($internalPath), + 'mtime' => $storage->filemtime($internalPath), + 'encrypted' => false, + 'permissions' => \OCP\Constants::PERMISSION_ALL + ], + $mount, + $owner + ); + } + + /** + * @param string $path + * @param string $fileName + * @param bool $readonly Check only if the path is allowed for read-only access + * @throws InvalidPathException + */ + public function verifyPath($path, $fileName, $readonly = false): void { + // All of the view's functions disallow '..' in the path so we can short cut if the path is invalid + if (!Filesystem::isValidPath($path ?: '/')) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Path contains invalid segments')); + } + + // Short cut for read-only validation + if ($readonly) { + $validator = Server::get(FilenameValidator::class); + if ($validator->isForbidden($fileName)) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Filename is a reserved word')); + } + return; + } + + try { + /** @type \OCP\Files\Storage $storage */ + [$storage, $internalPath] = $this->resolvePath($path); + $storage->verifyPath($internalPath, $fileName); + } catch (ReservedWordException $ex) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($ex->getMessage() ?: $l->t('Filename is a reserved word')); + } catch (InvalidCharacterInPathException $ex) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($ex->getMessage() ?: $l->t('Filename contains at least one invalid character')); + } catch (FileNameTooLongException $ex) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Filename is too long')); + } catch (InvalidDirectoryException $ex) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Dot files are not allowed')); + } catch (EmptyFileNameException $ex) { + $l = \OCP\Util::getL10N('lib'); + throw new InvalidPathException($l->t('Empty filename is not allowed')); + } + } + + /** + * get all parent folders of $path + * + * @param string $path + * @return string[] + */ + private function getParents($path) { + $path = trim($path, '/'); + if (!$path) { + return []; + } + + $parts = explode('/', $path); + + // remove the single file + array_pop($parts); + $result = ['/']; + $resultPath = ''; + foreach ($parts as $part) { + if ($part) { + $resultPath .= '/' . $part; + $result[] = $resultPath; + } + } + return $result; + } + + /** + * Returns the mount point for which to lock + * + * @param string $absolutePath absolute path + * @param bool $useParentMount true to return parent mount instead of whatever + * is mounted directly on the given path, false otherwise + * @return IMountPoint mount point for which to apply locks + */ + private function getMountForLock(string $absolutePath, bool $useParentMount = false): IMountPoint { + $mount = Filesystem::getMountManager()->find($absolutePath); + + if ($useParentMount) { + // find out if something is mounted directly on the path + $internalPath = $mount->getInternalPath($absolutePath); + if ($internalPath === '') { + // resolve the parent mount instead + $mount = Filesystem::getMountManager()->find(dirname($absolutePath)); + } + } + + return $mount; + } + + /** + * Lock the given path + * + * @param string $path the path of the file to lock, relative to the view + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @param bool $lockMountPoint true to lock the mount point, false to lock the attached mount/storage + * + * @return bool False if the path is excluded from locking, true otherwise + * @throws LockedException if the path is already locked + */ + private function lockPath($path, $type, $lockMountPoint = false) { + $absolutePath = $this->getAbsolutePath($path); + $absolutePath = Filesystem::normalizePath($absolutePath); + if (!$this->shouldLockFile($absolutePath)) { + return false; + } + + $mount = $this->getMountForLock($absolutePath, $lockMountPoint); + try { + $storage = $mount->getStorage(); + if ($storage && $storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $storage->acquireLock( + $mount->getInternalPath($absolutePath), + $type, + $this->lockingProvider + ); + } + } catch (LockedException $e) { + // rethrow with the human-readable path + throw new LockedException( + $path, + $e, + $e->getExistingLock() + ); + } + + return true; + } + + /** + * Change the lock type + * + * @param string $path the path of the file to lock, relative to the view + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @param bool $lockMountPoint true to lock the mount point, false to lock the attached mount/storage + * + * @return bool False if the path is excluded from locking, true otherwise + * @throws LockedException if the path is already locked + */ + public function changeLock($path, $type, $lockMountPoint = false) { + $path = Filesystem::normalizePath($path); + $absolutePath = $this->getAbsolutePath($path); + $absolutePath = Filesystem::normalizePath($absolutePath); + if (!$this->shouldLockFile($absolutePath)) { + return false; + } + + $mount = $this->getMountForLock($absolutePath, $lockMountPoint); + try { + $storage = $mount->getStorage(); + if ($storage && $storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $storage->changeLock( + $mount->getInternalPath($absolutePath), + $type, + $this->lockingProvider + ); + } + } catch (LockedException $e) { + // rethrow with the a human-readable path + throw new LockedException( + $path, + $e, + $e->getExistingLock() + ); + } + + return true; + } + + /** + * Unlock the given path + * + * @param string $path the path of the file to unlock, relative to the view + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @param bool $lockMountPoint true to lock the mount point, false to lock the attached mount/storage + * + * @return bool False if the path is excluded from locking, true otherwise + * @throws LockedException + */ + private function unlockPath($path, $type, $lockMountPoint = false) { + $absolutePath = $this->getAbsolutePath($path); + $absolutePath = Filesystem::normalizePath($absolutePath); + if (!$this->shouldLockFile($absolutePath)) { + return false; + } + + $mount = $this->getMountForLock($absolutePath, $lockMountPoint); + $storage = $mount->getStorage(); + if ($storage && $storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { + $storage->releaseLock( + $mount->getInternalPath($absolutePath), + $type, + $this->lockingProvider + ); + } + + return true; + } + + /** + * Lock a path and all its parents up to the root of the view + * + * @param string $path the path of the file to lock relative to the view + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @param bool $lockMountPoint true to lock the mount point, false to lock the attached mount/storage + * + * @return bool False if the path is excluded from locking, true otherwise + * @throws LockedException + */ + public function lockFile($path, $type, $lockMountPoint = false) { + $absolutePath = $this->getAbsolutePath($path); + $absolutePath = Filesystem::normalizePath($absolutePath); + if (!$this->shouldLockFile($absolutePath)) { + return false; + } + + $this->lockPath($path, $type, $lockMountPoint); + + $parents = $this->getParents($path); + foreach ($parents as $parent) { + $this->lockPath($parent, ILockingProvider::LOCK_SHARED); + } + + return true; + } + + /** + * Unlock a path and all its parents up to the root of the view + * + * @param string $path the path of the file to lock relative to the view + * @param int $type \OCP\Lock\ILockingProvider::LOCK_SHARED or \OCP\Lock\ILockingProvider::LOCK_EXCLUSIVE + * @param bool $lockMountPoint true to lock the mount point, false to lock the attached mount/storage + * + * @return bool False if the path is excluded from locking, true otherwise + * @throws LockedException + */ + public function unlockFile($path, $type, $lockMountPoint = false) { + $absolutePath = $this->getAbsolutePath($path); + $absolutePath = Filesystem::normalizePath($absolutePath); + if (!$this->shouldLockFile($absolutePath)) { + return false; + } + + $this->unlockPath($path, $type, $lockMountPoint); + + $parents = $this->getParents($path); + foreach ($parents as $parent) { + $this->unlockPath($parent, ILockingProvider::LOCK_SHARED); + } + + return true; + } + + /** + * Only lock files in data/user/files/ + * + * @param string $path Absolute path to the file/folder we try to (un)lock + * @return bool + */ + protected function shouldLockFile($path) { + $path = Filesystem::normalizePath($path); + + $pathSegments = explode('/', $path); + if (isset($pathSegments[2])) { + // E.g.: /username/files/path-to-file + return ($pathSegments[2] === 'files') && (count($pathSegments) > 3); + } + + return !str_starts_with($path, '/appdata_'); + } + + /** + * Shortens the given absolute path to be relative to + * "$user/files". + * + * @param string $absolutePath absolute path which is under "files" + * + * @return string path relative to "files" with trimmed slashes or null + * if the path was NOT relative to files + * + * @throws \InvalidArgumentException if the given path was not under "files" + * @since 8.1.0 + */ + public function getPathRelativeToFiles($absolutePath) { + $path = Filesystem::normalizePath($absolutePath); + $parts = explode('/', trim($path, '/'), 3); + // "$user", "files", "path/to/dir" + if (!isset($parts[1]) || $parts[1] !== 'files') { + $this->logger->error( + '$absolutePath must be relative to "files", value is "{absolutePath}"', + [ + 'absolutePath' => $absolutePath, + ] + ); + throw new \InvalidArgumentException('$absolutePath must be relative to "files"'); + } + if (isset($parts[2])) { + return $parts[2]; + } + return ''; + } + + /** + * @param string $filename + * @return array + * @throws \OC\User\NoUserException + * @throws NotFoundException + */ + public function getUidAndFilename($filename) { + $info = $this->getFileInfo($filename); + if (!$info instanceof \OCP\Files\FileInfo) { + throw new NotFoundException($this->getAbsolutePath($filename) . ' not found'); + } + $uid = $info->getOwner()->getUID(); + if ($uid != \OC_User::getUser()) { + Filesystem::initMountPoints($uid); + $ownerView = new View('/' . $uid . '/files'); + try { + $filename = $ownerView->getPath($info['fileid']); + } catch (NotFoundException $e) { + throw new NotFoundException('File with id ' . $info['fileid'] . ' not found for user ' . $uid); + } + } + return [$uid, $filename]; + } + + /** + * Creates parent non-existing folders + * + * @param string $filePath + * @return bool + */ + private function createParentDirectories($filePath) { + $directoryParts = explode('/', $filePath); + $directoryParts = array_filter($directoryParts); + foreach ($directoryParts as $key => $part) { + $currentPathElements = array_slice($directoryParts, 0, $key); + $currentPath = '/' . implode('/', $currentPathElements); + if ($this->is_file($currentPath)) { + return false; + } + if (!$this->file_exists($currentPath)) { + $this->mkdir($currentPath); + } + } + + return true; + } +} |