diff options
Diffstat (limited to 'lib/private/Files')
33 files changed, 652 insertions, 306 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 2c190720c23..cb160115851 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -663,7 +663,7 @@ class Cache implements ICache { $sourceData = $sourceCache->get($sourcePath); if (!$sourceData) { - throw new \Exception('Invalid source storage path: ' . $sourcePath); + throw new \Exception('Source path not found in cache: ' . $sourcePath); } $shardDefinition = $this->connection->getShardDefinition('filecache'); diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 1fb408a0655..b067f70b8cb 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -210,7 +210,7 @@ class Scanner extends BasicEmitter implements IScanner { * @var \OC\Files\Cache\CacheEntry $cacheData */ $newData = $this->array_diff_assoc_multi($data, $cacheData->getData()); - + // make it known to the caller that etag has been changed and needs propagation if (isset($newData['etag'])) { $data['etag_changed'] = true; @@ -351,23 +351,23 @@ class Scanner extends BasicEmitter implements IScanner { * */ protected function array_diff_assoc_multi(array $array1, array $array2) { - + $result = []; foreach ($array1 as $key => $value) { - + // if $array2 doesn't have the same key, that's a result if (!array_key_exists($key, $array2)) { $result[$key] = $value; continue; } - + // if $array2's value for the same key is different, that's a result if ($array2[$key] !== $value && !is_array($value)) { $result[$key] = $value; continue; } - + if (is_array($value)) { $nestedDiff = $this->array_diff_assoc_multi($value, $array2[$key]); if (!empty($nestedDiff)) { diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 2b49e65f0b4..1a3bda58e6a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -213,6 +213,7 @@ class Storage { $query = $db->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->in('storage', $query->createNamedParameter($storageIds, IQueryBuilder::PARAM_INT_ARRAY))); + $query->runAcrossAllShards(); $query->executeStatement(); $query = $db->getQueryBuilder(); diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index 5c7bd4334f3..5bc4ee8529d 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -31,10 +31,13 @@ class CacheJail extends CacheWrapper { ) { parent::__construct($cache, $dependencies); - if ($cache instanceof CacheJail) { - $this->unjailedRoot = $cache->getSourcePath($root); - } else { - $this->unjailedRoot = $root; + $this->unjailedRoot = $root; + $parent = $cache; + while ($parent instanceof CacheWrapper) { + if ($parent instanceof CacheJail) { + $this->unjailedRoot = $parent->getSourcePath($this->unjailedRoot); + } + $parent = $parent->getCache(); } } @@ -50,7 +53,7 @@ class CacheJail extends CacheWrapper { * * @return string */ - protected function getGetUnjailedRoot() { + public function getGetUnjailedRoot() { return $this->unjailedRoot; } diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php index 6a5407934c8..9d63184e05f 100644 --- a/lib/private/Files/Config/MountProviderCollection.php +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -24,60 +24,43 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { use EmitterTrait; /** - * @var \OCP\Files\Config\IHomeMountProvider[] + * @var list<IHomeMountProvider> */ - private $homeProviders = []; + private array $homeProviders = []; /** - * @var \OCP\Files\Config\IMountProvider[] + * @var list<IMountProvider> */ - private $providers = []; + private array $providers = []; - /** @var \OCP\Files\Config\IRootMountProvider[] */ - private $rootProviders = []; + /** @var list<IRootMountProvider> */ + private array $rootProviders = []; - /** - * @var \OCP\Files\Storage\IStorageFactory - */ - private $loader; - - /** - * @var \OCP\Files\Config\IUserMountCache - */ - private $mountCache; - - /** @var callable[] */ - private $mountFilters = []; + /** @var list<callable> */ + private array $mountFilters = []; - private IEventLogger $eventLogger; - - /** - * @param \OCP\Files\Storage\IStorageFactory $loader - * @param IUserMountCache $mountCache - */ public function __construct( - IStorageFactory $loader, - IUserMountCache $mountCache, - IEventLogger $eventLogger, + private IStorageFactory $loader, + private IUserMountCache $mountCache, + private IEventLogger $eventLogger, ) { - $this->loader = $loader; - $this->mountCache = $mountCache; - $this->eventLogger = $eventLogger; } + /** + * @return list<IMountPoint> + */ private function getMountsFromProvider(IMountProvider $provider, IUser $user, IStorageFactory $loader): array { $class = str_replace('\\', '_', get_class($provider)); $uid = $user->getUID(); $this->eventLogger->start('fs:setup:provider:' . $class, "Getting mounts from $class for $uid"); $mounts = $provider->getMountsForUser($user, $loader) ?? []; $this->eventLogger->end('fs:setup:provider:' . $class); - return $mounts; + return array_values($mounts); } /** - * @param IUser $user - * @param IMountProvider[] $providers - * @return IMountPoint[] + * @param list<IMountProvider> $providers + * @return list<IMountPoint> */ private function getUserMountsForProviders(IUser $user, array $providers): array { $loader = $this->loader; @@ -90,10 +73,16 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { return $this->filterMounts($user, $mounts); } + /** + * @return list<IMountPoint> + */ public function getMountsForUser(IUser $user): array { return $this->getUserMountsForProviders($user, $this->providers); } + /** + * @return list<IMountPoint> + */ public function getUserMountsForProviderClasses(IUser $user, array $mountProviderClasses): array { $providers = array_filter( $this->providers, @@ -102,7 +91,10 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { return $this->getUserMountsForProviders($user, $providers); } - public function addMountForUser(IUser $user, IMountManager $mountManager, ?callable $providerFilter = null) { + /** + * @return list<IMountPoint> + */ + public function addMountForUser(IUser $user, IMountManager $mountManager, ?callable $providerFilter = null): array { // shared mount provider gets to go last since it needs to know existing files // to check for name collisions $firstMounts = []; @@ -135,18 +127,15 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { array_walk($lateMounts, [$mountManager, 'addMount']); $this->eventLogger->end('fs:setup:add-mounts'); - return array_merge($lateMounts, $firstMounts); + return array_values(array_merge($lateMounts, $firstMounts)); } /** * Get the configured home mount for this user * - * @param \OCP\IUser $user - * @return \OCP\Files\Mount\IMountPoint * @since 9.1.0 */ - public function getHomeMountForUser(IUser $user) { - /** @var \OCP\Files\Config\IHomeMountProvider[] $providers */ + public function getHomeMountForUser(IUser $user): IMountPoint { $providers = array_reverse($this->homeProviders); // call the latest registered provider first to give apps an opportunity to overwrite builtin foreach ($providers as $homeProvider) { if ($mount = $homeProvider->getHomeMountForUser($user, $this->loader)) { @@ -159,34 +148,36 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { /** * Add a provider for mount points - * - * @param \OCP\Files\Config\IMountProvider $provider */ - public function registerProvider(IMountProvider $provider) { + public function registerProvider(IMountProvider $provider): void { $this->providers[] = $provider; $this->emit('\OC\Files\Config', 'registerMountProvider', [$provider]); } - public function registerMountFilter(callable $filter) { + public function registerMountFilter(callable $filter): void { $this->mountFilters[] = $filter; } - private function filterMounts(IUser $user, array $mountPoints) { - return array_filter($mountPoints, function (IMountPoint $mountPoint) use ($user) { + /** + * @param list<IMountPoint> $mountPoints + * @return list<IMountPoint> + */ + private function filterMounts(IUser $user, array $mountPoints): array { + return array_values(array_filter($mountPoints, function (IMountPoint $mountPoint) use ($user) { foreach ($this->mountFilters as $filter) { if ($filter($mountPoint, $user) === false) { return false; } } return true; - }); + })); } /** * Add a provider for home mount points * - * @param \OCP\Files\Config\IHomeMountProvider $provider + * @param IHomeMountProvider $provider * @since 9.1.0 */ public function registerHomeProvider(IHomeMountProvider $provider) { @@ -196,21 +187,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { /** * Get the mount cache which can be used to search for mounts without setting up the filesystem - * - * @return IUserMountCache */ - public function getMountCache() { + public function getMountCache(): IUserMountCache { return $this->mountCache; } - public function registerRootProvider(IRootMountProvider $provider) { + public function registerRootProvider(IRootMountProvider $provider): void { $this->rootProviders[] = $provider; } /** * Get all root mountpoints * - * @return \OCP\Files\Mount\IMountPoint[] + * @return list<IMountPoint> * @since 20.0.0 */ public function getRootMounts(): array { @@ -226,16 +215,33 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { throw new \Exception('No root mounts provided by any provider'); } - return $mounts; + return array_values($mounts); } - public function clearProviders() { + public function clearProviders(): void { $this->providers = []; $this->homeProviders = []; $this->rootProviders = []; } + /** + * @return list<IMountProvider> + */ public function getProviders(): array { return $this->providers; } + + /** + * @return list<IHomeMountProvider> + */ + public function getHomeProviders(): array { + return $this->homeProviders; + } + + /** + * @return list<IRootMountProvider> + */ + public function getRootProviders(): array { + return $this->rootProviders; + } } diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php index 9c1e2314262..09ac6d1416a 100644 --- a/lib/private/Files/Config/UserMountCache.php +++ b/lib/private/Files/Config/UserMountCache.php @@ -11,6 +11,10 @@ use OC\User\LazyUser; use OCP\Cache\CappedMemoryCache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Diagnostics\IEventLogger; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Config\Event\UserMountAddedEvent; +use OCP\Files\Config\Event\UserMountRemovedEvent; +use OCP\Files\Config\Event\UserMountUpdatedEvent; use OCP\Files\Config\ICachedMountFileInfo; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; @@ -46,6 +50,7 @@ class UserMountCache implements IUserMountCache { private IUserManager $userManager, private LoggerInterface $logger, private IEventLogger $eventLogger, + private IEventDispatcher $eventDispatcher, ) { $this->cacheInfoCache = new CappedMemoryCache(); $this->internalPathCache = new CappedMemoryCache(); @@ -108,17 +113,29 @@ class UserMountCache implements IUserMountCache { $this->removeFromCache($mount); unset($this->mountsForUsers[$userUID][$mount->getKey()]); } - foreach ($changedMounts as $mount) { - $this->logger->debug("Updating mount '{$mount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $mount->getMountProvider()]); - $this->updateCachedMount($mount); + foreach ($changedMounts as $mountPair) { + $newMount = $mountPair[1]; + $this->logger->debug("Updating mount '{$newMount->getKey()}' for user '$userUID'", ['app' => 'files', 'mount_provider' => $newMount->getMountProvider()]); + $this->updateCachedMount($newMount); /** @psalm-suppress InvalidArgument */ - $this->mountsForUsers[$userUID][$mount->getKey()] = $mount; + $this->mountsForUsers[$userUID][$newMount->getKey()] = $newMount; } $this->connection->commit(); } catch (\Throwable $e) { $this->connection->rollBack(); throw $e; } + + // Only fire events after all mounts have already been adjusted in the database. + foreach ($addedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountAddedEvent($mount)); + } + foreach ($removedMounts as $mount) { + $this->eventDispatcher->dispatchTyped(new UserMountRemovedEvent($mount)); + } + foreach ($changedMounts as $mountPair) { + $this->eventDispatcher->dispatchTyped(new UserMountUpdatedEvent($mountPair[0], $mountPair[1])); + } } $this->eventLogger->end('fs:setup:user:register'); } @@ -126,9 +143,9 @@ class UserMountCache implements IUserMountCache { /** * @param array<string, ICachedMountInfo> $newMounts * @param array<string, ICachedMountInfo> $cachedMounts - * @return ICachedMountInfo[] + * @return list<list{0: ICachedMountInfo, 1: ICachedMountInfo}> Pairs of old and new mounts */ - private function findChangedMounts(array $newMounts, array $cachedMounts) { + private function findChangedMounts(array $newMounts, array $cachedMounts): array { $changed = []; foreach ($cachedMounts as $key => $cachedMount) { if (isset($newMounts[$key])) { @@ -138,7 +155,7 @@ class UserMountCache implements IUserMountCache { $newMount->getMountId() !== $cachedMount->getMountId() || $newMount->getMountProvider() !== $cachedMount->getMountProvider() ) { - $changed[] = $newMount; + $changed[] = [$cachedMount, $newMount]; } } } diff --git a/lib/private/Files/FilenameValidator.php b/lib/private/Files/FilenameValidator.php index b1979789ec8..a78c6d3cc3c 100644 --- a/lib/private/Files/FilenameValidator.php +++ b/lib/private/Files/FilenameValidator.php @@ -228,6 +228,43 @@ class FilenameValidator implements IFilenameValidator { return false; } + public function sanitizeFilename(string $name, ?string $charReplacement = null): string { + $forbiddenCharacters = $this->getForbiddenCharacters(); + + if ($charReplacement === null) { + $charReplacement = array_diff(['_', '-', ' '], $forbiddenCharacters); + $charReplacement = reset($charReplacement) ?: ''; + } + if (mb_strlen($charReplacement) !== 1) { + throw new \InvalidArgumentException('No or invalid character replacement given'); + } + + $nameLowercase = mb_strtolower($name); + foreach ($this->getForbiddenExtensions() as $extension) { + if (str_ends_with($nameLowercase, $extension)) { + $name = substr($name, 0, strlen($name) - strlen($extension)); + } + } + + $basename = strlen($name) > 1 + ? substr($name, 0, strpos($name, '.', 1) ?: null) + : $name; + if (in_array(mb_strtolower($basename), $this->getForbiddenBasenames())) { + $name = str_replace($basename, $this->l10n->t('%1$s (renamed)', [$basename]), $name); + } + + if ($name === '') { + $name = $this->l10n->t('renamed file'); + } + + if (in_array(mb_strtolower($name), $this->getForbiddenFilenames())) { + $name = $this->l10n->t('%1$s (renamed)', [$name]); + } + + $name = str_replace($forbiddenCharacters, $charReplacement, $name); + return $name; + } + protected function checkForbiddenName(string $filename): void { $filename = mb_strtolower($filename); if ($this->isForbidden($filename)) { diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 99c52108fa8..4b088f2c808 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -7,117 +7,39 @@ */ namespace OC\Files\Mount; +use OC\Files\ObjectStore\HomeObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OCP\Files\Config\IHomeMountProvider; +use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage\IStorageFactory; -use OCP\IConfig; use OCP\IUser; -use Psr\Log\LoggerInterface; /** * Mount provider for object store home storages */ class ObjectHomeMountProvider implements IHomeMountProvider { - /** - * @var IConfig - */ - private $config; - - /** - * ObjectStoreHomeMountProvider constructor. - * - * @param IConfig $config - */ - public function __construct(IConfig $config) { - $this->config = $config; + public function __construct( + private PrimaryObjectStoreConfig $objectStoreConfig, + ) { } /** - * Get the cache mount for a user + * Get the home mount for a user * * @param IUser $user * @param IStorageFactory $loader - * @return \OCP\Files\Mount\IMountPoint + * @return ?IMountPoint */ - public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { - $config = $this->getMultiBucketObjectStoreConfig($user); - if ($config === null) { - $config = $this->getSingleBucketObjectStoreConfig($user); - } - - if ($config === null) { + public function getHomeMountForUser(IUser $user, IStorageFactory $loader): ?IMountPoint { + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForUser($user); + if ($objectStoreConfig === null) { return null; } + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + 'user' => $user, + ]); - return new HomeMountPoint($user, '\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader, null, null, self::class); - } - - /** - * @param IUser $user - * @return array|null - */ - private function getSingleBucketObjectStoreConfig(IUser $user) { - $config = $this->config->getSystemValue('objectstore'); - if (!is_array($config)) { - return null; - } - - // sanity checks - if (empty($config['class'])) { - \OC::$server->get(LoggerInterface::class)->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['user'] = $user; - - return $config; - } - - /** - * @param IUser $user - * @return array|null - */ - private function getMultiBucketObjectStoreConfig(IUser $user) { - $config = $this->config->getSystemValue('objectstore_multibucket'); - if (!is_array($config)) { - return null; - } - - // sanity checks - if (empty($config['class'])) { - \OC::$server->get(LoggerInterface::class)->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); - - if ($bucket === null) { - /* - * Use any provided bucket argument as prefix - * and add the mapping from username => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - $mapper = new \OC\Files\ObjectStore\Mapper($user, $this->config); - $numBuckets = $config['arguments']['num_buckets'] ?? 64; - $config['arguments']['bucket'] .= $mapper->getBucket($numBuckets); - - $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $config['arguments']['bucket']); - } else { - $config['arguments']['bucket'] = $bucket; - } - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['user'] = $user; - - return $config; + return new HomeMountPoint($user, HomeObjectStoreStorage::class, '/' . $user->getUID(), $arguments, $loader, null, null, self::class); } } diff --git a/lib/private/Files/Mount/RootMountProvider.php b/lib/private/Files/Mount/RootMountProvider.php index 86f8188978f..5e0c924ad38 100644 --- a/lib/private/Files/Mount/RootMountProvider.php +++ b/lib/private/Files/Mount/RootMountProvider.php @@ -10,79 +10,41 @@ namespace OC\Files\Mount; use OC; use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\Storage\LocalRootStorage; -use OC_App; use OCP\Files\Config\IRootMountProvider; use OCP\Files\Storage\IStorageFactory; use OCP\IConfig; -use Psr\Log\LoggerInterface; class RootMountProvider implements IRootMountProvider { + private PrimaryObjectStoreConfig $objectStoreConfig; private IConfig $config; - private LoggerInterface $logger; - public function __construct(IConfig $config, LoggerInterface $logger) { + public function __construct(PrimaryObjectStoreConfig $objectStoreConfig, IConfig $config) { + $this->objectStoreConfig = $objectStoreConfig; $this->config = $config; - $this->logger = $logger; } public function getRootMounts(IStorageFactory $loader): array { - $objectStore = $this->config->getSystemValue('objectstore', null); - $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - if ($objectStoreMultiBucket) { - return [$this->getMultiBucketStoreRootMount($loader, $objectStoreMultiBucket)]; - } elseif ($objectStore) { - return [$this->getObjectStoreRootMount($loader, $objectStore)]; + if ($objectStoreConfig) { + return [$this->getObjectStoreRootMount($loader, $objectStoreConfig)]; } else { return [$this->getLocalRootMount($loader)]; } } - private function validateObjectStoreConfig(array &$config) { - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - // instantiate object store implementation - $name = $config['class']; - if (str_starts_with($name, 'OCA\\') && substr_count($name, '\\') >= 2) { - $segments = explode('\\', $name); - OC_App::loadApp(strtolower($segments[1])); - } - } - private function getLocalRootMount(IStorageFactory $loader): MountPoint { $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); return new MountPoint(LocalRootStorage::class, '/', ['datadir' => $configDataDirectory], $loader, null, null, self::class); } - private function getObjectStoreRootMount(IStorageFactory $loader, array $config): MountPoint { - $this->validateObjectStoreConfig($config); - - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - // mount with plain / root object store implementation - $config['class'] = ObjectStoreStorage::class; - - return new MountPoint($config['class'], '/', $config['arguments'], $loader, null, null, self::class); - } - - private function getMultiBucketStoreRootMount(IStorageFactory $loader, array $config): MountPoint { - $this->validateObjectStoreConfig($config); - - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - // put the root FS always in first bucket for multibucket configuration - $config['arguments']['bucket'] .= '0'; - - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - // mount with plain / root object store implementation - $config['class'] = ObjectStoreStorage::class; + private function getObjectStoreRootMount(IStorageFactory $loader, array $objectStoreConfig): MountPoint { + $arguments = array_merge($objectStoreConfig['arguments'], [ + 'objectstore' => $this->objectStoreConfig->buildObjectStore($objectStoreConfig), + ]); - return new MountPoint($config['class'], '/', $config['arguments'], $loader, null, null, self::class); + return new MountPoint(ObjectStoreStorage::class, '/', $arguments, $loader, null, null, self::class); } } diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php index a894c69649a..c41838fd6b0 100644 --- a/lib/private/Files/Node/Folder.php +++ b/lib/private/Files/Node/Folder.php @@ -126,8 +126,21 @@ class Folder extends Node implements \OCP\Files\Folder { $fullPath = $this->getFullPath($path); $nonExisting = new NonExistingFolder($this->root, $this->view, $fullPath); $this->sendHooks(['preWrite', 'preCreate'], [$nonExisting]); - if (!$this->view->mkdir($fullPath) && !$this->view->is_dir($fullPath)) { - throw new NotPermittedException('Could not create folder "' . $fullPath . '"'); + if (!$this->view->mkdir($fullPath)) { + // maybe another concurrent process created the folder already + if (!$this->view->is_dir($fullPath)) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '"'); + } else { + // we need to ensure we don't return before the concurrent request has finished updating the cache + $tries = 5; + while (!$this->view->getFileInfo($fullPath)) { + if ($tries < 1) { + throw new NotPermittedException('Could not create folder "' . $fullPath . '", folder exists but unable to get cache entry'); + } + usleep(5 * 1000); + $tries--; + } + } } $parent = dirname($fullPath) === $this->getPath() ? $this : null; $node = new Folder($this->root, $this->view, $fullPath, null, $parent); @@ -447,4 +460,12 @@ class Folder extends Node implements \OCP\Files\Folder { return $this->search($query); } + + public function verifyPath($fileName, $readonly = false): void { + $this->view->verifyPath( + $this->getPath(), + $fileName, + $readonly, + ); + } } diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php index 5879748d951..37b1efa0fad 100644 --- a/lib/private/Files/Node/LazyFolder.php +++ b/lib/private/Files/Node/LazyFolder.php @@ -561,4 +561,8 @@ class LazyFolder implements Folder { public function getMetadata(): array { return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); } + + public function verifyPath($fileName, $readonly = false): void { + $this->__call(__FUNCTION__, func_get_args()); + } } diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 16ef4e7de63..752c3cf4fb7 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -22,6 +22,7 @@ use OCP\Files\FileInfo; use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; @@ -66,7 +67,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir(string $path, bool $force = false): bool { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); if (!$force && $this->file_exists($path)) { $this->logger->warning("Tried to create an object store folder that already exists: $path"); @@ -76,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -412,16 +413,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { $this->logger->error( 'Could not create object for ' . $path, @@ -479,6 +470,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -507,13 +503,21 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil ]); $size = $writtenSize; }); - $this->objectStore->writeObject($urn, $countStream, $mimetype); + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); + } else { + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); + } if (is_resource($countStream)) { fclose($countStream); } $stat['size'] = $size; } else { - $this->objectStore->writeObject($urn, $stream, $mimetype); + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $stream, $metadata); + } else { + $this->objectStore->writeObject($urn, $stream, $metadata['mimetype']); + } if (is_resource($stream)) { fclose($stream); } @@ -695,7 +699,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php new file mode 100644 index 00000000000..fdfe989addc --- /dev/null +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -0,0 +1,140 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\ObjectStore; + +use OCP\App\IAppManager; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +use OCP\IUser; + +/** + * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}} + */ +class PrimaryObjectStoreConfig { + public function __construct( + private readonly IConfig $config, + private readonly IAppManager $appManager, + ) { + } + + /** + * @param ObjectStoreConfig $config + */ + public function buildObjectStore(array $config): IObjectStore { + return new $config['class']($config['arguments']); + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForRoot(): ?array { + $config = $this->getObjectStoreConfig(); + + if ($config && $config['arguments']['multibucket']) { + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + // put the root FS always in first bucket for multibucket configuration + $config['arguments']['bucket'] .= '0'; + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForUser(IUser $user): ?array { + $config = $this->getObjectStoreConfig(); + + if ($config && $config['arguments']['multibucket']) { + $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + private function getObjectStoreConfig(): ?array { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + + // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config + if ($objectStoreMultiBucket) { + $objectStoreMultiBucket['arguments']['multibucket'] = true; + return $this->validateObjectStoreConfig($objectStoreMultiBucket); + } elseif ($objectStore) { + return $this->validateObjectStoreConfig($objectStore); + } else { + return null; + } + } + + /** + * @return ObjectStoreConfig + */ + private function validateObjectStoreConfig(array $config) { + if (!isset($config['class'])) { + throw new \Exception('No class configured for object store'); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $class = $config['class']; + $arguments = $config['arguments']; + if (!is_array($arguments)) { + throw new \Exception('Configured object store arguments are not an array'); + } + if (!isset($arguments['multibucket'])) { + $arguments['multibucket'] = false; + } + if (!is_bool($arguments['multibucket'])) { + throw new \Exception('arguments.multibucket must be a boolean in object store configuration'); + } + + if (!is_string($class)) { + throw new \Exception('Configured class for object store is not a string'); + } + + if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { + [$appId] = explode('\\', $class); + $this->appManager->loadApp(strtolower($appId)); + } + + if (!is_a($class, IObjectStore::class, true)) { + throw new \Exception('Configured class for object store is not an object store'); + } + return [ + 'class' => $class, + 'arguments' => $arguments, + ]; + } + + private function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new Mapper($user, $this->config); + $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); + } + + return $bucket; + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index e970fb6ac14..23c061db174 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -95,6 +95,16 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaD ]); } + private function parseS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + if (str_starts_with($key, 'x-amz-meta-')) { + $result[substr($key, strlen('x-amz-meta-'))] = $value; + } + } + return $result; + } + public function getObjectMetaData(string $urn): array { $object = $this->getConnection()->headObject([ 'Bucket' => $this->bucket, @@ -104,7 +114,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaD 'mtime' => $object['LastModified'], 'etag' => trim($object['ETag'], '"'), 'size' => (int)($object['Size'] ?? $object['ContentLength']), - ]; + ] + $this->parseS3Metadata($object['Metadata'] ?? []); } public function listObjects(string $prefix = ''): \Iterator { diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php index 63f14ac2d00..5b086db8f77 100644 --- a/lib/private/Files/ObjectStore/S3ConfigTrait.php +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -18,6 +18,10 @@ trait S3ConfigTrait { /** Maximum number of concurrent multipart uploads */ protected int $concurrency; + /** Timeout, in seconds, for the connection to S3 server, not for the + * request. */ + protected float $connectTimeout; + protected int $timeout; protected string|false $proxy; diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index b7017583dc2..062d2e4bde4 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -39,6 +39,7 @@ trait S3ConnectionTrait { // Default to 5 like the S3 SDK does $this->concurrency = $params['concurrency'] ?? 5; $this->proxy = $params['proxy'] ?? false; + $this->connectTimeout = $params['connect_timeout'] ?? 5; $this->timeout = $params['timeout'] ?? 15; $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; @@ -102,8 +103,7 @@ trait S3ConnectionTrait { 'use_arn_region' => false, 'http' => [ 'verify' => $this->getCertificateBundlePath(), - // Timeout for the connection to S3 server, not for the request. - 'connect_timeout' => 5 + 'connect_timeout' => $this->connectTimeout, ], 'use_aws_shared_config_files' => false, 'retries' => [ diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 9d7cfa644e6..5e6dcf88a42 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -77,22 +77,32 @@ trait S3ObjectTrait { return $fh; } + private function buildS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + $result['x-amz-meta-' . $key] = $value; + } + return $result; + } /** * Single object put helper * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeSingle(string $urn, StreamInterface $stream, ?string $mimetype = null): void { + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); $this->getConnection()->putObject([ 'Bucket' => $this->bucket, 'Key' => $urn, 'Body' => $stream, 'ACL' => 'private', 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), 'StorageClass' => $this->storageClass, ] + $this->getSSECParameters()); } @@ -103,43 +113,72 @@ trait S3ObjectTrait { * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeMultiPart(string $urn, StreamInterface $stream, ?string $mimetype = null): void { - $uploader = new MultipartUploader($this->getConnection(), $stream, [ - 'bucket' => $this->bucket, - 'concurrency' => $this->concurrency, - 'key' => $urn, - 'part_size' => $this->uploadPartSize, - 'params' => [ - 'ContentType' => $mimetype, - 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters(), - ]); + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + + $attempts = 0; + $uploaded = false; + $concurrency = $this->concurrency; + $exception = null; + $state = null; + + // retry multipart upload once with concurrency at half on failure + while (!$uploaded && $attempts <= 1) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'concurrency' => $concurrency, + 'key' => $urn, + 'part_size' => $this->uploadPartSize, + 'state' => $state, + 'params' => [ + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(), + ]); + + try { + $uploader->upload(); + $uploaded = true; + } catch (S3MultipartUploadException $e) { + $exception = $e; + $attempts++; + + if ($concurrency > 1) { + $concurrency = round($concurrency / 2); + } - try { - $uploader->upload(); - } catch (S3MultipartUploadException $e) { + if ($stream->isSeekable()) { + $stream->rewind(); + } + } + } + + if (!$uploaded) { // if anything goes wrong with multipart, make sure that you don´t poison and // slow down s3 bucket with orphaned fragments - $uploadInfo = $e->getState()->getId(); - if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { + $uploadInfo = $exception->getState()->getId(); + if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { $this->getConnection()->abortMultipartUpload($uploadInfo); } - throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $e); + + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception); } } - - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } + + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { $canSeek = fseek($stream, 0, SEEK_CUR) === 0; $psrStream = Utils::streamFor($stream); @@ -154,16 +193,16 @@ trait S3ObjectTrait { $buffer->seek(0); if ($buffer->getSize() < $this->putSizeLimit) { // buffer is fully seekable, so use it directly for the small upload - $this->writeSingle($urn, $buffer, $mimetype); + $this->writeSingle($urn, $buffer, $metaData); } else { $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); - $this->writeMultiPart($urn, $loadStream, $mimetype); + $this->writeMultiPart($urn, $loadStream, $metaData); } } else { if ($size < $this->putSizeLimit) { - $this->writeSingle($urn, $psrStream, $mimetype); + $this->writeSingle($urn, $psrStream, $metaData); } else { - $this->writeMultiPart($urn, $psrStream, $mimetype); + $this->writeMultiPart($urn, $psrStream, $metaData); } } $psrStream->close(); diff --git a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php index 3f2b36823ab..42d960cc22a 100644 --- a/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php +++ b/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php @@ -32,7 +32,7 @@ class OrEqualsToIn extends ReplacingOptimizerStep { $value = $comparison->getValue(); return $value; }, $group); - $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values); + $in = new SearchComparison(ISearchComparison::COMPARE_IN, $field, $values, $group[0]->getExtra()); $pathEqHash = array_reduce($group, function ($pathEqHash, ISearchComparison $comparison) { return $comparison->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true) && $pathEqHash; }, true); diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php index 5c7655b49c1..3e5258a715f 100644 --- a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php +++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php @@ -24,7 +24,7 @@ class SplitLargeIn extends ReplacingOptimizerStep { ) { $chunks = array_chunk($operator->getValue(), 1000); $chunkComparisons = array_map(function (array $values) use ($operator) { - return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values); + return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values, $operator->getExtra()); }, $chunks); $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons); diff --git a/lib/private/Files/SetupManager.php b/lib/private/Files/SetupManager.php index 2b8121a529d..4ab40b01f4a 100644 --- a/lib/private/Files/SetupManager.php +++ b/lib/private/Files/SetupManager.php @@ -23,7 +23,6 @@ use OC\Share\Share; use OC\Share20\ShareDisableChecker; use OC_App; use OC_Hook; -use OC_Util; use OCA\Files_External\Config\ExternalMountPoint; use OCA\Files_Sharing\External\Mount; use OCA\Files_Sharing\ISharedMountPoint; @@ -34,6 +33,7 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IHomeMountProvider; use OCP\Files\Config\IMountProvider; +use OCP\Files\Config\IRootMountProvider; use OCP\Files\Config\IUserMountCache; use OCP\Files\Events\BeforeFileSystemSetupEvent; use OCP\Files\Events\InvalidateMountCacheEvent; @@ -156,7 +156,7 @@ class SetupManager { if ($mount instanceof HomeMountPoint) { $user = $mount->getUser(); return new Quota(['storage' => $storage, 'quotaCallback' => function () use ($user) { - return OC_Util::getUserQuota($user); + return $user->getQuotaBytes(); }, 'root' => 'files', 'include_external_storage' => $quotaIncludeExternal]); } @@ -280,9 +280,13 @@ class SetupManager { $mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot) { return str_starts_with($mount->getMountPoint(), $userRoot); }); - $allProviders = array_map(function (IMountProvider $provider) { + $allProviders = array_map(function (IMountProvider|IHomeMountProvider|IRootMountProvider $provider) { return get_class($provider); - }, $this->mountProviderCollection->getProviders()); + }, array_merge( + $this->mountProviderCollection->getProviders(), + $this->mountProviderCollection->getHomeProviders(), + $this->mountProviderCollection->getRootProviders(), + )); $newProviders = array_diff($allProviders, $previouslySetupProviders); $mounts = array_filter($mounts, function (IMountPoint $mount) use ($previouslySetupProviders) { return !in_array($mount->getMountProvider(), $previouslySetupProviders); diff --git a/lib/private/Files/SimpleFS/SimpleFile.php b/lib/private/Files/SimpleFS/SimpleFile.php index cbb3af4db29..0bfaea21788 100644 --- a/lib/private/Files/SimpleFS/SimpleFile.php +++ b/lib/private/Files/SimpleFS/SimpleFile.php @@ -6,9 +6,11 @@ namespace OC\Files\SimpleFS; use OCP\Files\File; +use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\ISimpleFile; +use OCP\Lock\LockedException; class SimpleFile implements ISimpleFile { private File $file; @@ -48,8 +50,10 @@ class SimpleFile implements ISimpleFile { /** * Get the content * - * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException * @throws NotFoundException + * @throws NotPermittedException */ public function getContent(): string { $result = $this->file->getContent(); @@ -65,8 +69,10 @@ class SimpleFile implements ISimpleFile { * Overwrite the file * * @param string|resource $data - * @throws NotPermittedException + * @throws GenericFileException + * @throws LockedException * @throws NotFoundException + * @throws NotPermittedException */ public function putContent($data): void { try { diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php index ca0e38774c8..b1e02cf2e6c 100644 --- a/lib/private/Files/Storage/Common.php +++ b/lib/private/Files/Storage/Common.php @@ -19,6 +19,7 @@ use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\Storage\Wrapper\Encryption; use OC\Files\Storage\Wrapper\Jail; use OC\Files\Storage\Wrapper\Wrapper; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IScanner; @@ -205,7 +206,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, } else { $sourceStream = $this->fopen($source, 'r'); $targetStream = $this->fopen($target, 'w'); - [, $result] = \OC_Helper::streamCopy($sourceStream, $targetStream); + [, $result] = Files::streamCopy($sourceStream, $targetStream, true); if (!$result) { Server::get(LoggerInterface::class)->warning("Failed to write data while copying $source to $target"); } @@ -734,7 +735,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage, throw new GenericFileException("Failed to open $path for writing"); } try { - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + [$count, $result] = Files::streamCopy($stream, $target, true); if (!$result) { throw new GenericFileException('Failed to copy stream'); } diff --git a/lib/private/Files/Storage/DAV.php b/lib/private/Files/Storage/DAV.php index 10670d6331a..afd8f87e2de 100644 --- a/lib/private/Files/Storage/DAV.php +++ b/lib/private/Files/Storage/DAV.php @@ -350,7 +350,13 @@ class DAV extends Common { } } - return $response->getBody(); + $content = $response->getBody(); + + if ($content === null || is_string($content)) { + return false; + } + + return $content; case 'w': case 'wb': case 'a': @@ -390,6 +396,8 @@ class DAV extends Common { $this->writeBack($tmpFile, $path); }); } + + return false; } public function writeBack(string $tmpFile, string $path): void { diff --git a/lib/private/Files/Storage/LocalTempFileTrait.php b/lib/private/Files/Storage/LocalTempFileTrait.php index c8e3588f2e8..fffc3e789f3 100644 --- a/lib/private/Files/Storage/LocalTempFileTrait.php +++ b/lib/private/Files/Storage/LocalTempFileTrait.php @@ -7,6 +7,8 @@ */ namespace OC\Files\Storage; +use OCP\Files; + /** * Storage backend class for providing common filesystem operation methods * which are not storage-backend specific. @@ -45,7 +47,7 @@ trait LocalTempFileTrait { } $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($extension); $target = fopen($tmpFile, 'w'); - \OC_Helper::streamCopy($source, $target); + Files::streamCopy($source, $target); fclose($target); return $tmpFile; } diff --git a/lib/private/Files/Storage/Temporary.php b/lib/private/Files/Storage/Temporary.php index ff7a816930d..ecf8a1315a9 100644 --- a/lib/private/Files/Storage/Temporary.php +++ b/lib/private/Files/Storage/Temporary.php @@ -7,16 +7,20 @@ */ namespace OC\Files\Storage; +use OCP\Files; +use OCP\ITempManager; +use OCP\Server; + /** * local storage backend in temporary folder for testing purpose */ class Temporary extends Local { public function __construct(array $parameters = []) { - parent::__construct(['datadir' => \OC::$server->getTempManager()->getTemporaryFolder()]); + parent::__construct(['datadir' => Server::get(ITempManager::class)->getTemporaryFolder()]); } public function cleanUp(): void { - \OC_Helper::rmdirr($this->datadir); + Files::rmdirr($this->datadir); } public function __destruct() { diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php index ba23f3c43ec..4d38d2d37aa 100644 --- a/lib/private/Files/Storage/Wrapper/Encryption.php +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -8,7 +8,6 @@ namespace OC\Files\Storage\Wrapper; use OC\Encryption\Exceptions\ModuleDoesNotExistsException; -use OC\Encryption\Update; use OC\Encryption\Util; use OC\Files\Cache\CacheEntry; use OC\Files\Filesystem; @@ -18,9 +17,11 @@ use OC\Files\Storage\Common; use OC\Files\Storage\LocalTempFileTrait; use OC\Memcache\ArrayCache; use OCP\Cache\CappedMemoryCache; +use OCP\Encryption\Exceptions\InvalidHeaderException; use OCP\Encryption\IFile; use OCP\Encryption\IManager; use OCP\Encryption\Keys\IStorage; +use OCP\Files; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Mount\IMountPoint; use OCP\Files\Storage; @@ -49,7 +50,6 @@ class Encryption extends Wrapper { private IFile $fileHelper, private ?string $uid, private IStorage $keyStorage, - private Update $update, private Manager $mountManager, private ArrayCache $arrayCache, ) { @@ -64,7 +64,8 @@ class Encryption extends Wrapper { $info = $this->getCache()->get($path); if ($info === false) { - return false; + /* Pass call to wrapped storage, it may be a special file like a part file */ + return $this->storage->filesize($path); } if (isset($this->unencryptedSize[$fullPath])) { $size = $this->unencryptedSize[$fullPath]; @@ -318,7 +319,7 @@ class Encryption extends Wrapper { if (!empty($encryptionModuleId)) { $encryptionModule = $this->encryptionManager->getEncryptionModule($encryptionModuleId); $shouldEncrypt = true; - } elseif (empty($encryptionModuleId) && $info['encrypted'] === true) { + } elseif ($info !== false && $info['encrypted'] === true) { // we come from a old installation. No header and/or no module defined // but the file is encrypted. In this case we need to use the // OC_DEFAULT_MODULE to read the file @@ -344,6 +345,16 @@ class Encryption extends Wrapper { if ($shouldEncrypt === true && $encryptionModule !== null) { $this->encryptedPaths->set($this->util->stripPartialFileExtension($path), true); $headerSize = $this->getHeaderSize($path); + if ($mode === 'r' && $headerSize === 0) { + $firstBlock = $this->readFirstBlock($path); + if (!$firstBlock) { + throw new InvalidHeaderException("Unable to get header block for $path"); + } elseif (!str_starts_with($firstBlock, Util::HEADER_START)) { + throw new InvalidHeaderException("Unable to get header size for $path, file doesn't start with encryption header"); + } else { + throw new InvalidHeaderException("Unable to get header size for $path, even though file does start with encryption header"); + } + } $source = $this->storage->fopen($path, $mode); if (!is_resource($source)) { return false; @@ -526,10 +537,22 @@ class Encryption extends Wrapper { $result = $this->copyBetweenStorage($sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, true); if ($result) { - if ($sourceStorage->is_dir($sourceInternalPath)) { - $result = $sourceStorage->rmdir($sourceInternalPath); - } else { - $result = $sourceStorage->unlink($sourceInternalPath); + $setPreserveCacheOnDelete = $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && !$this->instanceOfStorage(ObjectStoreStorage::class); + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + try { + if ($sourceStorage->is_dir($sourceInternalPath)) { + $result = $sourceStorage->rmdir($sourceInternalPath); + } else { + $result = $sourceStorage->unlink($sourceInternalPath); + } + } finally { + if ($setPreserveCacheOnDelete) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } } } return $result; @@ -654,7 +677,7 @@ class Encryption extends Wrapper { if (is_resource($dh)) { while ($result && ($file = readdir($dh)) !== false) { if (!Filesystem::isIgnoredDir($file)) { - $result &= $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file, false, $isRename); + $result = $this->copyFromStorage($sourceStorage, $sourceInternalPath . '/' . $file, $targetInternalPath . '/' . $file, $preserveMtime, $isRename); } } } @@ -662,7 +685,7 @@ class Encryption extends Wrapper { try { $source = $sourceStorage->fopen($sourceInternalPath, 'r'); $target = $this->fopen($targetInternalPath, 'w'); - [, $result] = \OC_Helper::streamCopy($source, $target); + [, $result] = Files::streamCopy($source, $target, true); } finally { if (is_resource($source)) { fclose($source); @@ -871,7 +894,7 @@ class Encryption extends Wrapper { public function writeStream(string $path, $stream, ?int $size = null): int { // always fall back to fopen $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + [$count, $result] = Files::streamCopy($stream, $target, true); fclose($stream); fclose($target); @@ -894,4 +917,16 @@ class Encryption extends Wrapper { public function setEnabled(bool $enabled): void { $this->enabled = $enabled; } + + /** + * Check if the on-disk data for a file has a valid encrypted header + * + * @param string $path + * @return bool + */ + public function hasValidHeader(string $path): bool { + $firstBlock = $this->readFirstBlock($path); + $header = $this->util->parseRawHeader($firstBlock); + return (count($header) > 0); + } } diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php index c8db0e94e59..38b113cef88 100644 --- a/lib/private/Files/Storage/Wrapper/Jail.php +++ b/lib/private/Files/Storage/Wrapper/Jail.php @@ -11,6 +11,7 @@ use OC\Files\Cache\Wrapper\CacheJail; use OC\Files\Cache\Wrapper\JailPropagator; use OC\Files\Cache\Wrapper\JailWatcher; use OC\Files\Filesystem; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IWatcher; @@ -253,7 +254,7 @@ class Jail extends Wrapper { return $storage->writeStream($this->getUnjailedPath($path), $stream, $size); } else { $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + $count = Files::streamCopy($stream, $target); fclose($stream); fclose($target); return $count; diff --git a/lib/private/Files/Storage/Wrapper/Quota.php b/lib/private/Files/Storage/Wrapper/Quota.php index 3be77ba1b37..35a265f8c8e 100644 --- a/lib/private/Files/Storage/Wrapper/Quota.php +++ b/lib/private/Files/Storage/Wrapper/Quota.php @@ -21,6 +21,7 @@ class Quota extends Wrapper { protected string $sizeRoot; private SystemConfig $config; private bool $quotaIncludeExternalStorage; + private bool $enabled = true; /** * @param array $parameters @@ -46,6 +47,9 @@ class Quota extends Wrapper { } private function hasQuota(): bool { + if (!$this->enabled) { + return false; + } return $this->getQuota() !== FileInfo::SPACE_UNLIMITED; } @@ -197,4 +201,8 @@ class Quota extends Wrapper { return parent::touch($path, $mtime); } + + public function enableQuota(bool $enabled): void { + $this->enabled = $enabled; + } } diff --git a/lib/private/Files/Storage/Wrapper/Wrapper.php b/lib/private/Files/Storage/Wrapper/Wrapper.php index 6dea439fe25..7af11dd5ef7 100644 --- a/lib/private/Files/Storage/Wrapper/Wrapper.php +++ b/lib/private/Files/Storage/Wrapper/Wrapper.php @@ -9,6 +9,7 @@ namespace OC\Files\Storage\Wrapper; use OC\Files\Storage\FailedStorage; use OC\Files\Storage\Storage; +use OCP\Files; use OCP\Files\Cache\ICache; use OCP\Files\Cache\IPropagator; use OCP\Files\Cache\IScanner; @@ -322,7 +323,7 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea return $storage->writeStream($path, $stream, $size); } else { $target = $this->fopen($path, 'w'); - [$count, $result] = \OC_Helper::streamCopy($stream, $target); + $count = Files::streamCopy($stream, $target); fclose($stream); fclose($target); return $count; diff --git a/lib/private/Files/Template/TemplateManager.php b/lib/private/Files/Template/TemplateManager.php index 8032cdf941f..80ef5a14786 100644 --- a/lib/private/Files/Template/TemplateManager.php +++ b/lib/private/Files/Template/TemplateManager.php @@ -19,6 +19,7 @@ use OCP\Files\IRootFolder; use OCP\Files\Node; use OCP\Files\NotFoundException; use OCP\Files\Template\BeforeGetTemplatesEvent; +use OCP\Files\Template\Field; use OCP\Files\Template\FileCreatedFromTemplateEvent; use OCP\Files\Template\ICustomTemplateProvider; use OCP\Files\Template\ITemplateManager; @@ -125,6 +126,19 @@ class TemplateManager implements ITemplateManager { }, $this->listCreators())); } + public function listTemplateFields(int $fileId): array { + foreach ($this->listCreators() as $creator) { + $fields = $this->getTemplateFields($creator, $fileId); + if (empty($fields)) { + continue; + } + + return $fields; + } + + return []; + } + /** * @param string $filePath * @param string $templateId @@ -187,6 +201,20 @@ class TemplateManager implements ITemplateManager { * @return list<Template> */ private function getTemplateFiles(TemplateFileCreator $type): array { + $templates = array_merge( + $this->getProviderTemplates($type), + $this->getUserTemplates($type) + ); + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($templates, false)); + + return $templates; + } + + /** + * @return list<Template> + */ + private function getProviderTemplates(TemplateFileCreator $type): array { $templates = []; foreach ($this->getRegisteredProviders() as $provider) { foreach ($type->getMimetypes() as $mimetype) { @@ -195,11 +223,22 @@ class TemplateManager implements ITemplateManager { } } } + + return $templates; + } + + /** + * @return list<Template> + */ + private function getUserTemplates(TemplateFileCreator $type): array { + $templates = []; + try { $userTemplateFolder = $this->getTemplateFolder(); } catch (\Exception $e) { return $templates; } + foreach ($type->getMimetypes() as $mimetype) { foreach ($userTemplateFolder->searchByMime($mimetype) as $templateFile) { $template = new Template( @@ -212,11 +251,33 @@ class TemplateManager implements ITemplateManager { } } - $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($templates)); - return $templates; } + /* + * @return list<Field> + */ + private function getTemplateFields(TemplateFileCreator $type, int $fileId): array { + $providerTemplates = $this->getProviderTemplates($type); + $userTemplates = $this->getUserTemplates($type); + + $matchedTemplates = array_filter( + array_merge($providerTemplates, $userTemplates), + function (Template $template) use ($fileId) { + return $template->jsonSerialize()['fileid'] === $fileId; + }); + + if (empty($matchedTemplates)) { + return []; + } + + $this->eventDispatcher->dispatchTyped(new BeforeGetTemplatesEvent($matchedTemplates, true)); + + return array_values(array_map(function (Template $template) { + return $template->jsonSerialize()['fields'] ?? []; + }, $matchedTemplates)); + } + /** * @param Node|File $file * @return array diff --git a/lib/private/Files/Type/Detection.php b/lib/private/Files/Type/Detection.php index 42315247dbf..d5810a90868 100644 --- a/lib/private/Files/Type/Detection.php +++ b/lib/private/Files/Type/Detection.php @@ -9,6 +9,7 @@ declare(strict_types=1); namespace OC\Files\Type; use OCP\Files\IMimeTypeDetector; +use OCP\IBinaryFinder; use OCP\ITempManager; use OCP\IURLGenerator; use Psr\Log\LoggerInterface; @@ -23,6 +24,7 @@ use Psr\Log\LoggerInterface; class Detection implements IMimeTypeDetector { private const CUSTOM_MIMETYPEMAPPING = 'mimetypemapping.json'; private const CUSTOM_MIMETYPEALIASES = 'mimetypealiases.json'; + private const CUSTOM_MIMETYPENAMES = 'mimetypenames.json'; /** @var array<list{string, string|null}> */ protected array $mimeTypes = []; @@ -31,6 +33,8 @@ class Detection implements IMimeTypeDetector { protected array $mimeTypeIcons = []; /** @var array<string,string> */ protected array $mimeTypeAlias = []; + /** @var array<string,string> */ + protected array $mimeTypesNames = []; public function __construct( private IURLGenerator $urlGenerator, @@ -148,6 +152,25 @@ class Detection implements IMimeTypeDetector { return $this->mimeTypes; } + private function loadNamings(): void { + if (!empty($this->mimeTypesNames)) { + return; + } + + $mimeTypeMapping = json_decode(file_get_contents($this->defaultConfigDir . '/mimetypenames.dist.json'), true); + $mimeTypeMapping = $this->loadCustomDefinitions(self::CUSTOM_MIMETYPENAMES, $mimeTypeMapping); + + $this->mimeTypesNames = $mimeTypeMapping; + } + + /** + * @return array<string,string> + */ + public function getAllNamings(): array { + $this->loadNamings(); + return $this->mimeTypesNames; + } + /** * detect MIME type only based on filename, content of file is not used * @@ -225,11 +248,13 @@ class Detection implements IMimeTypeDetector { } } - if (\OC_Helper::canExecute('file')) { + $binaryFinder = \OCP\Server::get(IBinaryFinder::class); + $program = $binaryFinder->findBinaryPath('file'); + if ($program !== false) { // it looks like we have a 'file' command, // lets see if it does have mime support $path = escapeshellarg($path); - $fp = popen("test -f $path && file -b --mime-type $path", 'r'); + $fp = popen("test -f $path && $program -b --mime-type $path", 'r'); if ($fp !== false) { $mimeType = fgets($fp); pclose($fp); diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 4d94629443f..e9ed351b27b 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -29,6 +29,7 @@ use OCP\Files\Storage\IStorage; use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; use Psr\Log\LoggerInterface; /** @@ -260,7 +261,15 @@ class Scanner extends PublicEmitter { try { $propagator = $storage->getPropagator(); $propagator->beginBatch(); - $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + try { + $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + } catch (LockedException $e) { + if (is_string($e->getReadablePath()) && str_starts_with($e->getReadablePath(), 'scanner::')) { + throw new LockedException("scanner::$dir", $e, $e->getExistingLock()); + } else { + throw $e; + } + } $cache = $storage->getCache(); if ($cache instanceof Cache) { // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index bbad24d3e43..63eecf5e1d6 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -10,12 +10,14 @@ namespace OC\Files; use Icewind\Streams\CallbackWrapper; use OC\Files\Mount\MoveableMount; use OC\Files\Storage\Storage; +use OC\Files\Storage\Wrapper\Quota; use OC\Share\Share; use OC\User\LazyUser; use OC\User\Manager as UserManager; use OC\User\User; use OCA\Files_Sharing\SharedMount; use OCP\Constants; +use OCP\Files; use OCP\Files\Cache\ICacheEntry; use OCP\Files\ConnectionLostException; use OCP\Files\EmptyFileNameException; @@ -628,7 +630,7 @@ class View { [$storage, $internalPath] = $this->resolvePath($path); $target = $storage->fopen($internalPath, 'w'); if ($target) { - [, $result] = \OC_Helper::streamCopy($data, $target); + [, $result] = Files::streamCopy($data, $target, true); fclose($target); fclose($data); @@ -936,7 +938,7 @@ class View { try { $exists = $this->file_exists($target); - if ($this->shouldEmitHooks()) { + if ($this->shouldEmitHooks($target)) { \OC_Hook::emit( Filesystem::CLASSNAME, Filesystem::signal_copy, @@ -976,7 +978,7 @@ class View { $this->changeLock($target, ILockingProvider::LOCK_SHARED); $lockTypePath2 = ILockingProvider::LOCK_SHARED; - if ($this->shouldEmitHooks() && $result !== false) { + if ($this->shouldEmitHooks($target) && $result !== false) { \OC_Hook::emit( Filesystem::CLASSNAME, Filesystem::signal_post_copy, @@ -1466,8 +1468,7 @@ class View { public function addSubMounts(FileInfo $info, $extOnly = false): void { $mounts = Filesystem::getMountManager()->findIn($info->getPath()); $info->setSubMounts(array_filter($mounts, function (IMountPoint $mount) use ($extOnly) { - $subStorage = $mount->getStorage(); - return !($extOnly && $subStorage instanceof \OCA\Files_Sharing\SharedStorage); + return !($extOnly && $mount instanceof SharedMount); })); } @@ -1579,12 +1580,22 @@ class View { // Create parent folders if the mountpoint is inside a subfolder that doesn't exist yet if (!isset($files[$entryName])) { try { + [$storage, ] = $this->resolvePath($path . '/' . $entryName); + // make sure we can create the mountpoint folder, even if the user has a quota of 0 + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(false); + } + if ($this->mkdir($path . '/' . $entryName) !== false) { $info = $this->getFileInfo($path . '/' . $entryName); if ($info !== false) { $files[$entryName] = $info; } } + + if ($storage->instanceOfStorage(Quota::class)) { + $storage->enableQuota(true); + } } catch (\Exception $e) { // Creating the parent folder might not be possible, for example due to a lack of permissions. $this->logger->debug('Failed to create non-existent parent', ['exception' => $e, 'path' => $path . '/' . $entryName]); |