From 438ac23e2a3564fbce50e862336db56bbf0d747c Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2020 22:09:19 +0200 Subject: [PATCH] Distribute preview folders in appdata in multibucket setup to multiple buckets * introduces a new IRootMountProvider to register mount points inside the root storage * adds a AppdataPreviewObjectStoreStorage to handle the split between preview folders and bucket number Ref #22033 Signed-off-by: Morris Jobke --- lib/composer/composer/autoload_classmap.php | 3 + lib/composer/composer/autoload_static.php | 3 + .../Files/Config/MountProviderCollection.php | 19 ++++ .../ObjectStorePreviewCacheMountProvider.php | 102 ++++++++++++++++++ .../AppdataPreviewObjectStoreStorage.php | 44 ++++++++ lib/private/Preview/Storage/Root.php | 1 - lib/private/Server.php | 3 + lib/private/legacy/OC_Util.php | 11 ++ .../Files/Config/IRootMountProvider.php | 41 +++++++ 9 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php create mode 100644 lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php create mode 100644 lib/public/Files/Config/IRootMountProvider.php diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 254840b3542..65542e477e7 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -230,6 +230,7 @@ return array( 'OCP\\Files\\Config\\IHomeMountProvider' => $baseDir . '/lib/public/Files/Config/IHomeMountProvider.php', 'OCP\\Files\\Config\\IMountProvider' => $baseDir . '/lib/public/Files/Config/IMountProvider.php', 'OCP\\Files\\Config\\IMountProviderCollection' => $baseDir . '/lib/public/Files/Config/IMountProviderCollection.php', + 'OCP\\Files\\Config\\IRootMountProvider' => $baseDir . '/lib/public/Files/Config/IRootMountProvider.php', 'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php', 'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php', 'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php', @@ -1029,6 +1030,7 @@ return array( 'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', + 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php', 'OC\\Files\\Node\\HookConnector' => $baseDir . '/lib/private/Files/Node/HookConnector.php', @@ -1040,6 +1042,7 @@ return array( 'OC\\Files\\Node\\Root' => $baseDir . '/lib/private/Files/Node/Root.php', 'OC\\Files\\Notify\\Change' => $baseDir . '/lib/private/Files/Notify/Change.php', 'OC\\Files\\Notify\\RenameChange' => $baseDir . '/lib/private/Files/Notify/RenameChange.php', + 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 45bf93241d5..835603a3340 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -259,6 +259,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OCP\\Files\\Config\\IHomeMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IHomeMountProvider.php', 'OCP\\Files\\Config\\IMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProvider.php', 'OCP\\Files\\Config\\IMountProviderCollection' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProviderCollection.php', + 'OCP\\Files\\Config\\IRootMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IRootMountProvider.php', 'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php', 'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php', 'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php', @@ -1058,6 +1059,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', + 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php', 'OC\\Files\\Node\\HookConnector' => __DIR__ . '/../../..' . '/lib/private/Files/Node/HookConnector.php', @@ -1069,6 +1071,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Node\\Root' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Root.php', 'OC\\Files\\Notify\\Change' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/Change.php', 'OC\\Files\\Notify\\RenameChange' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/RenameChange.php', + 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php', diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php index 34db652290f..2b57ffe6e4c 100644 --- a/lib/private/Files/Config/MountProviderCollection.php +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -30,6 +30,7 @@ use OC\Hooks\EmitterTrait; use OCP\Files\Config\IHomeMountProvider; use OCP\Files\Config\IMountProvider; use OCP\Files\Config\IMountProviderCollection; +use OCP\Files\Config\IRootMountProvider; use OCP\Files\Config\IUserMountCache; use OCP\Files\Mount\IMountManager; use OCP\Files\Mount\IMountPoint; @@ -49,6 +50,9 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { */ private $providers = []; + /** @var \OCP\Files\Config\IRootMountProvider[] */ + private $rootProviders = []; + /** * @var \OCP\Files\Storage\IStorageFactory */ @@ -198,4 +202,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { public function getMountCache() { return $this->mountCache; } + + public function registerRootProvider(IRootMountProvider $provider) { + $this->rootProviders[] = $provider; + } + + public function getRootMounts(): array { + $loader = $this->loader; + $mounts = array_map(function (IRootMountProvider $provider) use ($loader) { + return $provider->getRootMounts($loader); + }, $this->rootProviders); + $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) { + return array_merge($mounts, $providerMounts); + }, []); + return $mounts; + } } diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php new file mode 100644 index 00000000000..a4acdb6bb0f --- /dev/null +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -0,0 +1,102 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Mount; + +use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; +use OCP\ILogger; + +/** + * Mount provider for object store app data folder for previews + */ +class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { + /** @var ILogger */ + private $logger; + /** @var IConfig */ + private $config; + + public function __construct(ILogger $logger, IConfig $config) { + $this->logger = $logger; + $this->config = $config; + } + + public function getRootMounts(IStorageFactory $loader): array { + if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { + return []; + } + + $instanceId = $this->config->getSystemValueString('instanceid', ''); + $mountPoints = []; + $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + $i = 0; + foreach ($directoryRange as $parent) { + foreach ($directoryRange as $child) { + $mountPoints[] = new MountPoint( + AppdataPreviewObjectStoreStorage::class, + '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, + $this->getMultiBucketObjectStore($i), + $loader + ); + $i++; + } + } + return $mountPoints; + } + + /** + * @return array + */ + protected function getMultiBucketObjectStore(int $number): array { + $config = $this->config->getSystemValue('objectstore_multibucket'); + + // sanity checks + if (empty($config['class'])) { + $this->logger->error('No class given for objectstore', ['app' => 'files']); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $config['arguments']['bucket'] .= (string)($number % $numBuckets); + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + $config['arguments']['internal-id'] = $number; + + return $config['arguments']; + } +} diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php new file mode 100644 index 00000000000..ab6cb07cdc3 --- /dev/null +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -0,0 +1,44 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { + + /** @var string */ + private $internalId; + + public function __construct($params) { + if (!isset($params['internal-id'])) { + throw new \Exception('missing id in parameters'); + } + $this->internalId = (string)$params['internal-id']; + parent::__construct($params); + } + + public function getId() { + return 'object::appdata::preview:' . $this->internalId; + } +} diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php index a9a72026a51..107d87c6301 100644 --- a/lib/private/Preview/Storage/Root.php +++ b/lib/private/Preview/Storage/Root.php @@ -1,7 +1,6 @@ * diff --git a/lib/private/Server.php b/lib/private/Server.php index e054d8230d9..8d771bec3a1 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -88,6 +88,7 @@ use OC\Files\Config\UserMountCacheListener; use OC\Files\Mount\CacheMountProvider; use OC\Files\Mount\LocalHomeMountProvider; use OC\Files\Mount\ObjectHomeMountProvider; +use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; use OC\Files\Node\HookConnector; use OC\Files\Node\LazyRoot; use OC\Files\Node\Root; @@ -904,9 +905,11 @@ class Server extends ServerContainer implements IServerContainer { // builtin providers $config = $c->getConfig(); + $logger = $c->getLogger(); $manager->registerProvider(new CacheMountProvider($config)); $manager->registerHomeProvider(new LocalHomeMountProvider()); $manager->registerHomeProvider(new ObjectHomeMountProvider($config)); + $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config)); return $manager; }); diff --git a/lib/private/legacy/OC_Util.php b/lib/private/legacy/OC_Util.php index ab386ab6172..fd55962447e 100644 --- a/lib/private/legacy/OC_Util.php +++ b/lib/private/legacy/OC_Util.php @@ -297,6 +297,17 @@ class OC_Util { self::initLocalStorageRootFS(); } + /** @var \OCP\Files\Config\IMountProviderCollection $mountProviderCollection */ + $mountProviderCollection = \OC::$server->query(\OCP\Files\Config\IMountProviderCollection::class); + /** @var \OCP\Files\Mount\IMountPoint[] $rootMountProviders */ + $rootMountProviders = $mountProviderCollection->getRootMounts(); + + /** @var \OC\Files\Mount\Manager $mountManager */ + $mountManager = \OC\Files\Filesystem::getMountManager(); + foreach ($rootMountProviders as $rootMountProvider) { + $mountManager->addMount($rootMountProvider); + } + if ($user != '' && !\OC::$server->getUserManager()->userExists($user)) { \OC::$server->getEventLogger()->end('setup_fs'); return false; diff --git a/lib/public/Files/Config/IRootMountProvider.php b/lib/public/Files/Config/IRootMountProvider.php new file mode 100644 index 00000000000..0f7b0eca3d4 --- /dev/null +++ b/lib/public/Files/Config/IRootMountProvider.php @@ -0,0 +1,41 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\Files\Config; + +use OCP\Files\Storage\IStorageFactory; + +/** + * @since 20.0.0 + */ +interface IRootMountProvider { + /** + * Get all root mountpoints + * + * @return \OCP\Files\Mount\IMountPoint[] + * @since 20.0.0 + */ + public function getRootMounts(IStorageFactory $loader): array; +} -- 2.39.5