From b725c777aa4bd80c6ace3695733b560a320e1250 Mon Sep 17 00:00:00 2001 From: Arthur Schiwon Date: Wed, 19 Oct 2022 23:22:22 +0200 Subject: [PATCH] optimize background image migration job - separate in two stages: to prepare, and to actually migrate - in step one, prepare a list of users to be migrated, and store it compressed as app config - gzcompress can be used, because we already require zlib - upon the next calls (step two), slice off the first 5000 users and migrate them. Re-add job if necessary to repeat. - downside is that an app config value will in the beginning use the RAM with any request, until it thins out. Examples: 2m UUIDs (75 MiB) result in ~40 MiB compressed data, while 0.2Mib for 10 000 UUIDs, 0.4MiB for 20 000 and 4.1 MiB for 200 000. Acceptable. Signed-off-by: Arthur Schiwon --- .../lib/Jobs/MigrateBackgroundImages.php | 94 ++++++++++++++----- .../InitBackgroundImagesMigration.php | 2 +- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/apps/theming/lib/Jobs/MigrateBackgroundImages.php b/apps/theming/lib/Jobs/MigrateBackgroundImages.php index b816a4c8775..802b15da6c7 100644 --- a/apps/theming/lib/Jobs/MigrateBackgroundImages.php +++ b/apps/theming/lib/Jobs/MigrateBackgroundImages.php @@ -36,45 +36,88 @@ use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; +use OCP\IDBConnection; class MigrateBackgroundImages extends QueuedJob { public const TIME_SENSITIVE = 0; + protected const STAGE_PREPARE = 'prepare'; + protected const STAGE_EXECUTE = 'execute'; + private IConfig $config; private IAppManager $appManager; private IAppDataFactory $appDataFactory; private IJobList $jobList; - - public function __construct(ITimeFactory $time, IAppDataFactory $appDataFactory, IConfig $config, IAppManager $appManager, IJobList $jobList) { + private IDBConnection $dbc; + + public function __construct( + ITimeFactory $time, + IAppDataFactory $appDataFactory, + IConfig $config, + IAppManager $appManager, + IJobList $jobList, + IDBConnection $dbc + ) { parent::__construct($time); $this->config = $config; $this->appManager = $appManager; $this->appDataFactory = $appDataFactory; $this->jobList = $jobList; + $this->dbc = $dbc; } protected function run($argument): void { - if (!$this->appManager->isEnabledForUser('dashboard')) { - return; + if (!isset($argument['stage'])) { + // not executed in 25.0.0?! + $argument['stage'] = 'prepare'; } - $dashboardData = $this->appDataFactory->get('dashboard'); + switch ($argument['stage']) { + case self::STAGE_PREPARE: + $this->runPreparation(); + break; + case self::STAGE_EXECUTE: + $this->runMigration(); + break; + default: + break; + } + } - $userIds = $this->config->getUsersForUserValue('theming', 'background', 'custom'); + protected function runPreparation(): void { + try { + $selector = $this->dbc->getQueryBuilder(); + $result = $selector->select('userid') + ->from('preferences') + ->where($selector->expr()->eq('appid', $selector->createNamedParameter('theming'))) + ->andWhere($selector->expr()->eq('configkey', $selector->createNamedParameter('background'))) + ->andWhere($selector->expr()->eq('configvalue', $selector->createNamedParameter('custom'))) + ->executeQuery(); + + $userIds = $result->fetchAll(\PDO::FETCH_COLUMN); + $this->storeUserIdsToProcess($userIds); + } catch (\Throwable $t) { + $this->jobList->add(self::class, self::STAGE_PREPARE); + throw $t; + } + $this->jobList->add(self::class, self::STAGE_EXECUTE); + } - $notSoFastMode = \count($userIds) > 5000; - $reTrigger = false; - $processed = 0; + protected function runMigration(): void { + $storedUserIds = $this->config->getAppValue('theming', '25_background_image_migration'); + if ($storedUserIds === '') { + return; + } + $allUserIds = \json_decode(\gzuncompress($storedUserIds), true); + $notSoFastMode = isset($allUserIds[5000]); + $dashboardData = $this->appDataFactory->get('dashboard'); + $userIds = $notSoFastMode ? array_slice($allUserIds, 0, 5000) : $allUserIds; foreach ($userIds as $userId) { try { // precondition - if ($notSoFastMode) { - if ($this->config->getUserValue($userId, 'theming', 'background-migrated', '0') === '1') { - // already migrated - continue; - } - $reTrigger = true; + if (!$this->appManager->isEnabledForUser('dashboard', $userId)) { + continue; } // migration @@ -87,21 +130,22 @@ class MigrateBackgroundImages extends QueuedJob { $file->delete(); } catch (NotFoundException|NotPermittedException $e) { } - // capture state - if ($notSoFastMode) { - $this->config->setUserValue($userId, 'theming', 'background-migrated', '1'); - $processed++; - } - if ($processed > 4999) { - break; - } } - if ($reTrigger) { - $this->jobList->add(self::class); + if ($notSoFastMode) { + $remainingUserIds = array_slice($allUserIds, 5000); + $this->storeUserIdsToProcess($remainingUserIds); + $this->jobList->add(self::class, ['stage' => self::STAGE_EXECUTE]); + } else { + $this->config->deleteAppValue('theming', '25_background_image_migration'); } } + protected function storeUserIdsToProcess(array $userIds): void { + $storableUserIds = \gzcompress(\json_encode($userIds), 9); + $this->config->setAppValue('theming', '25_background_image_migration', $storableUserIds); + } + /** * Get the root location for users theming data */ diff --git a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php index c23a9176843..04c6a84d19a 100644 --- a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php +++ b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php @@ -43,6 +43,6 @@ class InitBackgroundImagesMigration implements \OCP\Migration\IRepairStep { } public function run(IOutput $output) { - $this->jobList->add(MigrateBackgroundImages::class); + $this->jobList->add(MigrateBackgroundImages::class, ['stage' => 'prepare']); } } -- 2.39.5