optimize background image migration job

- separate in two stages: to prepare, and to actually migrate
- in step one, prepare a list of users to be migrated, and store it
  compressed as app config
  - gzcompress can be used, because we already require zlib
- upon the next calls (step two), slice off the first 5000 users
  and migrate them. Re-add job if necessary to repeat.
- downside is that an app config value will in the beginning use the
  RAM with any request, until it thins out. Examples: 2m UUIDs (75 MiB)
  result in ~40 MiB compressed data, while 0.2Mib for 10 000 UUIDs,
  0.4MiB for 20 000 and 4.1 MiB for 200 000. Acceptable.

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>

diff --git a/apps/theming/lib/Jobs/MigrateBackgroundImages.php b/apps/theming/lib/Jobs/MigrateBackgroundImages.php
index b816a4c8775afd61c50bdc5e9de8b2f8878961fc..802b15da6c7691a4827f926d770c872e87c45af4 100644 (file)
--- a/apps/theming/lib/Jobs/MigrateBackgroundImages.php
+++ b/apps/theming/lib/Jobs/MigrateBackgroundImages.php
@@ -36,45 +36,88 @@ use OCP\Files\NotFoundException;
 use OCP\Files\NotPermittedException;
 use OCP\Files\SimpleFS\ISimpleFolder;
 use OCP\IConfig;
+use OCP\IDBConnection;
 
 class MigrateBackgroundImages extends QueuedJob {
        public const TIME_SENSITIVE = 0;
 
+       protected const STAGE_PREPARE = 'prepare';
+       protected const STAGE_EXECUTE = 'execute';
+
        private IConfig $config;
        private IAppManager $appManager;
        private IAppDataFactory $appDataFactory;
        private IJobList $jobList;
-
-       public function __construct(ITimeFactory $time, IAppDataFactory $appDataFactory, IConfig $config, IAppManager $appManager, IJobList $jobList) {
+       private IDBConnection $dbc;
+
+       public function __construct(
+               ITimeFactory $time,
+               IAppDataFactory $appDataFactory,
+               IConfig $config,
+               IAppManager $appManager,
+               IJobList $jobList,
+               IDBConnection $dbc
+       ) {
                parent::__construct($time);
                $this->config = $config;
                $this->appManager = $appManager;
                $this->appDataFactory = $appDataFactory;
                $this->jobList = $jobList;
+               $this->dbc = $dbc;
        }
 
        protected function run($argument): void {
-               if (!$this->appManager->isEnabledForUser('dashboard')) {
-                       return;
+               if (!isset($argument['stage'])) {
+                       // not executed in 25.0.0?!
+                       $argument['stage'] = 'prepare';
                }
 
-               $dashboardData = $this->appDataFactory->get('dashboard');
+               switch ($argument['stage']) {
+                       case self::STAGE_PREPARE:
+                               $this->runPreparation();
+                               break;
+                       case self::STAGE_EXECUTE:
+                               $this->runMigration();
+                               break;
+                       default:
+                               break;
+               }
+       }
 
-               $userIds = $this->config->getUsersForUserValue('theming', 'background', 'custom');
+       protected function runPreparation(): void {
+               try {
+                       $selector = $this->dbc->getQueryBuilder();
+                       $result = $selector->select('userid')
+                               ->from('preferences')
+                               ->where($selector->expr()->eq('appid', $selector->createNamedParameter('theming')))
+                               ->andWhere($selector->expr()->eq('configkey', $selector->createNamedParameter('background')))
+                               ->andWhere($selector->expr()->eq('configvalue', $selector->createNamedParameter('custom')))
+                               ->executeQuery();
+
+                       $userIds = $result->fetchAll(\PDO::FETCH_COLUMN);
+                       $this->storeUserIdsToProcess($userIds);
+               } catch (\Throwable $t) {
+                       $this->jobList->add(self::class, self::STAGE_PREPARE);
+                       throw $t;
+               }
+               $this->jobList->add(self::class, self::STAGE_EXECUTE);
+       }
 
-               $notSoFastMode = \count($userIds) > 5000;
-               $reTrigger = false;
-               $processed = 0;
+       protected function runMigration(): void {
+               $storedUserIds = $this->config->getAppValue('theming', '25_background_image_migration');
+               if ($storedUserIds === '') {
+                       return;
+               }
+               $allUserIds = \json_decode(\gzuncompress($storedUserIds), true);
+               $notSoFastMode = isset($allUserIds[5000]);
+               $dashboardData = $this->appDataFactory->get('dashboard');
 
+               $userIds = $notSoFastMode ? array_slice($allUserIds, 0, 5000) : $allUserIds;
                foreach ($userIds as $userId) {
                        try {
                                // precondition
-                               if ($notSoFastMode) {
-                                       if ($this->config->getUserValue($userId, 'theming', 'background-migrated', '0') === '1') {
-                                               // already migrated
-                                               continue;
-                                       }
-                                       $reTrigger = true;
+                               if (!$this->appManager->isEnabledForUser('dashboard', $userId)) {
+                                       continue;
                                }
 
                                // migration
@@ -87,21 +130,22 @@ class MigrateBackgroundImages extends QueuedJob {
                                $file->delete();
                        } catch (NotFoundException|NotPermittedException $e) {
                        }
-                       // capture state
-                       if ($notSoFastMode) {
-                               $this->config->setUserValue($userId, 'theming', 'background-migrated', '1');
-                               $processed++;
-                       }
-                       if ($processed > 4999) {
-                               break;
-                       }
                }
 
-               if ($reTrigger) {
-                       $this->jobList->add(self::class);
+               if ($notSoFastMode) {
+                       $remainingUserIds = array_slice($allUserIds, 5000);
+                       $this->storeUserIdsToProcess($remainingUserIds);
+                       $this->jobList->add(self::class, ['stage' => self::STAGE_EXECUTE]);
+               } else {
+                       $this->config->deleteAppValue('theming', '25_background_image_migration');
                }
        }
 
+       protected function storeUserIdsToProcess(array $userIds): void {
+               $storableUserIds = \gzcompress(\json_encode($userIds), 9);
+               $this->config->setAppValue('theming', '25_background_image_migration', $storableUserIds);
+       }
+
        /**
         * Get the root location for users theming data
         */

diff --git a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
index c23a9176843b16c784712ca383904f419454103f..04c6a84d19a9affb07589129ca5ebc27a1e5a621 100644 (file)
--- a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
+++ b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
@@ -43,6 +43,6 @@ class InitBackgroundImagesMigration implements \OCP\Migration\IRepairStep {
        }
 
        public function run(IOutput $output) {
-               $this->jobList->add(MigrateBackgroundImages::class);
+               $this->jobList->add(MigrateBackgroundImages::class, ['stage' => 'prepare']);
        }
 }