From: Arthur Schiwon <blizzz@arthur-schiwon.de>
Date: Wed, 19 Oct 2022 21:22:22 +0000 (+0200)
Subject: optimize background image migration job
X-Git-Tag: v26.0.0beta1~384^2~7
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b725c777aa4bd80c6ace3695733b560a320e1250;p=nextcloud-server.git

optimize background image migration job

- separate in two stages: to prepare, and to actually migrate
- in step one, prepare a list of users to be migrated, and store it
  compressed as app config
  - gzcompress can be used, because we already require zlib
- upon the next calls (step two), slice off the first 5000 users
  and migrate them. Re-add job if necessary to repeat.
- downside is that an app config value will in the beginning use the
  RAM with any request, until it thins out. Examples: 2m UUIDs (75 MiB)
  result in ~40 MiB compressed data, while 0.2Mib for 10 000 UUIDs,
  0.4MiB for 20 000 and 4.1 MiB for 200 000. Acceptable.

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
---

diff --git a/apps/theming/lib/Jobs/MigrateBackgroundImages.php b/apps/theming/lib/Jobs/MigrateBackgroundImages.php
index b816a4c8775..802b15da6c7 100644
--- a/apps/theming/lib/Jobs/MigrateBackgroundImages.php
+++ b/apps/theming/lib/Jobs/MigrateBackgroundImages.php
@@ -36,45 +36,88 @@ use OCP\Files\NotFoundException;
 use OCP\Files\NotPermittedException;
 use OCP\Files\SimpleFS\ISimpleFolder;
 use OCP\IConfig;
+use OCP\IDBConnection;
 
 class MigrateBackgroundImages extends QueuedJob {
 	public const TIME_SENSITIVE = 0;
 
+	protected const STAGE_PREPARE = 'prepare';
+	protected const STAGE_EXECUTE = 'execute';
+
 	private IConfig $config;
 	private IAppManager $appManager;
 	private IAppDataFactory $appDataFactory;
 	private IJobList $jobList;
-
-	public function __construct(ITimeFactory $time, IAppDataFactory $appDataFactory, IConfig $config, IAppManager $appManager, IJobList $jobList) {
+	private IDBConnection $dbc;
+
+	public function __construct(
+		ITimeFactory $time,
+		IAppDataFactory $appDataFactory,
+		IConfig $config,
+		IAppManager $appManager,
+		IJobList $jobList,
+		IDBConnection $dbc
+	) {
 		parent::__construct($time);
 		$this->config = $config;
 		$this->appManager = $appManager;
 		$this->appDataFactory = $appDataFactory;
 		$this->jobList = $jobList;
+		$this->dbc = $dbc;
 	}
 
 	protected function run($argument): void {
-		if (!$this->appManager->isEnabledForUser('dashboard')) {
-			return;
+		if (!isset($argument['stage'])) {
+			// not executed in 25.0.0?!
+			$argument['stage'] = 'prepare';
 		}
 
-		$dashboardData = $this->appDataFactory->get('dashboard');
+		switch ($argument['stage']) {
+			case self::STAGE_PREPARE:
+				$this->runPreparation();
+				break;
+			case self::STAGE_EXECUTE:
+				$this->runMigration();
+				break;
+			default:
+				break;
+		}
+	}
 
-		$userIds = $this->config->getUsersForUserValue('theming', 'background', 'custom');
+	protected function runPreparation(): void {
+		try {
+			$selector = $this->dbc->getQueryBuilder();
+			$result = $selector->select('userid')
+				->from('preferences')
+				->where($selector->expr()->eq('appid', $selector->createNamedParameter('theming')))
+				->andWhere($selector->expr()->eq('configkey', $selector->createNamedParameter('background')))
+				->andWhere($selector->expr()->eq('configvalue', $selector->createNamedParameter('custom')))
+				->executeQuery();
+
+			$userIds = $result->fetchAll(\PDO::FETCH_COLUMN);
+			$this->storeUserIdsToProcess($userIds);
+		} catch (\Throwable $t) {
+			$this->jobList->add(self::class, self::STAGE_PREPARE);
+			throw $t;
+		}
+		$this->jobList->add(self::class, self::STAGE_EXECUTE);
+	}
 
-		$notSoFastMode = \count($userIds) > 5000;
-		$reTrigger = false;
-		$processed = 0;
+	protected function runMigration(): void {
+		$storedUserIds = $this->config->getAppValue('theming', '25_background_image_migration');
+		if ($storedUserIds === '') {
+			return;
+		}
+		$allUserIds = \json_decode(\gzuncompress($storedUserIds), true);
+		$notSoFastMode = isset($allUserIds[5000]);
+		$dashboardData = $this->appDataFactory->get('dashboard');
 
+		$userIds = $notSoFastMode ? array_slice($allUserIds, 0, 5000) : $allUserIds;
 		foreach ($userIds as $userId) {
 			try {
 				// precondition
-				if ($notSoFastMode) {
-					if ($this->config->getUserValue($userId, 'theming', 'background-migrated', '0') === '1') {
-						// already migrated
-						continue;
-					}
-					$reTrigger = true;
+				if (!$this->appManager->isEnabledForUser('dashboard', $userId)) {
+					continue;
 				}
 
 				// migration
@@ -87,21 +130,22 @@ class MigrateBackgroundImages extends QueuedJob {
 				$file->delete();
 			} catch (NotFoundException|NotPermittedException $e) {
 			}
-			// capture state
-			if ($notSoFastMode) {
-				$this->config->setUserValue($userId, 'theming', 'background-migrated', '1');
-				$processed++;
-			}
-			if ($processed > 4999) {
-				break;
-			}
 		}
 
-		if ($reTrigger) {
-			$this->jobList->add(self::class);
+		if ($notSoFastMode) {
+			$remainingUserIds = array_slice($allUserIds, 5000);
+			$this->storeUserIdsToProcess($remainingUserIds);
+			$this->jobList->add(self::class, ['stage' => self::STAGE_EXECUTE]);
+		} else {
+			$this->config->deleteAppValue('theming', '25_background_image_migration');
 		}
 	}
 
+	protected function storeUserIdsToProcess(array $userIds): void {
+		$storableUserIds = \gzcompress(\json_encode($userIds), 9);
+		$this->config->setAppValue('theming', '25_background_image_migration', $storableUserIds);
+	}
+
 	/**
 	 * Get the root location for users theming data
 	 */
diff --git a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
index c23a9176843..04c6a84d19a 100644
--- a/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
+++ b/apps/theming/lib/Migration/InitBackgroundImagesMigration.php
@@ -43,6 +43,6 @@ class InitBackgroundImagesMigration implements \OCP\Migration\IRepairStep {
 	}
 
 	public function run(IOutput $output) {
-		$this->jobList->add(MigrateBackgroundImages::class);
+		$this->jobList->add(MigrateBackgroundImages::class, ['stage' => 'prepare']);
 	}
 }