From a0ff998c0088000c0418986fea9879bc5643f9f3 Mon Sep 17 00:00:00 2001 From: Louis Chemineau Date: Thu, 16 Nov 2023 15:02:38 +0100 Subject: [PATCH] Add metadata generation job Signed-off-by: Louis Chemineau --- core/BackgroundJobs/GenerateMetadataJob.php | 111 ++++++++++++++++++ core/BackgroundJobs/MigrateMetadataJob.php | 107 ----------------- lib/composer/composer/autoload_classmap.php | 4 +- lib/composer/composer/autoload_static.php | 4 +- lib/private/Repair.php | 4 +- ...onJob.php => AddMetadataGenerationJob.php} | 12 +- 6 files changed, 123 insertions(+), 119 deletions(-) create mode 100644 core/BackgroundJobs/GenerateMetadataJob.php delete mode 100644 core/BackgroundJobs/MigrateMetadataJob.php rename lib/private/Repair/{AddMetadataMigrationJob.php => AddMetadataGenerationJob.php} (75%) diff --git a/core/BackgroundJobs/GenerateMetadataJob.php b/core/BackgroundJobs/GenerateMetadataJob.php new file mode 100644 index 00000000000..f0dfdb8de22 --- /dev/null +++ b/core/BackgroundJobs/GenerateMetadataJob.php @@ -0,0 +1,111 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\BackgroundJobs; + +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\BackgroundJob\TimedJob; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IConfig; +use OCP\IUserManager; + +class GenerateMetadataJob extends TimedJob { + public function __construct( + ITimeFactory $time, + private IConfig $config, + private IRootFolder $rootFolder, + private IUserManager $userManager, + private IFilesMetadataManager $filesMetadataManager, + private IJobList $jobList, + ) { + parent::__construct($time); + + $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); + $this->setInterval(24 * 3600); + } + + protected function run(mixed $argument): void { + $users = $this->userManager->search(''); + $lastMappedUser = $this->config->getAppValue('core', 'metadataGenerationLastHandledUser', ''); + + if ($lastMappedUser === '') { + $user = array_key_first($users); + if ($user === null) { + return; + } + + $lastMappedUser = $users[$user]->getUID(); + } + + $startTime = null; + foreach ($users as $user) { + if ($startTime === null) { + // Skip all user before lastMappedUser. + if ($lastMappedUser !== $user->getUID()) { + continue; + } + + $startTime = time(); + } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 60 * 60) { + return; + } + + $this->scanFilesForUser($user->getUID()); + $this->config->setAppValue('core', 'metadataGenerationLastHandledUser', $user->getUID()); + } + + $this->jobList->remove(GenerateMetadataJob::class); + } + + private function scanFilesForUser(string $userId): void { + $userFolder = $this->rootFolder->getUserFolder($userId); + $this->scanFolder($userFolder); + } + + private function scanFolder(Folder $folder): void { + // Do not scan share and other moveable mounts. + if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) { + return; + } + + foreach ($folder->getDirectoryListing() as $node) { + if ($node instanceof Folder) { + $this->scanFolder($node); + continue; + } + + $this->filesMetadataManager->refreshMetadata( + $node, + IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND + ); + } + } +} diff --git a/core/BackgroundJobs/MigrateMetadataJob.php b/core/BackgroundJobs/MigrateMetadataJob.php deleted file mode 100644 index 4f1856c2335..00000000000 --- a/core/BackgroundJobs/MigrateMetadataJob.php +++ /dev/null @@ -1,107 +0,0 @@ - - * - * @author Louis Chemineau - * - * @license AGPL-3.0-or-later - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OC\Core\BackgroundJobs; - -use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\IJobList; -use OCP\BackgroundJob\TimedJob; -use OCP\FilesMetadata\IFilesMetadataManager; -use OCP\IConfig; -use OCP\IDBConnection; - -class MigrateMetadataJob extends TimedJob { - public function __construct( - ITimeFactory $time, - private IConfig $config, - private IFilesMetadataManager $filesMetadataManager, - private IDBConnection $connection, - private IJobList $jobList, - ) { - parent::__construct($time); - - $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); - $this->setInterval(24 * 3600); - } - - protected function run(mixed $argument): void { - if (!$this->connection->tableExists('file_metadata')) { - return; - } - - $startTime = time(); - - $selectQuery = $this->connection->getQueryBuilder() - ->select('*') - ->from('file_metadata') - ->setMaxResults(200); - - $deleteQuery = $this->connection->getQueryBuilder(); - $deleteQuery->delete('file_metadata') - ->where($deleteQuery->expr()->eq('id', $deleteQuery->createParameter('id'))) - ->where($deleteQuery->expr()->eq('group_name', $deleteQuery->createParameter('group_name'))) - ->where($deleteQuery->expr()->eq('value', $deleteQuery->createParameter('value'))); - - do { - $this->connection->beginTransaction(); - - $results = $selectQuery->executeQuery(); - - while ($row = $results->fetch()) { - $metadata = $this->filesMetadataManager->getMetadata($row['id'], true); - - switch ($row['group_name']) { - case 'size': - $metadata->setArray('photos-size', json_decode($row['value'], true)); - break; - case 'gps': - $metadata->setArray('photos-gps', json_decode($row['value'], true)); - break; - case 'photos_place': - $metadata->setString('photos-place', $row['value'], true); - break; - } - - $this->filesMetadataManager->saveMetadata($metadata); - $deleteQuery->setParameter('id', $row['id']); - $deleteQuery->setParameter('group_name', $row['group_name']); - $deleteQuery->setParameter('value', $row['value']); - $deleteQuery->executeStatement(); - } - - $results->closeCursor(); - - $this->connection->commit(); - - // Stop if execution time is more than one hour. - if (time() - $startTime > 60 * 60) { - return; - } - } while ($results->rowCount() !== 0); - - $this->connection->dropTable('file_metadata'); - $this->jobList->remove(MigrateMetadataJob::class); - } -} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 39085325067..c5c4424648d 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1002,8 +1002,8 @@ return array( 'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => $baseDir . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php', 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', + 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', - 'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => $baseDir . '/core/BackgroundJobs/MigrateMetadataJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1603,7 +1603,7 @@ return array( 'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', - 'OC\\Repair\\AddMetadataMigrationJob' => $baseDir . '/lib/private/Repair/AddMetadataMigrationJob.php', + 'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 498f4d8c0b1..0991d1062a1 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1035,8 +1035,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php', 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', + 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', - 'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MigrateMetadataJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1636,7 +1636,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', - 'OC\\Repair\\AddMetadataMigrationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataMigrationJob.php', + 'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index e70c3f7e39d..a12e00f071c 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -46,7 +46,7 @@ use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Repair\AddBruteForceCleanupJob; use OC\Repair\AddCleanupUpdaterBackupsJob; -use OC\Repair\AddMetadataMigrationJob; +use OC\Repair\AddMetadataGenerationJob; use OC\Repair\CleanTags; use OC\Repair\ClearFrontendCaches; use OC\Repair\ClearGeneratedAvatarCache; @@ -212,7 +212,7 @@ class Repair implements IOutput { \OCP\Server::get(CleanUpAbandonedApps::class), \OCP\Server::get(AddMissingSecretJob::class), \OCP\Server::get(AddRemoveOldTasksBackgroundJob::class), - \OCP\Server::get(AddMetadataMigrationJob::class), + \OCP\Server::get(AddMetadataGenerationJob::class), ]; } diff --git a/lib/private/Repair/AddMetadataMigrationJob.php b/lib/private/Repair/AddMetadataGenerationJob.php similarity index 75% rename from lib/private/Repair/AddMetadataMigrationJob.php rename to lib/private/Repair/AddMetadataGenerationJob.php index f73768721dc..72e5df03bbd 100644 --- a/lib/private/Repair/AddMetadataMigrationJob.php +++ b/lib/private/Repair/AddMetadataGenerationJob.php @@ -1,8 +1,8 @@ + * @copyright Copyright (c) 2023 Louis Chmn * - * @author Morris Jobke + * @author Louis Chmn * * @license GNU AGPL version 3 or any later version * @@ -22,22 +22,22 @@ */ namespace OC\Repair; -use OC\Core\BackgroundJobs\MigrateMetadataJob; +use OC\Core\BackgroundJobs\GenerateMetadataJob; use OCP\BackgroundJob\IJobList; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; -class AddMetadataMigrationJob implements IRepairStep { +class AddMetadataGenerationJob implements IRepairStep { public function __construct( private IJobList $jobList, ) { } public function getName() { - return 'Queue a job to migrate legacy metadata tables'; + return 'Queue a job to generate metadata'; } public function run(IOutput $output) { - $this->jobList->add(MigrateMetadataJob::class); + $this->jobList->add(GenerateMetadataJob::class); } } -- 2.39.5