From 9fe94f282f26310adca844fc2b206add91e30d6c Mon Sep 17 00:00:00 2001 From: Julius Härtl Date: Mon, 7 Dec 2020 14:30:08 +0100 Subject: Readd repair steps that are relevant when migrating from ownCloud MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d9b1492e03ab9fe58bb87baaeba745790ca15c53. Signed-off-by: Julius Härtl --- lib/private/Repair.php | 20 +++ lib/private/Repair/Owncloud/CleanPreviews.php | 73 ++++++++++ .../Repair/Owncloud/CleanPreviewsBackgroundJob.php | 132 ++++++++++++++++++ lib/private/Repair/Owncloud/InstallCoreBundle.php | 80 +++++++++++ lib/private/Repair/Owncloud/MoveAvatars.php | 73 ++++++++++ .../Repair/Owncloud/MoveAvatarsBackgroundJob.php | 147 +++++++++++++++++++++ .../Repair/Owncloud/UpdateLanguageCodes.php | 90 +++++++++++++ 7 files changed, 615 insertions(+) create mode 100644 lib/private/Repair/Owncloud/CleanPreviews.php create mode 100644 lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php create mode 100644 lib/private/Repair/Owncloud/InstallCoreBundle.php create mode 100644 lib/private/Repair/Owncloud/MoveAvatars.php create mode 100644 lib/private/Repair/Owncloud/MoveAvatarsBackgroundJob.php create mode 100644 lib/private/Repair/Owncloud/UpdateLanguageCodes.php (limited to 'lib/private') diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 847a41aeb25..4793485a384 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -34,6 +34,7 @@ namespace OC; +use OC\App\AppStore\Bundles\BundleFetcher; use OC\Avatar\AvatarManager; use OC\Repair\AddBruteForceCleanupJob; use OC\Repair\AddCleanupUpdaterBackupsJob; @@ -42,7 +43,11 @@ use OC\Repair\ClearFrontendCaches; use OC\Repair\ClearGeneratedAvatarCache; use OC\Repair\Collation; use OC\Repair\MoveUpdaterStepFile; +use OC\Repair\Owncloud\CleanPreviews; use OC\Repair\NC11\FixMountStorages; +use OC\Repair\Owncloud\MoveAvatars; +use OC\Repair\Owncloud\InstallCoreBundle; +use OC\Repair\Owncloud\UpdateLanguageCodes; use OC\Repair\NC13\AddLogRotateJob; use OC\Repair\NC14\AddPreviewBackgroundCleanupJob; use OC\Repair\NC16\AddClenupLoginFlowV2BackgroundJob; @@ -151,7 +156,22 @@ class Repair implements IOutput { new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()), new RepairInvalidShares(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()), new MoveUpdaterStepFile(\OC::$server->getConfig()), + new MoveAvatars( + \OC::$server->getJobList(), + \OC::$server->getConfig() + ), + new CleanPreviews( + \OC::$server->getJobList(), + \OC::$server->getUserManager(), + \OC::$server->getConfig() + ), new FixMountStorages(\OC::$server->getDatabaseConnection()), + new UpdateLanguageCodes(\OC::$server->getDatabaseConnection(), \OC::$server->getConfig()), + new InstallCoreBundle( + \OC::$server->query(BundleFetcher::class), + \OC::$server->getConfig(), + \OC::$server->query(Installer::class) + ), new AddLogRotateJob(\OC::$server->getJobList()), new ClearFrontendCaches(\OC::$server->getMemCacheFactory(), \OC::$server->query(SCSSCacher::class), \OC::$server->query(JSCombiner::class)), new ClearGeneratedAvatarCache(\OC::$server->getConfig(), \OC::$server->query(AvatarManager::class)), diff --git a/lib/private/Repair/Owncloud/CleanPreviews.php b/lib/private/Repair/Owncloud/CleanPreviews.php new file mode 100644 index 00000000000..5c183451d67 --- /dev/null +++ b/lib/private/Repair/Owncloud/CleanPreviews.php @@ -0,0 +1,73 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair\Owncloud; + +use OCP\BackgroundJob\IJobList; +use OCP\IConfig; +use OCP\IUser; +use OCP\IUserManager; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class CleanPreviews implements IRepairStep { + + /** @var IJobList */ + private $jobList; + + /** @var IUserManager */ + private $userManager; + + /** @var IConfig */ + private $config; + + /** + * MoveAvatars constructor. + * + * @param IJobList $jobList + * @param IUserManager $userManager + * @param IConfig $config + */ + public function __construct(IJobList $jobList, + IUserManager $userManager, + IConfig $config) { + $this->jobList = $jobList; + $this->userManager = $userManager; + $this->config = $config; + } + + /** + * @return string + */ + public function getName() { + return 'Add preview cleanup background jobs'; + } + + public function run(IOutput $output) { + if (!$this->config->getAppValue('core', 'previewsCleanedUp', false)) { + $this->userManager->callForSeenUsers(function (IUser $user) { + $this->jobList->add(CleanPreviewsBackgroundJob::class, ['uid' => $user->getUID()]); + }); + $this->config->setAppValue('core', 'previewsCleanedUp', '1'); + } + } +} diff --git a/lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php b/lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php new file mode 100644 index 00000000000..e8d89c9c7a0 --- /dev/null +++ b/lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php @@ -0,0 +1,132 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair\Owncloud; + +use OC\BackgroundJob\QueuedJob; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\ILogger; +use OCP\IUserManager; + +class CleanPreviewsBackgroundJob extends QueuedJob { + /** @var IRootFolder */ + private $rootFolder; + + /** @var ILogger */ + private $logger; + + /** @var IJobList */ + private $jobList; + + /** @var ITimeFactory */ + private $timeFactory; + + /** @var IUserManager */ + private $userManager; + + /** + * CleanPreviewsBackgroundJob constructor. + * + * @param IRootFolder $rootFolder + * @param ILogger $logger + * @param IJobList $jobList + * @param ITimeFactory $timeFactory + * @param IUserManager $userManager + */ + public function __construct(IRootFolder $rootFolder, + ILogger $logger, + IJobList $jobList, + ITimeFactory $timeFactory, + IUserManager $userManager) { + $this->rootFolder = $rootFolder; + $this->logger = $logger; + $this->jobList = $jobList; + $this->timeFactory = $timeFactory; + $this->userManager = $userManager; + } + + public function run($arguments) { + $uid = $arguments['uid']; + if (!$this->userManager->userExists($uid)) { + $this->logger->info('User no longer exists, skip user ' . $uid); + return; + } + $this->logger->info('Started preview cleanup for ' . $uid); + $empty = $this->cleanupPreviews($uid); + + if (!$empty) { + $this->jobList->add(self::class, ['uid' => $uid]); + $this->logger->info('New preview cleanup scheduled for ' . $uid); + } else { + $this->logger->info('Preview cleanup done for ' . $uid); + } + } + + /** + * @param $uid + * @return bool + */ + private function cleanupPreviews($uid) { + try { + $userFolder = $this->rootFolder->getUserFolder($uid); + } catch (NotFoundException $e) { + return true; + } + + $userRoot = $userFolder->getParent(); + + try { + /** @var Folder $thumbnailFolder */ + $thumbnailFolder = $userRoot->get('thumbnails'); + } catch (NotFoundException $e) { + return true; + } + + $thumbnails = $thumbnailFolder->getDirectoryListing(); + + $start = $this->timeFactory->getTime(); + foreach ($thumbnails as $thumbnail) { + try { + $thumbnail->delete(); + } catch (NotPermittedException $e) { + // Ignore + } + + if (($this->timeFactory->getTime() - $start) > 15) { + return false; + } + } + + try { + $thumbnailFolder->delete(); + } catch (NotPermittedException $e) { + // Ignore + } + + return true; + } +} diff --git a/lib/private/Repair/Owncloud/InstallCoreBundle.php b/lib/private/Repair/Owncloud/InstallCoreBundle.php new file mode 100644 index 00000000000..6d07ec9b962 --- /dev/null +++ b/lib/private/Repair/Owncloud/InstallCoreBundle.php @@ -0,0 +1,80 @@ + + * + * @author Lukas Reschke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Repair\Owncloud; + +use OC\App\AppStore\Bundles\BundleFetcher; +use OC\Installer; +use OCP\IConfig; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class InstallCoreBundle implements IRepairStep { + /** @var BundleFetcher */ + private $bundleFetcher; + /** @var IConfig */ + private $config; + /** @var Installer */ + private $installer; + + /** + * @param BundleFetcher $bundleFetcher + * @param IConfig $config + * @param Installer $installer + */ + public function __construct(BundleFetcher $bundleFetcher, + IConfig $config, + Installer $installer) { + $this->bundleFetcher = $bundleFetcher; + $this->config = $config; + $this->installer = $installer; + } + + /** + * {@inheritdoc} + */ + public function getName() { + return 'Install new core bundle components'; + } + + /** + * {@inheritdoc} + */ + public function run(IOutput $output) { + $versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); + + if (version_compare($versionFromBeforeUpdate, '12.0.0.14', '>')) { + return; + } + + $defaultBundle = $this->bundleFetcher->getDefaultInstallationBundle(); + foreach ($defaultBundle as $bundle) { + try { + $this->installer->installAppBundle($bundle); + $output->info('Successfully installed core app bundle.'); + } catch (\Exception $e) { + $output->warning('Could not install core app bundle: ' . $e->getMessage()); + } + } + } +} diff --git a/lib/private/Repair/Owncloud/MoveAvatars.php b/lib/private/Repair/Owncloud/MoveAvatars.php new file mode 100644 index 00000000000..53f3097aeec --- /dev/null +++ b/lib/private/Repair/Owncloud/MoveAvatars.php @@ -0,0 +1,73 @@ + + * + * @author Morris Jobke + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair\Owncloud; + +use OCP\BackgroundJob\IJobList; +use OCP\IConfig; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class MoveAvatars implements IRepairStep { + + /** @var IJobList */ + private $jobList; + + /** @var IConfig */ + private $config; + + /** + * MoveAvatars constructor. + * + * @param IJobList $jobList + * @param IConfig $config + */ + public function __construct(IJobList $jobList, + IConfig $config) { + $this->jobList = $jobList; + $this->config = $config; + } + + /** + * @return string + */ + public function getName() { + return 'Add move avatar background job'; + } + + public function run(IOutput $output) { + // only run once + if ($this->config->getAppValue('core', 'moveavatarsdone') === 'yes') { + $output->info('Repair step already executed'); + return; + } + if ($this->config->getSystemValue('enable_avatars', true) === false) { + $output->info('Avatars are disabled'); + } else { + $output->info('Add background job'); + $this->jobList->add(MoveAvatarsBackgroundJob::class); + // if all were done, no need to redo the repair during next upgrade + $this->config->setAppValue('core', 'moveavatarsdone', 'yes'); + } + } +} diff --git a/lib/private/Repair/Owncloud/MoveAvatarsBackgroundJob.php b/lib/private/Repair/Owncloud/MoveAvatarsBackgroundJob.php new file mode 100644 index 00000000000..ecc2d6eb3da --- /dev/null +++ b/lib/private/Repair/Owncloud/MoveAvatarsBackgroundJob.php @@ -0,0 +1,147 @@ + + * + * @author Joas Schilling + * @author Morris Jobke + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair\Owncloud; + +use OC\BackgroundJob\QueuedJob; +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\IAppData; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\SimpleFS\ISimpleFolder; +use OCP\ILogger; +use OCP\IUser; +use OCP\IUserManager; + +class MoveAvatarsBackgroundJob extends QueuedJob { + + /** @var IUserManager */ + private $userManager; + + /** @var IRootFolder */ + private $rootFolder; + + /** @var IAppData */ + private $appData; + + /** @var ILogger */ + private $logger; + + /** + * MoveAvatars constructor. + */ + public function __construct() { + $this->userManager = \OC::$server->getUserManager(); + $this->rootFolder = \OC::$server->getRootFolder(); + $this->logger = \OC::$server->getLogger(); + $this->appData = \OC::$server->getAppDataDir('avatar'); + } + + public function run($arguments) { + $this->logger->info('Started migrating avatars to AppData folder'); + $this->moveAvatars(); + $this->logger->info('All avatars migrated to AppData folder'); + } + + private function moveAvatars() { + try { + $ownCloudAvatars = $this->rootFolder->get('avatars'); + } catch (NotFoundException $e) { + $this->logger->info('No legacy avatars available, skipping migration'); + return; + } + + $counter = 0; + $this->userManager->callForSeenUsers(function (IUser $user) use ($counter, $ownCloudAvatars) { + $uid = $user->getUID(); + + \OC\Files\Filesystem::initMountPoints($uid); + /** @var Folder $userFolder */ + $userFolder = $this->rootFolder->get($uid); + + try { + $userData = $this->appData->getFolder($uid); + } catch (NotFoundException $e) { + $userData = $this->appData->newFolder($uid); + } + + $foundAvatars = $this->copyAvatarsFromFolder($userFolder, $userData); + + // ownCloud migration? + if ($foundAvatars === 0 && $ownCloudAvatars instanceof Folder) { + $parts = $this->buildOwnCloudAvatarPath($uid); + $userOwnCloudAvatar = $ownCloudAvatars; + foreach ($parts as $part) { + try { + $userOwnCloudAvatar = $userOwnCloudAvatar->get($part); + } catch (NotFoundException $e) { + return; + } + } + + $this->copyAvatarsFromFolder($userOwnCloudAvatar, $userData); + } + + $counter++; + if ($counter % 100 === 0) { + $this->logger->info('{amount} avatars migrated', ['amount' => $counter]); + } + }); + } + + /** + * @param Folder $source + * @param ISimpleFolder $target + * @return int + * @throws \OCP\Files\NotPermittedException + * @throws NotFoundException + */ + protected function copyAvatarsFromFolder(Folder $source, ISimpleFolder $target) { + $foundAvatars = 0; + $avatars = $source->getDirectoryListing(); + $regex = '/^avatar\.([0-9]+\.)?(jpg|png)$/'; + + foreach ($avatars as $avatar) { + /** @var File $avatar */ + if (preg_match($regex, $avatar->getName())) { + /* + * This is not the most effective but it is the most abstract way + * to handle this. Avatars should be small anyways. + */ + $newAvatar = $target->newFile($avatar->getName()); + $newAvatar->putContent($avatar->getContent()); + $avatar->delete(); + $foundAvatars++; + } + } + + return $foundAvatars; + } + + protected function buildOwnCloudAvatarPath($userId) { + $avatar = substr_replace(substr_replace(md5($userId), '/', 4, 0), '/', 2, 0); + return explode('/', $avatar); + } +} diff --git a/lib/private/Repair/Owncloud/UpdateLanguageCodes.php b/lib/private/Repair/Owncloud/UpdateLanguageCodes.php new file mode 100644 index 00000000000..b7da0b00528 --- /dev/null +++ b/lib/private/Repair/Owncloud/UpdateLanguageCodes.php @@ -0,0 +1,90 @@ + + * + * @author Joas Schilling + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Repair\Owncloud; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class UpdateLanguageCodes implements IRepairStep { + /** @var IDBConnection */ + private $connection; + + /** @var IConfig */ + private $config; + + /** + * @param IDBConnection $connection + * @param IConfig $config + */ + public function __construct(IDBConnection $connection, + IConfig $config) { + $this->connection = $connection; + $this->config = $config; + } + + /** + * {@inheritdoc} + */ + public function getName() { + return 'Repair language codes'; + } + + /** + * {@inheritdoc} + */ + public function run(IOutput $output) { + $versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); + + if (version_compare($versionFromBeforeUpdate, '12.0.0.13', '>')) { + return; + } + + $languages = [ + 'bg_BG' => 'bg', + 'cs_CZ' => 'cs', + 'fi_FI' => 'fi', + 'hu_HU' => 'hu', + 'nb_NO' => 'nb', + 'sk_SK' => 'sk', + 'th_TH' => 'th', + ]; + + foreach ($languages as $oldCode => $newCode) { + $qb = $this->connection->getQueryBuilder(); + + $affectedRows = $qb->update('preferences') + ->set('configvalue', $qb->createNamedParameter($newCode)) + ->where($qb->expr()->eq('appid', $qb->createNamedParameter('core'))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter('lang'))) + ->andWhere($qb->expr()->eq('configvalue', $qb->createNamedParameter($oldCode), IQueryBuilder::PARAM_STR)) + ->execute(); + + $output->info('Changed ' . $affectedRows . ' setting(s) from "' . $oldCode . '" to "' . $newCode . '" in preferences table.'); + } + } +} -- cgit v1.2.3