diff options
| -rw-r--r-- | core/Command/Preview/Repair.php | 280 | ||||
| -rw-r--r-- | core/register_command.php | 2 | ||||
| -rw-r--r-- | lib/composer/composer/autoload_classmap.php | 1 | ||||
| -rw-r--r-- | lib/composer/composer/autoload_static.php | 1 | ||||
| -rw-r--r-- | lib/private/Preview/Storage/Root.php | 6 |
5 files changed, 287 insertions, 3 deletions
diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php new file mode 100644 index 00000000000..26760cb0e34 --- /dev/null +++ b/core/Command/Preview/Repair.php @@ -0,0 +1,280 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> + * + * @author Morris Jobke <hey@morrisjobke.de> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Core\Command\Preview; + +use bantu\IniGetWrapper\IniGetWrapper; +use OC\Preview\Storage\Root; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\IConfig; +use OCP\ILogger; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Question\ConfirmationQuestion; + +class Repair extends Command { + /** @var IConfig */ + protected $config; + /** @var IRootFolder */ + private $rootFolder; + /** @var ILogger */ + private $logger; + + /** @var bool */ + private $stopSignalReceived = false; + /** @var int */ + private $memoryLimit; + /** @var int */ + private $memoryTreshold; + + public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni) { + $this->config = $config; + $this->rootFolder = $rootFolder; + $this->logger = $logger; + + $this->memoryLimit = $phpIni->getBytes('memory_limit'); + $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024; + + parent::__construct(); + } + + protected function configure() { + $this + ->setName('preview:repair') + ->setDescription('distributes the existing previews into subfolders') + ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.') + ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + if ($this->memoryLimit !== -1) { + $limitInMiB = round($this->memoryLimit / 1024 /1024, 1); + $thresholdInMiB = round($this->memoryTreshold / 1024 /1024, 1); + $output->writeln("Memory limit is $limitInMiB MiB"); + $output->writeln("Memory threshold is $thresholdInMiB MiB"); + $output->writeln(""); + $memoryCheckEnabled = true; + } else { + $output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit."); + $output->writeln(""); + $memoryCheckEnabled = false; + } + + $dryMode = $input->getOption('dry'); + + if ($dryMode) { + $output->writeln("INFO: The migration is run in dry mode and will not modify anything."); + $output->writeln(""); + } + + $verbose = $output->isVerbose(); + + $instanceId = $this->config->getSystemValueString('instanceid'); + + $output->writeln("This will migrate all previews from the old preview location to the new one."); + $output->writeln(''); + + $output->writeln('Fetching previews that need to be migrated …'); + /** @var \OCP\Files\Folder $currentPreviewFolder */ + $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview"); + + $directoryListing = $currentPreviewFolder->getDirectoryListing(); + + $total = count($directoryListing); + /** + * by default there could be 0-9 a-f and the old-multibucket folder which are all fine + */ + if ($total < 18) { + foreach ($directoryListing as $index => $dir) { + if ($dir->getName() === 'old-multibucket') { + unset($directoryListing[$index]); + } + // a-f can't be a file ID -> removing from migration + if (preg_match('!^[a-f]$!', $dir->getName())) { + unset($directoryListing[$index]); + } + if (preg_match('!^[0-9]$!', $dir->getName())) { + // ignore folders that only has folders in them + if ($dir instanceof Folder) { + $hasFile = false; + foreach ($dir->getDirectoryListing() as $entry) { + if (!$entry instanceof Folder) { + $hasFile = true; + break; + } + } + if (!$hasFile) { + unset($directoryListing[$index]); + } + } + } + } + $total = count($directoryListing); + } + + if ($total === 0) { + $output->writeln("All previews are already migrated."); + return 0; + } + + $output->writeln("A total of $total preview files need to be migrated."); + $output->writeln(""); + $output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue."); + + if ($input->getOption('batch')) { + $output->writeln('Batch mode active: migration is started right away.'); + } else { + $helper = $this->getHelper('question'); + $question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false); + + if (!$helper->ask($input, $output, $question)) { + return 0; + } + } + + // register the SIGINT listener late in here to be able to exit in the early process of this command + pcntl_signal(SIGINT, [$this, 'sigIntHandler']); + + $output->writeln(""); + $output->writeln(""); + $section1 = $output->section(); + $section2 = $output->section(); + $progressBar = new ProgressBar($section2, $total); + $progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%"); + $time = (new \DateTime())->format('H:i:s'); + $progressBar->setMessage("$time Starting …"); + $progressBar->maxSecondsBetweenRedraws(0.2); + $progressBar->start(); + + foreach ($directoryListing as $oldPreviewFolder) { + pcntl_signal_dispatch(); + $name = $oldPreviewFolder->getName(); + $time = (new \DateTime())->format('H:i:s'); + $section1->writeln("$time Migrating previews of file with fileId $name …"); + $progressBar->display(); + + if ($this->stopSignalReceived) { + $section1->writeln("$time Stopping migration …"); + return 0; + } + if (!$oldPreviewFolder instanceof Folder) { + $section1->writeln(" Skipping non-folder $name …"); + $progressBar->advance(); + continue; + } + if ($name === 'old-multibucket') { + $section1->writeln(" Skipping fallback mount point $name …"); + $progressBar->advance(); + continue; + } + if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) { + $section1->writeln(" Skipping hex-digit folder $name …"); + $progressBar->advance(); + continue; + } + if (!preg_match('!^\d+$!', $name)) { + $section1->writeln(" Skipping non-numeric folder $name …"); + $progressBar->advance(); + continue; + } + + $newFoldername = Root::getInternalFolder($name); + + $memoryUsage = memory_get_usage(); + if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) { + $section1->writeln(""); + $section1->writeln(""); + $section1->writeln(""); + $section1->writeln(" Stopped process 25 MB before reaching the memory limit to avoid a hard crash."); + $time = (new \DateTime())->format('H:i:s'); + $section1->writeln("$time Reached memory limit and stopped to avoid hard crash."); + return 1; + } + + $previews = $oldPreviewFolder->getDirectoryListing(); + if ($previews !== []) { + try { + $this->rootFolder->get("appdata_$instanceId/preview/$newFoldername"); + } catch (NotFoundException $e) { + if ($verbose) { + $section1->writeln(" Create folder preview/$newFoldername"); + } + if (!$dryMode) { + $this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername"); + } + } + + foreach ($previews as $preview) { + pcntl_signal_dispatch(); + $previewName = $preview->getName(); + + if ($preview instanceof Folder) { + $section1->writeln(" Skipping folder $name/$previewName …"); + $progressBar->advance(); + continue; + } + if ($verbose) { + $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername"); + } + if (!$dryMode) { + try { + $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); + } catch (\Exception $e) { + $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]); + } + } + } + } + if ($oldPreviewFolder->getDirectoryListing() === []) { + if ($verbose) { + $section1->writeln(" Delete empty folder preview/$name"); + } + if (!$dryMode) { + try { + $oldPreviewFolder->delete(); + } catch (\Exception $e) { + $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]); + } + } + } + $section1->writeln(" Finished migrating previews of file with fileId $name …"); + $progressBar->advance(); + } + + $progressBar->finish(); + $output->writeln(""); + return 0; + } + + protected function sigIntHandler() { + echo "\nSignal received - will finish the step and then stop the migration.\n\n\n"; + $this->stopSignalReceived = true; + } +} diff --git a/core/register_command.php b/core/register_command.php index d818423d1ab..03d458a9cbc 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -155,6 +155,8 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { \OC::$server->getAppManager() )); + $application->add(\OC::$server->query(\OC\Core\Command\Preview\Repair::class)); + $application->add(new OC\Core\Command\User\Add(\OC::$server->getUserManager(), \OC::$server->getGroupManager())); $application->add(new OC\Core\Command\User\Delete(\OC::$server->getUserManager())); $application->add(new OC\Core\Command\User\Disable(\OC::$server->getUserManager())); diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 254840b3542..e29e8ce2e7b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -829,6 +829,7 @@ return array( 'OC\\Core\\Command\\Maintenance\\Repair' => $baseDir . '/core/Command/Maintenance/Repair.php', 'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => $baseDir . '/core/Command/Maintenance/UpdateHtaccess.php', 'OC\\Core\\Command\\Maintenance\\UpdateTheme' => $baseDir . '/core/Command/Maintenance/UpdateTheme.php', + 'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Security\\ImportCertificate' => $baseDir . '/core/Command/Security/ImportCertificate.php', 'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php', 'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 45bf93241d5..8bb4f615520 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -858,6 +858,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Core\\Command\\Maintenance\\Repair' => __DIR__ . '/../../..' . '/core/Command/Maintenance/Repair.php', 'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateHtaccess.php', 'OC\\Core\\Command\\Maintenance\\UpdateTheme' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateTheme.php', + 'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Security\\ImportCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/ImportCertificate.php', 'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php', 'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php', diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php index a9a72026a51..1f332c32085 100644 --- a/lib/private/Preview/Storage/Root.php +++ b/lib/private/Preview/Storage/Root.php @@ -39,7 +39,7 @@ class Root extends AppData { public function getFolder(string $name): ISimpleFolder { - $internalFolder = $this->getInternalFolder($name); + $internalFolder = self::getInternalFolder($name); try { return parent::getFolder($internalFolder); @@ -54,7 +54,7 @@ class Root extends AppData { } public function newFolder(string $name): ISimpleFolder { - $internalFolder = $this->getInternalFolder($name); + $internalFolder = self::getInternalFolder($name); return parent::newFolder($internalFolder); } @@ -66,7 +66,7 @@ class Root extends AppData { return []; } - private function getInternalFolder(string $name): string { + public static function getInternalFolder(string $name): string { return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; } } |