diff options
Diffstat (limited to 'core/Command/Preview')
-rw-r--r-- | core/Command/Preview/Cleanup.php | 88 | ||||
-rw-r--r-- | core/Command/Preview/Generate.php | 118 | ||||
-rw-r--r-- | core/Command/Preview/Repair.php | 146 | ||||
-rw-r--r-- | core/Command/Preview/ResetRenderedTexts.php | 164 |
4 files changed, 444 insertions, 72 deletions
diff --git a/core/Command/Preview/Cleanup.php b/core/Command/Preview/Cleanup.php new file mode 100644 index 00000000000..dad981a5243 --- /dev/null +++ b/core/Command/Preview/Cleanup.php @@ -0,0 +1,88 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Core\Command\Preview; + +use OC\Core\Command\Base; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class Cleanup extends Base { + + public function __construct( + private IRootFolder $rootFolder, + private LoggerInterface $logger, + ) { + parent::__construct(); + } + + protected function configure(): void { + $this + ->setName('preview:cleanup') + ->setDescription('Removes existing preview files'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $appDataFolder = $this->rootFolder->get($this->rootFolder->getAppDataDirectoryName()); + + if (!$appDataFolder instanceof Folder) { + $this->logger->error("Previews can't be removed: appdata is not a folder"); + $output->writeln("Previews can't be removed: appdata is not a folder"); + return 1; + } + + /** @var Folder $previewFolder */ + $previewFolder = $appDataFolder->get('preview'); + + } catch (NotFoundException $e) { + $this->logger->error("Previews can't be removed: appdata folder can't be found", ['exception' => $e]); + $output->writeln("Previews can't be removed: preview folder isn't deletable"); + return 1; + } + + if (!$previewFolder->isDeletable()) { + $this->logger->error("Previews can't be removed: preview folder isn't deletable"); + $output->writeln("Previews can't be removed: preview folder isn't deletable"); + return 1; + } + + try { + $previewFolder->delete(); + $this->logger->debug('Preview folder deleted'); + $output->writeln('Preview folder deleted', OutputInterface::VERBOSITY_VERBOSE); + } catch (NotFoundException $e) { + $output->writeln("Previews weren't deleted: preview folder was not found while deleting it"); + $this->logger->error("Previews weren't deleted: preview folder was not found while deleting it", ['exception' => $e]); + return 1; + } catch (NotPermittedException $e) { + $output->writeln("Previews weren't deleted: you don't have the permission to delete preview folder"); + $this->logger->error("Previews weren't deleted: you don't have the permission to delete preview folder", ['exception' => $e]); + return 1; + } + + try { + $appDataFolder->newFolder('preview'); + $this->logger->debug('Preview folder recreated'); + $output->writeln('Preview folder recreated', OutputInterface::VERBOSITY_VERBOSE); + } catch (NotPermittedException $e) { + $output->writeln("Preview folder was deleted, but you don't have the permission to create preview folder"); + $this->logger->error("Preview folder was deleted, but you don't have the permission to create preview folder", ['exception' => $e]); + return 1; + } + + $output->writeln('Previews removed'); + return 0; + } +} diff --git a/core/Command/Preview/Generate.php b/core/Command/Preview/Generate.php new file mode 100644 index 00000000000..222c42f613b --- /dev/null +++ b/core/Command/Preview/Generate.php @@ -0,0 +1,118 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Core\Command\Preview; + +use OCP\Files\Config\IUserMountCache; +use OCP\Files\File; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\IPreview; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Generate extends Command { + public function __construct( + private IRootFolder $rootFolder, + private IUserMountCache $userMountCache, + private IPreview $previewManager, + ) { + parent::__construct(); + } + + protected function configure() { + $this + ->setName('preview:generate') + ->setDescription('generate a preview for a file') + ->addArgument('file', InputArgument::REQUIRED, 'path or fileid of the file to generate the preview for') + ->addOption('size', 's', InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED, 'size to generate the preview for in pixels, defaults to 64x64', ['64x64']) + ->addOption('crop', 'c', InputOption::VALUE_NONE, 'crop the previews instead of maintaining aspect ratio') + ->addOption('mode', 'm', InputOption::VALUE_REQUIRED, "mode for generating uncropped previews, 'cover' or 'fill'", IPreview::MODE_FILL); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileInput = $input->getArgument('file'); + $sizes = $input->getOption('size'); + $sizes = array_map(function (string $size) use ($output) { + if (str_contains($size, 'x')) { + $sizeParts = explode('x', $size, 2); + } else { + $sizeParts = [$size, $size]; + } + if (!is_numeric($sizeParts[0]) || !is_numeric($sizeParts[1] ?? null)) { + $output->writeln("<error>Invalid size $size</error>"); + return null; + } + + return array_map('intval', $sizeParts); + }, $sizes); + if (in_array(null, $sizes)) { + return 1; + } + + $mode = $input->getOption('mode'); + if ($mode !== IPreview::MODE_FILL && $mode !== IPreview::MODE_COVER) { + $output->writeln("<error>Invalid mode $mode</error>"); + return 1; + } + $crop = $input->getOption('crop'); + $file = $this->getFile($fileInput); + if (!$file) { + $output->writeln("<error>File $fileInput not found</error>"); + return 1; + } + if (!$file instanceof File) { + $output->writeln("<error>Can't generate previews for folders</error>"); + return 1; + } + + if (!$this->previewManager->isAvailable($file)) { + $output->writeln('<error>No preview generator available for file of type' . $file->getMimetype() . '</error>'); + return 1; + } + + $specifications = array_map(function (array $sizes) use ($crop, $mode) { + return [ + 'width' => $sizes[0], + 'height' => $sizes[1], + 'crop' => $crop, + 'mode' => $mode, + ]; + }, $sizes); + + $this->previewManager->generatePreviews($file, $specifications); + if (count($specifications) > 1) { + $output->writeln('generated <info>' . count($specifications) . '</info> previews'); + } else { + $output->writeln('preview generated'); + } + return 0; + } + + private function getFile(string $fileInput): ?Node { + if (is_numeric($fileInput)) { + $mounts = $this->userMountCache->getMountsForFileId((int)$fileInput); + if (!$mounts) { + return null; + } + $mount = $mounts[0]; + $userFolder = $this->rootFolder->getUserFolder($mount->getUser()->getUID()); + return $userFolder->getFirstNodeById((int)$fileInput); + } else { + try { + return $this->rootFolder->get($fileInput); + } catch (NotFoundException $e) { + return null; + } + } + } +} diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php index 198bb099179..a92a4cf8ed0 100644 --- a/core/Command/Preview/Repair.php +++ b/core/Command/Preview/Repair.php @@ -3,28 +3,9 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Core\Command\Preview; use bantu\IniGetWrapper\IniGetWrapper; @@ -33,40 +14,32 @@ use OCP\Files\Folder; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\IConfig; -use OCP\ILogger; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; +use Psr\Log\LoggerInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Helper\QuestionHelper; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Question\ConfirmationQuestion; +use function pcntl_signal; + class Repair extends Command { - /** @var IConfig */ - protected $config; - /** @var IRootFolder */ - private $rootFolder; - /** @var ILogger */ - private $logger; - - /** @var bool */ - private $stopSignalReceived = false; - /** @var int */ - private $memoryLimit; - /** @var int */ - private $memoryTreshold; - /** @var ILockingProvider */ - private $lockingProvider; - - public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni, ILockingProvider $lockingProvider) { - $this->config = $config; - $this->rootFolder = $rootFolder; - $this->logger = $logger; - $this->lockingProvider = $lockingProvider; - - $this->memoryLimit = $phpIni->getBytes('memory_limit'); + private bool $stopSignalReceived = false; + private int $memoryLimit; + private int $memoryTreshold; + + public function __construct( + protected IConfig $config, + private IRootFolder $rootFolder, + private LoggerInterface $logger, + IniGetWrapper $phpIni, + private ILockingProvider $lockingProvider, + ) { + $this->memoryLimit = (int)$phpIni->getBytes('memory_limit'); $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024; parent::__construct(); @@ -77,7 +50,8 @@ class Repair extends Command { ->setName('preview:repair') ->setDescription('distributes the existing previews into subfolders') ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.') - ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.'); + ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.') + ->addOption('delete', null, InputOption::VALUE_NONE, 'Delete instead of migrating them. Usefull if too many entries to migrate.'); } protected function execute(InputInterface $input, OutputInterface $output): int { @@ -86,28 +60,33 @@ class Repair extends Command { $thresholdInMiB = round($this->memoryTreshold / 1024 / 1024, 1); $output->writeln("Memory limit is $limitInMiB MiB"); $output->writeln("Memory threshold is $thresholdInMiB MiB"); - $output->writeln(""); + $output->writeln(''); $memoryCheckEnabled = true; } else { - $output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit."); - $output->writeln(""); + $output->writeln('No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.'); + $output->writeln(''); $memoryCheckEnabled = false; } $dryMode = $input->getOption('dry'); + $deleteMode = $input->getOption('delete'); + if ($dryMode) { - $output->writeln("INFO: The migration is run in dry mode and will not modify anything."); - $output->writeln(""); + $output->writeln('INFO: The migration is run in dry mode and will not modify anything.'); + $output->writeln(''); + } elseif ($deleteMode) { + $output->writeln('WARN: The migration will _DELETE_ old previews.'); + $output->writeln(''); } $instanceId = $this->config->getSystemValueString('instanceid'); - $output->writeln("This will migrate all previews from the old preview location to the new one."); + $output->writeln('This will migrate all previews from the old preview location to the new one.'); $output->writeln(''); $output->writeln('Fetching previews that need to be migrated …'); - /** @var \OCP\Files\Folder $currentPreviewFolder */ + /** @var Folder $currentPreviewFolder */ $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview"); $directoryListing = $currentPreviewFolder->getDirectoryListing(); @@ -144,17 +123,18 @@ class Repair extends Command { } if ($total === 0) { - $output->writeln("All previews are already migrated."); + $output->writeln('All previews are already migrated.'); return 0; } $output->writeln("A total of $total preview files need to be migrated."); - $output->writeln(""); - $output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue."); + $output->writeln(''); + $output->writeln('The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This will finish the current batch and then stop the migration. This migration can then just be started and it will continue.'); if ($input->getOption('batch')) { $output->writeln('Batch mode active: migration is started right away.'); } else { + /** @var QuestionHelper $helper */ $helper = $this->getHelper('question'); $question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false); @@ -166,12 +146,12 @@ class Repair extends Command { // register the SIGINT listener late in here to be able to exit in the early process of this command pcntl_signal(SIGINT, [$this, 'sigIntHandler']); - $output->writeln(""); - $output->writeln(""); + $output->writeln(''); + $output->writeln(''); $section1 = $output->section(); $section2 = $output->section(); $progressBar = new ProgressBar($section2, $total); - $progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%"); + $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%'); $time = (new \DateTime())->format('H:i:s'); $progressBar->setMessage("$time Starting …"); $progressBar->maxSecondsBetweenRedraws(0.2); @@ -213,10 +193,10 @@ class Repair extends Command { $memoryUsage = memory_get_usage(); if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) { - $section1->writeln(""); - $section1->writeln(""); - $section1->writeln(""); - $section1->writeln(" Stopped process 25 MB before reaching the memory limit to avoid a hard crash."); + $section1->writeln(''); + $section1->writeln(''); + $section1->writeln(''); + $section1->writeln(' Stopped process 25 MB before reaching the memory limit to avoid a hard crash.'); $time = (new \DateTime())->format('H:i:s'); $section1->writeln("$time Reached memory limit and stopped to avoid hard crash."); return 1; @@ -227,7 +207,7 @@ class Repair extends Command { $section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE); $this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); } catch (LockedException $e) { - $section1->writeln(" Skipping because it is locked - another process seems to work on this …"); + $section1->writeln(' Skipping because it is locked - another process seems to work on this …'); continue; } @@ -251,36 +231,58 @@ class Repair extends Command { $progressBar->advance(); continue; } - $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); + + // Execute process if (!$dryMode) { - try { - $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); - } catch (\Exception $e) { - $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]); + // Delete preview instead of moving + if ($deleteMode) { + try { + $section1->writeln(" Delete preview/$name/$previewName", OutputInterface::VERBOSITY_VERBOSE); + $preview->delete(); + } catch (\Exception $e) { + $this->logger->error("Failed to delete preview at preview/$name/$previewName", [ + 'app' => 'core', + 'exception' => $e, + ]); + } + } else { + try { + $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); + $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); + } catch (\Exception $e) { + $this->logger->error("Failed to move preview from preview/$name/$previewName to preview/$newFoldername", [ + 'app' => 'core', + 'exception' => $e, + ]); + } } } } } + if ($oldPreviewFolder->getDirectoryListing() === []) { $section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE); if (!$dryMode) { try { $oldPreviewFolder->delete(); } catch (\Exception $e) { - $this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]); + $this->logger->error("Failed to delete empty folder preview/$name", [ + 'app' => 'core', + 'exception' => $e, + ]); } } } $this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - $section1->writeln(" Unlocked", OutputInterface::VERBOSITY_VERBOSE); + $section1->writeln(' Unlocked', OutputInterface::VERBOSITY_VERBOSE); $section1->writeln(" Finished migrating previews of file with fileId $name …"); $progressBar->advance(); } $progressBar->finish(); - $output->writeln(""); + $output->writeln(''); return 0; } diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php new file mode 100644 index 00000000000..4cae315e48b --- /dev/null +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -0,0 +1,164 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Core\Command\Preview; + +use OC\Preview\Storage\Root; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\IAvatarManager; +use OCP\IDBConnection; +use OCP\IUserManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class ResetRenderedTexts extends Command { + public function __construct( + protected IDBConnection $connection, + protected IUserManager $userManager, + protected IAvatarManager $avatarManager, + private Root $previewFolder, + private IMimeTypeLoader $mimeTypeLoader, + ) { + parent::__construct(); + } + + protected function configure() { + $this + ->setName('preview:reset-rendered-texts') + ->setDescription('Deletes all generated avatars and previews of text and md files') + ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $dryMode = $input->getOption('dry'); + + if ($dryMode) { + $output->writeln('INFO: The command is run in dry mode and will not modify anything.'); + $output->writeln(''); + } + + $this->deleteAvatars($output, $dryMode); + $this->deletePreviews($output, $dryMode); + + return 0; + } + + private function deleteAvatars(OutputInterface $output, bool $dryMode): void { + $avatarsToDeleteCount = 0; + + foreach ($this->getAvatarsToDelete() as [$userId, $avatar]) { + $output->writeln('Deleting avatar for ' . $userId, OutputInterface::VERBOSITY_VERBOSE); + + $avatarsToDeleteCount++; + + if ($dryMode) { + continue; + } + + try { + $avatar->remove(); + } catch (NotFoundException $e) { + // continue + } catch (NotPermittedException $e) { + // continue + } + } + + $output->writeln('Deleted ' . $avatarsToDeleteCount . ' avatars'); + $output->writeln(''); + } + + private function getAvatarsToDelete(): \Iterator { + foreach ($this->userManager->search('') as $user) { + $avatar = $this->avatarManager->getAvatar($user->getUID()); + + if (!$avatar->isCustomAvatar()) { + yield [$user->getUID(), $avatar]; + } + } + } + + private function deletePreviews(OutputInterface $output, bool $dryMode): void { + $previewsToDeleteCount = 0; + + foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) { + $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); + + $previewsToDeleteCount++; + + if ($dryMode) { + continue; + } + + try { + $preview = $this->previewFolder->getFolder((string)$previewFileId); + $preview->delete(); + } catch (NotFoundException $e) { + // continue + } catch (NotPermittedException $e) { + // continue + } + } + + $output->writeln('Deleted ' . $previewsToDeleteCount . ' previews'); + } + + // Copy pasted and adjusted from + // "lib/private/Preview/BackgroundCleanupJob.php". + private function getPreviewsToDelete(): \Iterator { + $qb = $this->connection->getQueryBuilder(); + $qb->select('path', 'mimetype') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); + $cursor = $qb->executeQuery(); + $data = $cursor->fetch(); + $cursor->closeCursor(); + + if ($data === null) { + return []; + } + + /* + * This lovely like is the result of the way the new previews are stored + * We take the md5 of the name (fileid) and split the first 7 chars. That way + * there are not a gazillion files in the root of the preview appdata. + */ + $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; + + $qb = $this->connection->getQueryBuilder(); + $qb->select('a.name', 'b.path') + ->from('filecache', 'a') + ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( + $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + )) + ->where( + $qb->expr()->andX( + $qb->expr()->like('a.path', $qb->createNamedParameter($like)), + $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), + $qb->expr()->orX( + $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), + $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), + $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) + ) + ) + ); + + $cursor = $qb->executeQuery(); + + while ($row = $cursor->fetch()) { + yield $row; + } + + $cursor->closeCursor(); + } +} |