diff options
| -rw-r--r-- | apps/files/appinfo/info.xml | 1 | ||||
| -rw-r--r-- | apps/files/composer/composer/autoload_classmap.php | 1 | ||||
| -rw-r--r-- | apps/files/composer/composer/autoload_static.php | 1 | ||||
| -rw-r--r-- | apps/files/lib/Command/Object/ListObject.php | 72 | ||||
| -rw-r--r-- | apps/files/lib/Command/Object/ObjectUtil.php | 78 | ||||
| -rw-r--r-- | apps/files/lib/Command/Object/Orphans.php | 73 |
6 files changed, 154 insertions, 72 deletions
diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 5f6f7e52c30..95f6153e1d7 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -51,6 +51,7 @@ <command>OCA\Files\Command\Object\Put</command> <command>OCA\Files\Command\Object\Info</command> <command>OCA\Files\Command\Object\ListObject</command> + <command>OCA\Files\Command\Object\Orphans</command> </commands> <settings> diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 256210ac818..de41b717fcc 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -37,6 +37,7 @@ return array( 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index df1a18e984c..f226a656e0d 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -52,6 +52,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php index d3b8feae7c7..f63eb9c2260 100644 --- a/apps/files/lib/Command/Object/ListObject.php +++ b/apps/files/lib/Command/Object/ListObject.php @@ -41,79 +41,9 @@ class ListObject extends Base { $output->writeln('<error>Configured object store does currently not support listing objects</error>'); return self::FAILURE; } - $outputType = $input->getOption('output'); - $humanOutput = $outputType === self::OUTPUT_FORMAT_PLAIN; - - if (!$humanOutput) { - $output->writeln('['); - } $objects = $objectStore->listObjects(); - $first = true; - - foreach ($this->chunkIterator($objects, self::CHUNK_SIZE) as $chunk) { - if ($outputType === self::OUTPUT_FORMAT_PLAIN) { - $this->outputChunk($input, $output, $chunk); - } else { - foreach ($chunk as $object) { - if (!$first) { - $output->writeln(','); - } - $row = $this->formatObject($object, $humanOutput); - if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { - $output->write(json_encode($row, JSON_PRETTY_PRINT)); - } else { - $output->write(json_encode($row)); - } - $first = false; - } - } - } - - if (!$humanOutput) { - $output->writeln("\n]"); - } + $this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE); return self::SUCCESS; } - - private function formatObject(array $object, bool $humanOutput): array { - $row = array_merge([ - 'urn' => $object['urn'], - ], ($object['metadata'] ?? [])); - - if ($humanOutput && isset($row['size'])) { - $row['size'] = \OC_Helper::humanFileSize($row['size']); - } - if (isset($row['mtime'])) { - $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); - } - return $row; - } - - private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { - $result = []; - $humanOutput = $input->getOption('output') === "plain"; - - foreach ($chunk as $object) { - $result[] = $this->formatObject($object, $humanOutput); - } - $this->writeTableInOutputFormat($input, $output, $result); - } - - function chunkIterator(\Iterator $iterator, int $count): \Iterator { - $chunk = []; - - for($i = 0; $iterator->valid(); $i++){ - $chunk[] = $iterator->current(); - $iterator->next(); - if(count($chunk) == $count){ - yield $chunk; - $chunk = []; - } - } - - if(count($chunk)){ - yield $chunk; - } - } } diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index c4ab59608fb..8460e225b61 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -8,13 +8,15 @@ declare(strict_types=1); namespace OCA\Files\Command\Object; +use OC\Core\Command\Base; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; +use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -class ObjectUtil { +class ObjectUtil extends Base { public function __construct( private IConfig $config, private IDBConnection $connection, @@ -91,4 +93,78 @@ class ObjectUtil { return $fileId; } + + public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void { + $outputType = $input->getOption('output'); + $humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN; + $first = true; + + if (!$humanOutput) { + $output->writeln('['); + } + + foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) { + if ($outputType === Base::OUTPUT_FORMAT_PLAIN) { + $this->outputChunk($input, $output, $chunk); + } else { + foreach ($chunk as $object) { + if (!$first) { + $output->writeln(','); + } + $row = $this->formatObject($object, $humanOutput); + if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($row, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($row)); + } + $first = false; + } + } + } + + if (!$humanOutput) { + $output->writeln("\n]"); + } + } + + private function formatObject(array $object, bool $humanOutput): array { + $row = array_merge([ + 'urn' => $object['urn'], + ], ($object['metadata'] ?? [])); + + if ($humanOutput && isset($row['size'])) { + $row['size'] = \OC_Helper::humanFileSize($row['size']); + } + if (isset($row['mtime'])) { + $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); + } + return $row; + } + + private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void { + $result = []; + $humanOutput = $input->getOption('output') === 'plain'; + + foreach ($chunk as $object) { + $result[] = $this->formatObject($object, $humanOutput); + } + $this->writeTableInOutputFormat($input, $output, $result); + } + + public function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for ($i = 0; $iterator->valid(); $i++) { + $chunk[] = $iterator->current(); + $iterator->next(); + if (count($chunk) == $count) { + yield $chunk; + $chunk = []; + } + } + + if (count($chunk)) { + yield $chunk; + } + } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php new file mode 100644 index 00000000000..22538cf7b91 --- /dev/null +++ b/apps/files/lib/Command/Object/Orphans.php @@ -0,0 +1,73 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OCA\Files\Command\Object; + +use OC\Core\Command\Base; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\IDBConnection; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Orphans extends Base { + private const CHUNK_SIZE = 100; + + private IQueryBuilder $query; + + public function __construct( + private readonly ObjectUtil $objectUtils, + IDBConnection $connection, + ) { + parent::__construct(); + + $this->query = $connection->getQueryBuilder(); + $this->query->select('fileid') + ->from('filecache') + ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + } + + protected function configure(): void { + parent::configure(); + $this + ->setName('files:object:orphans') + ->setDescription('List all objects in the object store that don\'t have a matching entry in the database') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('<error>Configured object store does currently not support listing objects</error>'); + return self::FAILURE; + } + $prefixLength = strlen('urn:oid:'); + + $objects = $objectStore->listObjects('urn:oid:'); + $objects->rewind(); + $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) { + $fileId = (int)substr($object['urn'], $prefixLength); + return !$this->fileIdInDb($fileId); + }); + $orphans = new \ArrayIterator(iterator_to_array($orphans)); + $this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE); + + return self::SUCCESS; + } + + private function fileIdInDb(int $fileId): bool { + $this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); + $result = $this->query->executeQuery(); + return $result->fetchOne() !== false; + } +} |