feat: add command to get object metadatabackport/object-store-orphan/27

feat: add command to list objects

feat: add command to list orphan objects

Signed-off-by: Robin Appelman <robin@icewind.nl>

11 files changed, 366 insertions, 4 deletions

diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml
index 598f8b619c1..78e53f91608 100644
--- a/apps/files/appinfo/info.xml
+++ b/apps/files/appinfo/info.xml
@@ -41,6 +41,9 @@
 		<command>OCA\Files\Command\Object\Delete</command>
 		<command>OCA\Files\Command\Object\Get</command>
 		<command>OCA\Files\Command\Object\Put</command>
+		<command>OCA\Files\Command\Object\Info</command>
+		<command>OCA\Files\Command\Object\ListObject</command>
+		<command>OCA\Files\Command\Object\Orphans</command>
 	</commands>
 
 	<activity>
diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php
index 88d3ae10413..71962464edc 100644
--- a/apps/files/composer/composer/autoload_classmap.php
+++ b/apps/files/composer/composer/autoload_classmap.php
@@ -32,7 +32,10 @@ return array(
     'OCA\\Files\\Command\\Get' => $baseDir . '/../lib/Command/Get.php',
     'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
     'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
+    'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
+    'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
     'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
+    'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
     'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
     'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
     'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',
diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php
index c27db581e61..fa4050cfadc 100644
--- a/apps/files/composer/composer/autoload_static.php
+++ b/apps/files/composer/composer/autoload_static.php
@@ -47,7 +47,10 @@ class ComposerStaticInitFiles
         'OCA\\Files\\Command\\Get' => __DIR__ . '/..' . '/../lib/Command/Get.php',
         'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
         'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
+        'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
+        'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
         'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
+        'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
         'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
         'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
         'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',
diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php
new file mode 100644
index 00000000000..5dcc552ea34
--- /dev/null
+++ b/apps/files/lib/Command/Object/Info.php
@@ -0,0 +1,79 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OC\Core\Command\Base;
+use OCP\Files\IMimeTypeDetector;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Info extends Base {
+	public function __construct(
+		private ObjectUtil $objectUtils,
+		private IMimeTypeDetector $mimeTypeDetector,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		parent::configure();
+		$this
+			->setName('files:object:info')
+			->setDescription('Get the metadata of an object')
+			->addArgument('object', InputArgument::REQUIRED, 'Object to get')
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$object = $input->getArgument('object');
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+		if (!$objectStore) {
+			return self::FAILURE;
+		}
+
+		if (!$objectStore instanceof IObjectStoreMetaData) {
+			$output->writeln('<error>Configured object store does currently not support retrieve metadata</error>');
+			return self::FAILURE;
+		}
+
+		if (!$objectStore->objectExists($object)) {
+			$output->writeln("<error>Object $object does not exist</error>");
+			return self::FAILURE;
+		}
+
+		try {
+			$meta = $objectStore->getObjectMetaData($object);
+		} catch (\Exception $e) {
+			$msg = $e->getMessage();
+			$output->writeln("<error>Failed to read $object from object store: $msg</error>");
+			return self::FAILURE;
+		}
+
+		if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
+			$meta['size'] = \OC_Helper::humanFileSize($meta['size']);
+		}
+		if (isset($meta['mtime'])) {
+			$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
+		}
+		if (!isset($meta['mimetype'])) {
+			$handle = $objectStore->readObject($object);
+			$head = fread($handle, 8192);
+			fclose($handle);
+			$meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
+		}
+
+		$this->writeArrayInOutputFormat($input, $output, $meta);
+
+		return self::SUCCESS;
+	}
+
+}
diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php
new file mode 100644
index 00000000000..9d1ff93c6ee
--- /dev/null
+++ b/apps/files/lib/Command/Object/ListObject.php
@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OC\Core\Command\Base;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ListObject extends Base {
+	private const CHUNK_SIZE = 100;
+
+	public function __construct(
+		private ObjectUtil $objectUtils,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		parent::configure();
+		$this
+			->setName('files:object:list')
+			->setDescription('List all objects in the object store')
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+		if (!$objectStore) {
+			return self::FAILURE;
+		}
+
+		if (!$objectStore instanceof IObjectStoreMetaData) {
+			$output->writeln('<error>Configured object store does currently not support listing objects</error>');
+			return self::FAILURE;
+		}
+		$objects = $objectStore->listObjects();
+		$this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE);
+
+		return self::SUCCESS;
+	}
+}
diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php
index b7359dfa193..04902c829bf 100644
--- a/apps/files/lib/Command/Object/ObjectUtil.php
+++ b/apps/files/lib/Command/Object/ObjectUtil.php
@@ -23,13 +23,15 @@ declare(strict_types=1);
 
 namespace OCA\Files\Command\Object;
 
+use OC\Core\Command\Base;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\IConfig;
 use OCP\IDBConnection;
+use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
-class ObjectUtil {
+class ObjectUtil extends Base {
 	private IConfig $config;
 	private IDBConnection $connection;
 
@@ -107,4 +109,78 @@ class ObjectUtil {
 			return false;
 		}
 	}
+
+	public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void {
+		$outputType = $input->getOption('output');
+		$humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN;
+		$first = true;
+
+		if (!$humanOutput) {
+			$output->writeln('[');
+		}
+
+		foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) {
+			if ($outputType === Base::OUTPUT_FORMAT_PLAIN) {
+				$this->outputChunk($input, $output, $chunk);
+			} else {
+				foreach ($chunk as $object) {
+					if (!$first) {
+						$output->writeln(',');
+					}
+					$row = $this->formatObject($object, $humanOutput);
+					if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) {
+						$output->write(json_encode($row, JSON_PRETTY_PRINT));
+					} else {
+						$output->write(json_encode($row));
+					}
+					$first = false;
+				}
+			}
+		}
+
+		if (!$humanOutput) {
+			$output->writeln("\n]");
+		}
+	}
+
+	private function formatObject(array $object, bool $humanOutput): array {
+		$row = array_merge([
+			'urn' => $object['urn'],
+		], ($object['metadata'] ?? []));
+
+		if ($humanOutput && isset($row['size'])) {
+			$row['size'] = \OC_Helper::humanFileSize($row['size']);
+		}
+		if (isset($row['mtime'])) {
+			$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
+		}
+		return $row;
+	}
+
+	private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
+		$result = [];
+		$humanOutput = $input->getOption('output') === 'plain';
+
+		foreach ($chunk as $object) {
+			$result[] = $this->formatObject($object, $humanOutput);
+		}
+		$this->writeTableInOutputFormat($input, $output, $result);
+	}
+
+	public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
+		$chunk = [];
+
+		for ($i = 0; $iterator->valid(); $i++) {
+			$chunk[] = $iterator->current();
+			$iterator->next();
+			if (count($chunk) == $count) {
+				yield $chunk;
+				$chunk = [];
+			}
+		}
+
+		if (count($chunk)) {
+			yield $chunk;
+		}
+	}
 }
diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php
new file mode 100644
index 00000000000..8d6daba622e
--- /dev/null
+++ b/apps/files/lib/Command/Object/Orphans.php
@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Command\Object;
+
+use OC\Core\Command\Base;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Orphans extends Base {
+	private const CHUNK_SIZE = 100;
+
+	private IQueryBuilder $query;
+
+	public function __construct(
+		private ObjectUtil $objectUtils,
+		IDBConnection $connection,
+	) {
+		parent::__construct();
+
+		$this->query = $connection->getQueryBuilder();
+		$this->query->select('fileid')
+			->from('filecache')
+			->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
+	}
+
+	protected function configure(): void {
+		parent::configure();
+		$this
+			->setName('files:object:orphans')
+			->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+		if (!$objectStore) {
+			return self::FAILURE;
+		}
+
+		if (!$objectStore instanceof IObjectStoreMetaData) {
+			$output->writeln('<error>Configured object store does currently not support listing objects</error>');
+			return self::FAILURE;
+		}
+		$prefixLength = strlen('urn:oid:');
+
+		$objects = $objectStore->listObjects('urn:oid:');
+		$objects->rewind();
+		$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
+			$fileId = (int)substr($object['urn'], $prefixLength);
+			return !$this->fileIdInDb($fileId);
+		});
+		$orphans = new \ArrayIterator(iterator_to_array($orphans));
+		$this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE);
+
+		return self::SUCCESS;
+	}
+
+	private function fileIdInDb(int $fileId): bool {
+		$this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
+		$result = $this->query->executeQuery();
+		return $result->fetchOne() !== false;
+	}
+}
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index d57fdf4c38c..20b927c022f 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -354,6 +354,7 @@ return array(
     'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php',
     'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php',
     'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php',
+    'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
     'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
     'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php',
     'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index a62b945f501..5eb5e9bbd3f 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -387,6 +387,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php',
         'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php',
         'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php',
+        'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
         'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
         'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php',
         'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php',
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
index b1cd89388ae..492659f2bde 100644
--- a/lib/private/Files/ObjectStore/S3.php
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -21,14 +21,16 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  */
+
 namespace OC\Files\ObjectStore;
 
 use Aws\Result;
 use Exception;
 use OCP\Files\ObjectStore\IObjectStore;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
 use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
 
-class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
+class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData {
 	use S3ConnectionTrait;
 	use S3ObjectTrait;
 
@@ -79,7 +81,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
 				'Key' => $urn,
 				'UploadId' => $uploadId,
 				'MaxParts' => 1000,
-				'PartNumberMarker' => $partNumberMarker
+				'PartNumberMarker' => $partNumberMarker,
 			] + $this->getSSECParameters());
 			$parts = array_merge($parts, $result->get('Parts') ?? []);
 			$isTruncated = $result->get('IsTruncated');
@@ -107,7 +109,41 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
 		$this->getConnection()->abortMultipartUpload([
 			'Bucket' => $this->bucket,
 			'Key' => $urn,
-			'UploadId' => $uploadId
+			'UploadId' => $uploadId,
 		]);
 	}
+
+	public function getObjectMetaData(string $urn): array {
+		$object = $this->getConnection()->headObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn
+		] + $this->getSSECParameters())->toArray();
+		return [
+			'mtime' => $object['LastModified'],
+			'etag' => trim($object['ETag'], '"'),
+			'size' => (int)($object['Size'] ?? $object['ContentLength']),
+		];
+	}
+
+	public function listObjects(string $prefix = ''): \Iterator {
+		$results = $this->getConnection()->getPaginator('ListObjectsV2', [
+			'Bucket' => $this->bucket,
+			'Prefix' => $prefix,
+		] + $this->getSSECParameters());
+
+		foreach ($results as $result) {
+			if (is_array($result['Contents'])) {
+				foreach ($result['Contents'] as $object) {
+					yield [
+						'urn' => basename($object['Key']),
+						'metadata' => [
+							'mtime' => $object['LastModified'],
+							'etag' => trim($object['ETag'], '"'),
+							'size' => (int)($object['Size'] ?? $object['ContentLength']),
+						],
+					];
+				}
+			}
+		}
+	}
 }
diff --git a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php
new file mode 100644
index 00000000000..8359e83f573
--- /dev/null
+++ b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php
@@ -0,0 +1,38 @@
+<?php
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+namespace OCP\Files\ObjectStore;
+
+/**
+ * Interface IObjectStoreMetaData
+ *
+ * @psalm-type ObjectMetaData = array{mtime?: \DateTime, etag?: string, size?: int, mimetype?: string, filename?: string}
+ *
+ * @since 32.0.0
+ */
+interface IObjectStoreMetaData {
+	/**
+	 * Get metadata for an object.
+	 *
+	 * @param string $urn
+	 * @return ObjectMetaData
+	 *
+	 * @since 32.0.0
+	 */
+	public function getObjectMetaData(string $urn): array;
+
+	/**
+	 * List all objects in the object store.
+	 *
+	 * If the object store implementation can do it efficiently, the metadata for each object is also included.
+	 *
+	 * @param string $prefix
+	 * @return \Iterator<array{urn: string, metadata: ?ObjectMetaData}>
+	 *
+	 * @since 32.0.0
+	 */
+	public function listObjects(string $prefix = ''): \Iterator;
+}