1 files changed, 126 insertions, 41 deletions

diff --git a/apps/files/lib/Command/DeleteOrphanedFiles.php b/apps/files/lib/Command/DeleteOrphanedFiles.php
index 2e5893cfd1f..37cb3159f4a 100644
--- a/apps/files/lib/Command/DeleteOrphanedFiles.php
+++ b/apps/files/lib/Command/DeleteOrphanedFiles.php
@@ -1,83 +1,168 @@
 <?php
+
 /**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Joas Schilling <coding@schilljs.com>
- * @author Morris Jobke <hey@morrisjobke.de>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program.  If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
  */
-
 namespace OCA\Files\Command;
 
+use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IDBConnection;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 /**
  * Delete all file entries that have no matching entries in the storage table.
  */
 class DeleteOrphanedFiles extends Command {
+	public const CHUNK_SIZE = 200;
+
+	public function __construct(
+		protected IDBConnection $connection,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		$this
+			->setName('files:cleanup')
+			->setDescription('Clean up orphaned filecache and mount entries')
+			->setHelp('Deletes orphaned filecache and mount entries (those without an existing storage).')
+			->addOption('skip-filecache-extended', null, InputOption::VALUE_NONE, 'don\'t remove orphaned entries from filecache_extended');
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$fileIdsByStorage = [];
+
+		$deletedStorages = array_diff($this->getReferencedStorages(), $this->getExistingStorages());
+
+		$deleteExtended = !$input->getOption('skip-filecache-extended');
+		if ($deleteExtended) {
+			$fileIdsByStorage = $this->getFileIdsForStorages($deletedStorages);
+		}
 
-	const CHUNK_SIZE = 200;
+		$deletedEntries = $this->cleanupOrphanedFileCache($deletedStorages);
+		$output->writeln("$deletedEntries orphaned file cache entries deleted");
+
+		if ($deleteExtended) {
+			$deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended($fileIdsByStorage);
+			$output->writeln("$deletedFileCacheExtended orphaned file cache extended entries deleted");
+		}
+
+		$deletedMounts = $this->cleanupOrphanedMounts();
+		$output->writeln("$deletedMounts orphaned mount entries deleted");
+
+		return self::SUCCESS;
+	}
+
+	private function getReferencedStorages(): array {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('storage')
+			->from('filecache')
+			->groupBy('storage')
+			->runAcrossAllShards();
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+	}
+
+	private function getExistingStorages(): array {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('numeric_id')
+			->from('storages')
+			->groupBy('numeric_id');
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+	}
 
 	/**
-	 * @var IDBConnection
+	 * @param int[] $storageIds
+	 * @return array<int, int[]>
 	 */
-	protected $connection;
+	private function getFileIdsForStorages(array $storageIds): array {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('storage', 'fileid')
+			->from('filecache')
+			->where($query->expr()->in('storage', $query->createParameter('storage_ids')));
 
-	public function __construct(IDBConnection $connection) {
-		$this->connection = $connection;
-		parent::__construct();
+		$result = [];
+		$storageIdChunks = array_chunk($storageIds, self::CHUNK_SIZE);
+		foreach ($storageIdChunks as $storageIdChunk) {
+			$query->setParameter('storage_ids', $storageIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
+			$chunk = $query->executeQuery()->fetchAll();
+			foreach ($chunk as $row) {
+				$result[$row['storage']][] = $row['fileid'];
+			}
+		}
+		return $result;
 	}
 
-	protected function configure() {
-		$this
-			->setName('files:cleanup')
-			->setDescription('cleanup filecache');
+	private function cleanupOrphanedFileCache(array $deletedStorages): int {
+		$deletedEntries = 0;
+
+		$deleteQuery = $this->connection->getQueryBuilder();
+		$deleteQuery->delete('filecache')
+			->where($deleteQuery->expr()->in('storage', $deleteQuery->createParameter('storage_ids')));
+
+		$deletedStorageChunks = array_chunk($deletedStorages, self::CHUNK_SIZE);
+		foreach ($deletedStorageChunks as $deletedStorageChunk) {
+			$deleteQuery->setParameter('storage_ids', $deletedStorageChunk, IQueryBuilder::PARAM_INT_ARRAY);
+			$deletedEntries += $deleteQuery->executeStatement();
+		}
+
+		return $deletedEntries;
+	}
+
+	/**
+	 * @param array<int, int[]> $fileIdsByStorage
+	 * @return int
+	 */
+	private function cleanupOrphanedFileCacheExtended(array $fileIdsByStorage): int {
+		$deletedEntries = 0;
+
+		$deleteQuery = $this->connection->getQueryBuilder();
+		$deleteQuery->delete('filecache_extended')
+			->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('file_ids')));
+
+		foreach ($fileIdsByStorage as $storageId => $fileIds) {
+			$deleteQuery->hintShardKey('storage', $storageId, true);
+			$fileChunks = array_chunk($fileIds, self::CHUNK_SIZE);
+			foreach ($fileChunks as $fileChunk) {
+				$deleteQuery->setParameter('file_ids', $fileChunk, IQueryBuilder::PARAM_INT_ARRAY);
+				$deletedEntries += $deleteQuery->executeStatement();
+			}
+		}
+
+		return $deletedEntries;
 	}
 
-	public function execute(InputInterface $input, OutputInterface $output) {
+	private function cleanupOrphanedMounts(): int {
 		$deletedEntries = 0;
 
 		$query = $this->connection->getQueryBuilder();
-		$query->select('fc.fileid')
-			->from('filecache', 'fc')
+		$query->select('m.storage_id')
+			->from('mounts', 'm')
 			->where($query->expr()->isNull('s.numeric_id'))
-			->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
+			->leftJoin('m', 'storages', 's', $query->expr()->eq('m.storage_id', 's.numeric_id'))
+			->groupBy('storage_id')
 			->setMaxResults(self::CHUNK_SIZE);
 
 		$deleteQuery = $this->connection->getQueryBuilder();
-		$deleteQuery->delete('filecache')
-			->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid')));
+		$deleteQuery->delete('mounts')
+			->where($deleteQuery->expr()->eq('storage_id', $deleteQuery->createParameter('storageid')));
 
 		$deletedInLastChunk = self::CHUNK_SIZE;
 		while ($deletedInLastChunk === self::CHUNK_SIZE) {
 			$deletedInLastChunk = 0;
-			$result = $query->execute();
+			$result = $query->executeQuery();
 			while ($row = $result->fetch()) {
 				$deletedInLastChunk++;
-				$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row['fileid'])
-					->execute();
+				$deletedEntries += $deleteQuery->setParameter('storageid', (int)$row['storage_id'])
+					->executeStatement();
 			}
 			$result->closeCursor();
 		}
 
-		$output->writeln("$deletedEntries orphaned file cache entries deleted");
+		return $deletedEntries;
 	}
-
 }