aboutsummaryrefslogtreecommitdiffstats
path: root/apps/files/lib
diff options
context:
space:
mode:
Diffstat (limited to 'apps/files/lib')
-rw-r--r--apps/files/lib/BackgroundJob/DeleteOrphanedItems.php86
-rw-r--r--apps/files/lib/Command/DeleteOrphanedFiles.php102
2 files changed, 135 insertions, 53 deletions
diff --git a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
index 32fb569a3d4..b1a795b775c 100644
--- a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
+++ b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
@@ -52,35 +52,87 @@ class DeleteOrphanedItems extends TimedJob {
* @param string $typeCol
* @return int Number of deleted entries
*/
- protected function cleanUp($table, $idCol, $typeCol) {
+ protected function cleanUp(string $table, string $idCol, string $typeCol): int {
$deletedEntries = 0;
- $query = $this->connection->getQueryBuilder();
- $query->select('t1.' . $idCol)
- ->from($table, 't1')
- ->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
- ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
- ->andWhere($query->expr()->isNull('t2.fileid'))
- ->groupBy('t1.' . $idCol)
- ->setMaxResults(self::CHUNK_SIZE);
-
$deleteQuery = $this->connection->getQueryBuilder();
$deleteQuery->delete($table)
- ->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid')));
+ ->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid')));
+
+ if ($this->connection->getShardDefinition('filecache')) {
+ $sourceIdChunks = $this->getItemIds($table, $idCol, $typeCol, 1000);
+ foreach ($sourceIdChunks as $sourceIdChunk) {
+ $deletedSources = $this->findMissingSources($sourceIdChunk);
+ $deleteQuery->setParameter('objectid', $deletedSources, IQueryBuilder::PARAM_INT_ARRAY);
+ $deletedEntries += $deleteQuery->executeStatement();
+ }
+ } else {
+ $query = $this->connection->getQueryBuilder();
+ $query->select('t1.' . $idCol)
+ ->from($table, 't1')
+ ->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
+ ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
+ ->andWhere($query->expr()->isNull('t2.fileid'))
+ ->groupBy('t1.' . $idCol)
+ ->setMaxResults(self::CHUNK_SIZE);
+
+ $deleteQuery = $this->connection->getQueryBuilder();
+ $deleteQuery->delete($table)
+ ->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid')));
- $deletedInLastChunk = self::CHUNK_SIZE;
- while ($deletedInLastChunk === self::CHUNK_SIZE) {
- $chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
- $deletedInLastChunk = count($chunk);
+ $deletedInLastChunk = self::CHUNK_SIZE;
+ while ($deletedInLastChunk === self::CHUNK_SIZE) {
+ $chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ $deletedInLastChunk = count($chunk);
- $deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
- $deletedEntries += $deleteQuery->executeStatement();
+ $deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
+ $deletedEntries += $deleteQuery->executeStatement();
+ }
}
return $deletedEntries;
}
/**
+ * @param string $table
+ * @param string $idCol
+ * @param string $typeCol
+ * @param int $chunkSize
+ * @return \Iterator<int[]>
+ * @throws \OCP\DB\Exception
+ */
+ private function getItemIds(string $table, string $idCol, string $typeCol, int $chunkSize): \Iterator {
+ $query = $this->connection->getQueryBuilder();
+ $query->select($idCol)
+ ->from($table)
+ ->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
+ ->groupBy($idCol)
+ ->andWhere($query->expr()->gt($idCol, $query->createParameter('min_id')))
+ ->setMaxResults($chunkSize);
+
+ $minId = 0;
+ while (true) {
+ $query->setParameter('min_id', $minId);
+ $rows = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ if (count($rows) > 0) {
+ $minId = $rows[count($rows) - 1];
+ yield $rows;
+ } else {
+ break;
+ }
+ }
+ }
+
+ private function findMissingSources(array $ids): array {
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('fileid')
+ ->from('filecache')
+ ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)));
+ $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ return array_diff($ids, $found);
+ }
+
+ /**
* Deleting orphaned system tag mappings
*
* @return int Number of deleted entries
diff --git a/apps/files/lib/Command/DeleteOrphanedFiles.php b/apps/files/lib/Command/DeleteOrphanedFiles.php
index 50031da4f63..4bbee0b45f4 100644
--- a/apps/files/lib/Command/DeleteOrphanedFiles.php
+++ b/apps/files/lib/Command/DeleteOrphanedFiles.php
@@ -35,34 +35,29 @@ class DeleteOrphanedFiles extends Command {
public function execute(InputInterface $input, OutputInterface $output): int {
$deletedEntries = 0;
+ $fileIdsByStorage = [];
- $query = $this->connection->getQueryBuilder();
- $query->select('fc.fileid')
- ->from('filecache', 'fc')
- ->where($query->expr()->isNull('s.numeric_id'))
- ->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
- ->setMaxResults(self::CHUNK_SIZE);
+ $deletedStorages = array_diff($this->getReferencedStorages(), $this->getExistingStorages());
+
+ $deleteExtended = !$input->getOption('skip-filecache-extended');
+ if ($deleteExtended) {
+ $fileIdsByStorage = $this->getFileIdsForStorages($deletedStorages);
+ }
$deleteQuery = $this->connection->getQueryBuilder();
$deleteQuery->delete('filecache')
- ->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid')));
+ ->where($deleteQuery->expr()->in('storage', $deleteQuery->createParameter('storage_ids')));
- $deletedInLastChunk = self::CHUNK_SIZE;
- while ($deletedInLastChunk === self::CHUNK_SIZE) {
- $deletedInLastChunk = 0;
- $result = $query->execute();
- while ($row = $result->fetch()) {
- $deletedInLastChunk++;
- $deletedEntries += $deleteQuery->setParameter('objectid', (int)$row['fileid'])
- ->execute();
- }
- $result->closeCursor();
+ $deletedStorageChunks = array_chunk($deletedStorages, self::CHUNK_SIZE);
+ foreach ($deletedStorageChunks as $deletedStorageChunk) {
+ $deleteQuery->setParameter('storage_ids', $deletedStorageChunk, IQueryBuilder::PARAM_INT_ARRAY);
+ $deletedEntries += $deleteQuery->executeStatement();
}
$output->writeln("$deletedEntries orphaned file cache entries deleted");
- if (!$input->getOption('skip-filecache-extended')) {
- $deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended();
+ if ($deleteExtended) {
+ $deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended($fileIdsByStorage);
$output->writeln("$deletedFileCacheExtended orphaned file cache extended entries deleted");
}
@@ -72,28 +67,63 @@ class DeleteOrphanedFiles extends Command {
return self::SUCCESS;
}
- private function cleanupOrphanedFileCacheExtended(): int {
- $deletedEntries = 0;
-
+ private function getReferencedStorages(): array {
$query = $this->connection->getQueryBuilder();
- $query->select('fce.fileid')
- ->from('filecache_extended', 'fce')
- ->leftJoin('fce', 'filecache', 'fc', $query->expr()->eq('fce.fileid', 'fc.fileid'))
- ->where($query->expr()->isNull('fc.fileid'))
- ->setMaxResults(self::CHUNK_SIZE);
+ $query->select('storage')
+ ->from('filecache')
+ ->groupBy('storage')
+ ->runAcrossAllShards();
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
- $deleteQuery = $this->connection->getQueryBuilder();
- $deleteQuery->delete('filecache_extended')
- ->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('idsToDelete')));
+ private function getExistingStorages(): array {
+ $query = $this->connection->getQueryBuilder();
+ $query->select('numeric_id')
+ ->from('storages')
+ ->groupBy('numeric_id');
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
- $result = $query->executeQuery();
- while ($result->rowCount() > 0) {
- $idsToDelete = $result->fetchAll(\PDO::FETCH_COLUMN);
+ /**
+ * @param int[] $storageIds
+ * @return array<int, int[]>
+ */
+ private function getFileIdsForStorages(array $storageIds): array {
+ $query = $this->connection->getQueryBuilder();
+ $query->select('storage', 'fileid')
+ ->from('filecache')
+ ->where($query->expr()->in('storage', $query->createParameter('storage_ids')));
+
+ $result = [];
+ $storageIdChunks = array_chunk($storageIds, self::CHUNK_SIZE);
+ foreach ($storageIdChunks as $storageIdChunk) {
+ $query->setParameter('storage_ids', $storageIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
+ $chunk = $query->executeQuery()->fetchAll();
+ foreach ($chunk as $row) {
+ $result[$row['storage']][] = $row['fileid'];
+ }
+ }
+ return $result;
+ }
- $deleteQuery->setParameter('idsToDelete', $idsToDelete, IQueryBuilder::PARAM_INT_ARRAY);
- $deletedEntries += $deleteQuery->executeStatement();
+ /**
+ * @param array<int, int[]> $fileIdsByStorage
+ * @return int
+ */
+ private function cleanupOrphanedFileCacheExtended(array $fileIdsByStorage): int {
+ $deletedEntries = 0;
- $result = $query->executeQuery();
+ $deleteQuery = $this->connection->getQueryBuilder();
+ $deleteQuery->delete('filecache_extended')
+ ->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('file_ids')));
+
+ foreach ($fileIdsByStorage as $storageId => $fileIds) {
+ $deleteQuery->hintShardKey('storage', $storageId, true);
+ $fileChunks = array_chunk($fileIds, self::CHUNK_SIZE);
+ foreach ($fileChunks as $fileChunk) {
+ $deleteQuery->setParameter('file_ids', $fileChunk, IQueryBuilder::PARAM_INT_ARRAY);
+ $deletedEntries += $deleteQuery->executeStatement();
+ }
}
return $deletedEntries;
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-17 00:55:18 +0000