diff options
Diffstat (limited to 'apps/files/lib/BackgroundJob')
5 files changed, 139 insertions, 68 deletions
diff --git a/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php b/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php index 376e1049469..a1032b2787d 100644 --- a/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php +++ b/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php @@ -13,15 +13,12 @@ use OCP\BackgroundJob\TimedJob; use OCP\DirectEditing\IManager; class CleanupDirectEditingTokens extends TimedJob { - private const INTERVAL_MINUTES = 15 * 60; - - private IManager $manager; - - public function __construct(ITimeFactory $time, - IManager $manager) { + public function __construct( + ITimeFactory $time, + private IManager $manager, + ) { parent::__construct($time); - $this->interval = self::INTERVAL_MINUTES; - $this->manager = $manager; + $this->setInterval(15 * 60); } /** diff --git a/apps/files/lib/BackgroundJob/CleanupFileLocks.php b/apps/files/lib/BackgroundJob/CleanupFileLocks.php index 69efb664302..91bb145884b 100644 --- a/apps/files/lib/BackgroundJob/CleanupFileLocks.php +++ b/apps/files/lib/BackgroundJob/CleanupFileLocks.php @@ -10,25 +10,19 @@ namespace OCA\Files\BackgroundJob; use OC\Lock\DBLockingProvider; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; +use OCP\Lock\ILockingProvider; +use OCP\Server; /** * Clean up all file locks that are expired for the DB file locking provider */ class CleanupFileLocks extends TimedJob { /** - * Default interval in minutes - * - * @var int $defaultIntervalMin - **/ - protected $defaultIntervalMin = 5; - - /** * sets the correct interval for this timed job */ public function __construct(ITimeFactory $time) { parent::__construct($time); - - $this->interval = $this->defaultIntervalMin * 60; + $this->setInterval(5 * 60); } /** @@ -38,7 +32,7 @@ class CleanupFileLocks extends TimedJob { * @throws \Exception */ public function run($argument) { - $lockingProvider = \OC::$server->getLockingProvider(); + $lockingProvider = Server::get(ILockingProvider::class); if ($lockingProvider instanceof DBLockingProvider) { $lockingProvider->cleanExpiredLocks(); } diff --git a/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php b/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php index 81f3d229bc0..8a20b6dfb0c 100644 --- a/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php +++ b/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php @@ -11,25 +11,21 @@ namespace OCA\Files\BackgroundJob; use OCA\Files\Db\OpenLocalEditorMapper; use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\IJob; use OCP\BackgroundJob\TimedJob; /** * Delete all expired "Open local editor" token */ class DeleteExpiredOpenLocalEditor extends TimedJob { - protected OpenLocalEditorMapper $mapper; - public function __construct( ITimeFactory $time, - OpenLocalEditorMapper $mapper + protected OpenLocalEditorMapper $mapper, ) { parent::__construct($time); - $this->mapper = $mapper; // Run every 12h $this->interval = 12 * 3600; - $this->setTimeSensitivity(IJob::TIME_INSENSITIVE); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); } /** diff --git a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php index a4b2a6cf3f8..b925974f24a 100644 --- a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php +++ b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php @@ -18,7 +18,6 @@ use Psr\Log\LoggerInterface; */ class DeleteOrphanedItems extends TimedJob { public const CHUNK_SIZE = 200; - protected $defaultIntervalMin = 60; /** * sets the correct interval for this timed job @@ -29,7 +28,7 @@ class DeleteOrphanedItems extends TimedJob { protected LoggerInterface $logger, ) { parent::__construct($time); - $this->interval = $this->defaultIntervalMin * 60; + $this->setInterval(60 * 60); } /** @@ -52,38 +51,87 @@ class DeleteOrphanedItems extends TimedJob { * @param string $typeCol * @return int Number of deleted entries */ - protected function cleanUp($table, $idCol, $typeCol) { + protected function cleanUp(string $table, string $idCol, string $typeCol): int { $deletedEntries = 0; - $query = $this->connection->getQueryBuilder(); - $query->select('t1.' . $idCol) - ->from($table, 't1') - ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) - ->andWhere($query->expr()->isNull('t2.fileid')) - ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid')) - ->groupBy('t1.' . $idCol) - ->setMaxResults(self::CHUNK_SIZE); - $deleteQuery = $this->connection->getQueryBuilder(); $deleteQuery->delete($table) ->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid'))); - $deletedInLastChunk = self::CHUNK_SIZE; - while ($deletedInLastChunk === self::CHUNK_SIZE) { - $result = $query->execute(); - $deletedInLastChunk = 0; - while ($row = $result->fetch()) { - $deletedInLastChunk++; - $deletedEntries += $deleteQuery->setParameter('objectid', (int) $row[$idCol]) - ->execute(); + if ($this->connection->getShardDefinition('filecache')) { + $sourceIdChunks = $this->getItemIds($table, $idCol, $typeCol, 1000); + foreach ($sourceIdChunks as $sourceIdChunk) { + $deletedSources = $this->findMissingSources($sourceIdChunk); + $deleteQuery->setParameter('objectid', $deletedSources, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); + } + } else { + $query = $this->connection->getQueryBuilder(); + $query->select('t1.' . $idCol) + ->from($table, 't1') + ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) + ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid')) + ->andWhere($query->expr()->isNull('t2.fileid')) + ->groupBy('t1.' . $idCol) + ->setMaxResults(self::CHUNK_SIZE); + + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete($table) + ->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid'))); + + $deletedInLastChunk = self::CHUNK_SIZE; + while ($deletedInLastChunk === self::CHUNK_SIZE) { + $chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + $deletedInLastChunk = count($chunk); + + $deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); } - $result->closeCursor(); } return $deletedEntries; } /** + * @param string $table + * @param string $idCol + * @param string $typeCol + * @param int $chunkSize + * @return \Iterator<int[]> + * @throws \OCP\DB\Exception + */ + private function getItemIds(string $table, string $idCol, string $typeCol, int $chunkSize): \Iterator { + $query = $this->connection->getQueryBuilder(); + $query->select($idCol) + ->from($table) + ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) + ->groupBy($idCol) + ->andWhere($query->expr()->gt($idCol, $query->createParameter('min_id'))) + ->setMaxResults($chunkSize); + + $minId = 0; + while (true) { + $query->setParameter('min_id', $minId); + $rows = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + if (count($rows) > 0) { + $minId = $rows[count($rows) - 1]; + yield $rows; + } else { + break; + } + } + } + + private function findMissingSources(array $ids): array { + $qb = $this->connection->getQueryBuilder(); + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + return array_diff($ids, $found); + } + + /** * Deleting orphaned system tag mappings * * @return int Number of deleted entries diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php index 97122e738e2..f3f9093d648 100644 --- a/apps/files/lib/BackgroundJob/ScanFiles.php +++ b/apps/files/lib/BackgroundJob/ScanFiles.php @@ -24,29 +24,19 @@ use Psr\Log\LoggerInterface; * @package OCA\Files\BackgroundJob */ class ScanFiles extends TimedJob { - private IConfig $config; - private IEventDispatcher $dispatcher; - private LoggerInterface $logger; - private IDBConnection $connection; - /** Amount of users that should get scanned per execution */ public const USERS_PER_SESSION = 500; public function __construct( - IConfig $config, - IEventDispatcher $dispatcher, - LoggerInterface $logger, - IDBConnection $connection, - ITimeFactory $time + private IConfig $config, + private IEventDispatcher $dispatcher, + private LoggerInterface $logger, + private IDBConnection $connection, + ITimeFactory $time, ) { parent::__construct($time); // Run once per 10 minutes $this->setInterval(60 * 10); - - $this->config = $config; - $this->dispatcher = $dispatcher; - $this->logger = $logger; - $this->connection = $connection; } protected function runScanner(string $user): void { @@ -70,15 +60,61 @@ class ScanFiles extends TimedJob { * @return string|false */ private function getUserToScan() { + if ($this->connection->getShardDefinition('filecache')) { + // for sharded filecache, the "LIMIT" from the normal query doesn't work + + // first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted. + // we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted + $query = $this->connection->getQueryBuilder(); + $query->select('m.user_id') + ->from('filecache', 'f') + ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->setMaxResults(10) + ->groupBy('f.storage') + ->runAcrossAllShards(); + + $result = $query->executeQuery(); + while ($res = $result->fetch()) { + if ($res['user_id']) { + return $res['user_id']; + } + } + + // as a fallback, we try a slower approach where we find all mounted storages first + // this is essentially doing the inner join manually + $storages = $this->getAllMountedStorages(); + + $query = $this->connection->getQueryBuilder(); + $query->select('m.user_id') + ->from('filecache', 'f') + ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY))) + ->setMaxResults(1) + ->runAcrossAllShards(); + return $query->executeQuery()->fetchOne(); + } else { + $query = $this->connection->getQueryBuilder(); + $query->select('m.user_id') + ->from('filecache', 'f') + ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage')) + ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->setMaxResults(1) + ->runAcrossAllShards(); + + return $query->executeQuery()->fetchOne(); + } + } + + private function getAllMountedStorages(): array { $query = $this->connection->getQueryBuilder(); - $query->select('user_id') - ->from('filecache', 'f') - ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('storage_id', 'storage')) - ->where($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->gt('parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) - ->setMaxResults(1); - - return $query->executeQuery()->fetchOne(); + $query->selectDistinct('storage_id') + ->from('mounts'); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); } /** |