6 files changed, 157 insertions, 187 deletions

diff --git a/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php b/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php
index 2f700b8773e..a1032b2787d 100644
--- a/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php
+++ b/apps/files/lib/BackgroundJob/CleanupDirectEditingTokens.php
@@ -3,26 +3,8 @@
 declare(strict_types=1);
 
 /**
- * @copyright Copyright (c) 2016 Julius Härtl <jus@bitgrid.net>
- *
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Julius Härtl <jus@bitgrid.net>
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
+ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 namespace OCA\Files\BackgroundJob;
 
@@ -31,15 +13,12 @@ use OCP\BackgroundJob\TimedJob;
 use OCP\DirectEditing\IManager;
 
 class CleanupDirectEditingTokens extends TimedJob {
-	private const INTERVAL_MINUTES = 15 * 60;
-
-	private IManager $manager;
-
-	public function __construct(ITimeFactory $time,
-		IManager $manager) {
+	public function __construct(
+		ITimeFactory $time,
+		private IManager $manager,
+	) {
 		parent::__construct($time);
-		$this->interval = self::INTERVAL_MINUTES;
-		$this->manager = $manager;
+		$this->setInterval(15 * 60);
 	}
 
 	/**
diff --git a/apps/files/lib/BackgroundJob/CleanupFileLocks.php b/apps/files/lib/BackgroundJob/CleanupFileLocks.php
index 7ff28a50155..91bb145884b 100644
--- a/apps/files/lib/BackgroundJob/CleanupFileLocks.php
+++ b/apps/files/lib/BackgroundJob/CleanupFileLocks.php
@@ -1,50 +1,28 @@
 <?php
+
 /**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Max Kovalenko <mxss1998@yandex.ru>
- * @author Morris Jobke <hey@morrisjobke.de>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2019-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
  */
 namespace OCA\Files\BackgroundJob;
 
 use OC\Lock\DBLockingProvider;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\TimedJob;
+use OCP\Lock\ILockingProvider;
+use OCP\Server;
 
 /**
  * Clean up all file locks that are expired for the DB file locking provider
  */
 class CleanupFileLocks extends TimedJob {
 	/**
-	 * Default interval in minutes
-	 *
-	 * @var int $defaultIntervalMin
-	 **/
-	protected $defaultIntervalMin = 5;
-
-	/**
 	 * sets the correct interval for this timed job
 	 */
 	public function __construct(ITimeFactory $time) {
 		parent::__construct($time);
-
-		$this->interval = $this->defaultIntervalMin * 60;
+		$this->setInterval(5 * 60);
 	}
 
 	/**
@@ -54,7 +32,7 @@ class CleanupFileLocks extends TimedJob {
 	 * @throws \Exception
 	 */
 	public function run($argument) {
-		$lockingProvider = \OC::$server->getLockingProvider();
+		$lockingProvider = Server::get(ILockingProvider::class);
 		if ($lockingProvider instanceof DBLockingProvider) {
 			$lockingProvider->cleanExpiredLocks();
 		}
diff --git a/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php b/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php
index 9839bcde5b7..8a20b6dfb0c 100644
--- a/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php
+++ b/apps/files/lib/BackgroundJob/DeleteExpiredOpenLocalEditor.php
@@ -3,50 +3,29 @@
 declare(strict_types=1);
 
 /**
- * @copyright Copyright (c) 2022 Joas Schilling <coding@schilljs.com>
- *
- * @author Joas Schilling <coding@schilljs.com>
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
+ * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
 namespace OCA\Files\BackgroundJob;
 
 use OCA\Files\Db\OpenLocalEditorMapper;
 use OCP\AppFramework\Utility\ITimeFactory;
-use OCP\BackgroundJob\IJob;
 use OCP\BackgroundJob\TimedJob;
 
 /**
  * Delete all expired "Open local editor" token
  */
 class DeleteExpiredOpenLocalEditor extends TimedJob {
-	protected OpenLocalEditorMapper $mapper;
-
 	public function __construct(
 		ITimeFactory $time,
-		OpenLocalEditorMapper $mapper
+		protected OpenLocalEditorMapper $mapper,
 	) {
 		parent::__construct($time);
-		$this->mapper = $mapper;
 
 		// Run every 12h
 		$this->interval = 12 * 3600;
-		$this->setTimeSensitivity(IJob::TIME_INSENSITIVE);
+		$this->setTimeSensitivity(self::TIME_INSENSITIVE);
 	}
 
 	/**
diff --git a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
index 6163955ddfd..b925974f24a 100644
--- a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
+++ b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php
@@ -1,26 +1,9 @@
 <?php
+
 /**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Joas Schilling <coding@schilljs.com>
- * @author Vincent Petry <vincent@nextcloud.com>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2019-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
  */
 namespace OCA\Files\BackgroundJob;
 
@@ -35,7 +18,6 @@ use Psr\Log\LoggerInterface;
  */
 class DeleteOrphanedItems extends TimedJob {
 	public const CHUNK_SIZE = 200;
-	protected $defaultIntervalMin = 60;
 
 	/**
 	 * sets the correct interval for this timed job
@@ -46,7 +28,7 @@ class DeleteOrphanedItems extends TimedJob {
 		protected LoggerInterface $logger,
 	) {
 		parent::__construct($time);
-		$this->interval = $this->defaultIntervalMin * 60;
+		$this->setInterval(60 * 60);
 	}
 
 	/**
@@ -69,38 +51,87 @@ class DeleteOrphanedItems extends TimedJob {
 	 * @param string $typeCol
 	 * @return int Number of deleted entries
 	 */
-	protected function cleanUp($table, $idCol, $typeCol) {
+	protected function cleanUp(string $table, string $idCol, string $typeCol): int {
 		$deletedEntries = 0;
 
-		$query = $this->connection->getQueryBuilder();
-		$query->select('t1.' . $idCol)
-			->from($table, 't1')
-			->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
-			->andWhere($query->expr()->isNull('t2.fileid'))
-			->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
-			->groupBy('t1.' . $idCol)
-			->setMaxResults(self::CHUNK_SIZE);
-
 		$deleteQuery = $this->connection->getQueryBuilder();
 		$deleteQuery->delete($table)
 			->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid')));
 
-		$deletedInLastChunk = self::CHUNK_SIZE;
-		while ($deletedInLastChunk === self::CHUNK_SIZE) {
-			$result = $query->execute();
-			$deletedInLastChunk = 0;
-			while ($row = $result->fetch()) {
-				$deletedInLastChunk++;
-				$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row[$idCol])
-					->execute();
+		if ($this->connection->getShardDefinition('filecache')) {
+			$sourceIdChunks = $this->getItemIds($table, $idCol, $typeCol, 1000);
+			foreach ($sourceIdChunks as $sourceIdChunk) {
+				$deletedSources = $this->findMissingSources($sourceIdChunk);
+				$deleteQuery->setParameter('objectid', $deletedSources, IQueryBuilder::PARAM_INT_ARRAY);
+				$deletedEntries += $deleteQuery->executeStatement();
+			}
+		} else {
+			$query = $this->connection->getQueryBuilder();
+			$query->select('t1.' . $idCol)
+				->from($table, 't1')
+				->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
+				->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
+				->andWhere($query->expr()->isNull('t2.fileid'))
+				->groupBy('t1.' . $idCol)
+				->setMaxResults(self::CHUNK_SIZE);
+
+			$deleteQuery = $this->connection->getQueryBuilder();
+			$deleteQuery->delete($table)
+				->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid')));
+
+			$deletedInLastChunk = self::CHUNK_SIZE;
+			while ($deletedInLastChunk === self::CHUNK_SIZE) {
+				$chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+				$deletedInLastChunk = count($chunk);
+
+				$deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
+				$deletedEntries += $deleteQuery->executeStatement();
 			}
-			$result->closeCursor();
 		}
 
 		return $deletedEntries;
 	}
 
 	/**
+	 * @param string $table
+	 * @param string $idCol
+	 * @param string $typeCol
+	 * @param int $chunkSize
+	 * @return \Iterator<int[]>
+	 * @throws \OCP\DB\Exception
+	 */
+	private function getItemIds(string $table, string $idCol, string $typeCol, int $chunkSize): \Iterator {
+		$query = $this->connection->getQueryBuilder();
+		$query->select($idCol)
+			->from($table)
+			->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
+			->groupBy($idCol)
+			->andWhere($query->expr()->gt($idCol, $query->createParameter('min_id')))
+			->setMaxResults($chunkSize);
+
+		$minId = 0;
+		while (true) {
+			$query->setParameter('min_id', $minId);
+			$rows = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+			if (count($rows) > 0) {
+				$minId = $rows[count($rows) - 1];
+				yield $rows;
+			} else {
+				break;
+			}
+		}
+	}
+
+	private function findMissingSources(array $ids): array {
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('fileid')
+			->from('filecache')
+			->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)));
+		$found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+		return array_diff($ids, $found);
+	}
+
+	/**
 	 * Deleting orphaned system tag mappings
 	 *
 	 * @return int Number of deleted entries
diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php
index 8dc9dcf37ff..f3f9093d648 100644
--- a/apps/files/lib/BackgroundJob/ScanFiles.php
+++ b/apps/files/lib/BackgroundJob/ScanFiles.php
@@ -1,25 +1,9 @@
 <?php
+
 /**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Lukas Reschke <lukas@statuscode.ch>
- * @author Robin Appelman <robin@icewind.nl>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2019-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
  */
 
 namespace OCA\Files\BackgroundJob;
@@ -40,29 +24,19 @@ use Psr\Log\LoggerInterface;
  * @package OCA\Files\BackgroundJob
  */
 class ScanFiles extends TimedJob {
-	private IConfig $config;
-	private IEventDispatcher $dispatcher;
-	private LoggerInterface $logger;
-	private IDBConnection $connection;
-
 	/** Amount of users that should get scanned per execution */
 	public const USERS_PER_SESSION = 500;
 
 	public function __construct(
-		IConfig $config,
-		IEventDispatcher $dispatcher,
-		LoggerInterface $logger,
-		IDBConnection $connection,
-		ITimeFactory $time
+		private IConfig $config,
+		private IEventDispatcher $dispatcher,
+		private LoggerInterface $logger,
+		private IDBConnection $connection,
+		ITimeFactory $time,
 	) {
 		parent::__construct($time);
 		// Run once per 10 minutes
 		$this->setInterval(60 * 10);
-
-		$this->config = $config;
-		$this->dispatcher = $dispatcher;
-		$this->logger = $logger;
-		$this->connection = $connection;
 	}
 
 	protected function runScanner(string $user): void {
@@ -86,15 +60,61 @@ class ScanFiles extends TimedJob {
 	 * @return string|false
 	 */
 	private function getUserToScan() {
+		if ($this->connection->getShardDefinition('filecache')) {
+			// for sharded filecache, the "LIMIT" from the normal query doesn't work
+
+			// first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted.
+			// we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->setMaxResults(10)
+				->groupBy('f.storage')
+				->runAcrossAllShards();
+
+			$result = $query->executeQuery();
+			while ($res = $result->fetch()) {
+				if ($res['user_id']) {
+					return $res['user_id'];
+				}
+			}
+
+			// as a fallback, we try a slower approach where we find all mounted storages first
+			// this is essentially doing the inner join manually
+			$storages = $this->getAllMountedStorages();
+
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY)))
+				->setMaxResults(1)
+				->runAcrossAllShards();
+			return $query->executeQuery()->fetchOne();
+		} else {
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->setMaxResults(1)
+				->runAcrossAllShards();
+
+			return $query->executeQuery()->fetchOne();
+		}
+	}
+
+	private function getAllMountedStorages(): array {
 		$query = $this->connection->getQueryBuilder();
-		$query->select('user_id')
-			->from('filecache', 'f')
-			->innerJoin('f', 'mounts', 'm', $query->expr()->eq('storage_id', 'storage'))
-			->where($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->gt('parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
-			->setMaxResults(1);
-
-		return $query->executeQuery()->fetchOne();
+		$query->selectDistinct('storage_id')
+			->from('mounts');
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
 	}
 
 	/**
diff --git a/apps/files/lib/BackgroundJob/TransferOwnership.php b/apps/files/lib/BackgroundJob/TransferOwnership.php
index 5cf35396d43..de8d1989733 100644
--- a/apps/files/lib/BackgroundJob/TransferOwnership.php
+++ b/apps/files/lib/BackgroundJob/TransferOwnership.php
@@ -3,25 +3,8 @@
 declare(strict_types=1);
 
 /**
- * @copyright Copyright (c) 2019, Roeland Jago Douma <roeland@famdouma.nl>
- *
- * @author Roeland Jago Douma <roeland@famdouma.nl>
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
+ * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 namespace OCA\Files\BackgroundJob;