aboutsummaryrefslogtreecommitdiffstats
path: root/apps/files/lib/BackgroundJob/ScanFiles.php
diff options
context:
space:
mode:
Diffstat (limited to 'apps/files/lib/BackgroundJob/ScanFiles.php')
-rw-r--r--apps/files/lib/BackgroundJob/ScanFiles.php174
1 files changed, 101 insertions, 73 deletions
diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php
index cb2b9d4b1c9..f3f9093d648 100644
--- a/apps/files/lib/BackgroundJob/ScanFiles.php
+++ b/apps/files/lib/BackgroundJob/ScanFiles.php
@@ -1,33 +1,21 @@
<?php
+
/**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Lukas Reschke <lukas@statuscode.ch>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2019-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
*/
namespace OCA\Files\BackgroundJob;
use OC\Files\Utils\Scanner;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\EventDispatcher\IEventDispatcher;
use OCP\IConfig;
use OCP\IDBConnection;
-use OCP\ILogger;
-use OCP\IUser;
-use OCP\IUserManager;
+use Psr\Log\LoggerInterface;
/**
* Class ScanFiles is a background job used to run the file scanner over the user
@@ -35,81 +23,121 @@ use OCP\IUserManager;
*
* @package OCA\Files\BackgroundJob
*/
-class ScanFiles extends \OC\BackgroundJob\TimedJob {
- /** @var IConfig */
- private $config;
- /** @var IUserManager */
- private $userManager;
- /** @var IDBConnection */
- private $dbConnection;
- /** @var ILogger */
- private $logger;
+class ScanFiles extends TimedJob {
/** Amount of users that should get scanned per execution */
- const USERS_PER_SESSION = 500;
+ public const USERS_PER_SESSION = 500;
- /**
- * @param IConfig|null $config
- * @param IUserManager|null $userManager
- * @param IDBConnection|null $dbConnection
- * @param ILogger|null $logger
- */
- public function __construct(IConfig $config = null,
- IUserManager $userManager = null,
- IDBConnection $dbConnection = null,
- ILogger $logger = null) {
+ public function __construct(
+ private IConfig $config,
+ private IEventDispatcher $dispatcher,
+ private LoggerInterface $logger,
+ private IDBConnection $connection,
+ ITimeFactory $time,
+ ) {
+ parent::__construct($time);
// Run once per 10 minutes
$this->setInterval(60 * 10);
-
- if (is_null($userManager) || is_null($config)) {
- $this->fixDIForJobs();
- } else {
- $this->config = $config;
- $this->userManager = $userManager;
- $this->logger = $logger;
- }
}
- protected function fixDIForJobs() {
- $this->config = \OC::$server->getConfig();
- $this->userManager = \OC::$server->getUserManager();
- $this->logger = \OC::$server->getLogger();
- }
-
- /**
- * @param IUser $user
- */
- protected function runScanner(IUser $user) {
+ protected function runScanner(string $user): void {
try {
$scanner = new Scanner(
- $user->getUID(),
- $this->dbConnection,
- $this->logger
+ $user,
+ null,
+ $this->dispatcher,
+ $this->logger
);
$scanner->backgroundScan('');
} catch (\Exception $e) {
- $this->logger->logException($e, ['app' => 'files']);
+ $this->logger->error($e->getMessage(), ['exception' => $e, 'app' => 'files']);
}
\OC_Util::tearDownFS();
}
/**
+ * Find a storage which have unindexed files and return a user with access to the storage
+ *
+ * @return string|false
+ */
+ private function getUserToScan() {
+ if ($this->connection->getShardDefinition('filecache')) {
+ // for sharded filecache, the "LIMIT" from the normal query doesn't work
+
+ // first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted.
+ // we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted
+ $query = $this->connection->getQueryBuilder();
+ $query->select('m.user_id')
+ ->from('filecache', 'f')
+ ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+ ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->setMaxResults(10)
+ ->groupBy('f.storage')
+ ->runAcrossAllShards();
+
+ $result = $query->executeQuery();
+ while ($res = $result->fetch()) {
+ if ($res['user_id']) {
+ return $res['user_id'];
+ }
+ }
+
+ // as a fallback, we try a slower approach where we find all mounted storages first
+ // this is essentially doing the inner join manually
+ $storages = $this->getAllMountedStorages();
+
+ $query = $this->connection->getQueryBuilder();
+ $query->select('m.user_id')
+ ->from('filecache', 'f')
+ ->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+ ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY)))
+ ->setMaxResults(1)
+ ->runAcrossAllShards();
+ return $query->executeQuery()->fetchOne();
+ } else {
+ $query = $this->connection->getQueryBuilder();
+ $query->select('m.user_id')
+ ->from('filecache', 'f')
+ ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+ ->where($query->expr()->eq('f.size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+ ->setMaxResults(1)
+ ->runAcrossAllShards();
+
+ return $query->executeQuery()->fetchOne();
+ }
+ }
+
+ private function getAllMountedStorages(): array {
+ $query = $this->connection->getQueryBuilder();
+ $query->selectDistinct('storage_id')
+ ->from('mounts');
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
+
+ /**
* @param $argument
* @throws \Exception
*/
protected function run($argument) {
- $offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
- $users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
- if (!count($users)) {
- // No users found, reset offset and retry
- $offset = 0;
- $users = $this->userManager->search('', self::USERS_PER_SESSION);
+ if ($this->config->getSystemValueBool('files_no_background_scan', false)) {
+ return;
}
- $offset += self::USERS_PER_SESSION;
- $this->config->setAppValue('files', 'cronjob_scan_files', $offset);
-
- foreach ($users as $user) {
+ $usersScanned = 0;
+ $lastUser = '';
+ $user = $this->getUserToScan();
+ while ($user && $usersScanned < self::USERS_PER_SESSION && $lastUser !== $user) {
$this->runScanner($user);
+ $lastUser = $user;
+ $user = $this->getUserToScan();
+ $usersScanned += 1;
+ }
+
+ if ($lastUser === $user) {
+ $this->logger->warning("User $user still has unscanned files after running background scan, background scan might be stopped prematurely");
}
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-15 18:30:03 +0000