1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Core\BackgroundJobs;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\IAppConfig;
use OCP\IConfig;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
class GenerateMetadataJob extends TimedJob {
// Default file size limit for metadata generation (MBytes).
protected const DEFAULT_MAX_FILESIZE = 256;
public function __construct(
ITimeFactory $time,
private IConfig $config,
private IAppConfig $appConfig,
private IRootFolder $rootFolder,
private IUserManager $userManager,
private IFilesMetadataManager $filesMetadataManager,
private IJobList $jobList,
private LoggerInterface $logger,
) {
parent::__construct($time);
$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
$this->setInterval(24 * 3600);
}
protected function run(mixed $argument): void {
if ($this->appConfig->getValueBool('core', 'metadataGenerationDone', false)) {
return;
}
$lastHandledUser = $this->appConfig->getValueString('core', 'metadataGenerationLastHandledUser', '');
$users = $this->userManager->search('');
// we'll only start timer once we have found a valid user to handle
// meaning NOW if we have not handled any user from a previous run
$startTime = ($lastHandledUser === '') ? time() : null;
foreach ($users as $user) {
$userId = $user->getUID();
// if we already handled a previous run, we start timer only when we face the last handled user
if ($startTime === null) {
if ($userId === $lastHandledUser) {
$startTime = time();
}
continue;
}
$this->appConfig->setValueString('core', 'metadataGenerationLastHandledUser', $userId);
$this->scanFilesForUser($user->getUID());
// Stop if execution time is more than one hour.
if (time() - $startTime > 3600) {
return;
}
}
$this->appConfig->deleteKey('core', 'metadataGenerationLastHandledUser');
$this->appConfig->setValueBool('core', 'metadataGenerationDone', true);
}
private function scanFilesForUser(string $userId): void {
$userFolder = $this->rootFolder->getUserFolder($userId);
$this->scanFolder($userFolder);
}
private function scanFolder(Folder $folder): void {
// Do not scan share and other moveable mounts.
if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) {
return;
}
foreach ($folder->getDirectoryListing() as $node) {
if ($node instanceof Folder) {
$this->scanFolder($node);
continue;
}
// Don't generate metadata for files bigger than configured metadata_max_filesize
// Files are loaded in memory so very big files can lead to an OOM on the server
$nodeSize = $node->getSize();
$nodeLimit = $this->config->getSystemValueInt('metadata_max_filesize', self::DEFAULT_MAX_FILESIZE);
if ($nodeSize > $nodeLimit * 1000000) {
$this->logger->debug('Skipping generating metadata for fileid ' . $node->getId() . " as its size exceeds configured 'metadata_max_filesize'.");
continue;
}
try {
$this->filesMetadataManager->getMetadata($node->getId(), false);
} catch (FilesMetadataNotFoundException) {
try {
$this->filesMetadataManager->refreshMetadata(
$node,
IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND
);
} catch (\Throwable $ex) {
$this->logger->warning('Error while generating metadata for fileid ' . $node->getId(), ['exception' => $ex]);
}
}
}
}
}
|
