aboutsummaryrefslogtreecommitdiffstats
path: root/lib/private/Preview/BackgroundCleanupJob.php
diff options
context:
space:
mode:
Diffstat (limited to 'lib/private/Preview/BackgroundCleanupJob.php')
-rw-r--r--lib/private/Preview/BackgroundCleanupJob.php145
1 files changed, 95 insertions, 50 deletions
diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php
index ab40aeaaa79..3138abb1bf9 100644
--- a/lib/private/Preview/BackgroundCleanupJob.php
+++ b/lib/private/Preview/BackgroundCleanupJob.php
@@ -3,30 +3,14 @@
declare(strict_types=1);
/**
- * @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
- *
- * @author Roeland Jago Douma <roeland@famdouma.nl>
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Preview;
-use OC\BackgroundJob\TimedJob;
use OC\Preview\Storage\Root;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
@@ -35,29 +19,17 @@ use OCP\IDBConnection;
class BackgroundCleanupJob extends TimedJob {
- /** @var IDBConnection */
- private $connection;
-
- /** @var Root */
- private $previewFolder;
-
- /** @var bool */
- private $isCLI;
-
- /** @var IMimeTypeLoader */
- private $mimeTypeLoader;
-
- public function __construct(IDBConnection $connection,
- Root $previewFolder,
- IMimeTypeLoader $mimeTypeLoader,
- bool $isCLI) {
+ public function __construct(
+ ITimeFactory $timeFactory,
+ private IDBConnection $connection,
+ private Root $previewFolder,
+ private IMimeTypeLoader $mimeTypeLoader,
+ private bool $isCLI,
+ ) {
+ parent::__construct($timeFactory);
// Run at most once an hour
- $this->setInterval(3600);
-
- $this->connection = $connection;
- $this->previewFolder = $previewFolder;
- $this->isCLI = $isCLI;
- $this->mimeTypeLoader = $mimeTypeLoader;
+ $this->setInterval(60 * 60);
+ $this->setTimeSensitivity(self::TIME_INSENSITIVE);
}
public function run($argument) {
@@ -79,6 +51,11 @@ class BackgroundCleanupJob extends TimedJob {
}
private function getOldPreviewLocations(): \Iterator {
+ if ($this->connection->getShardDefinition('filecache')) {
+ // sharding is new enough that we don't need to support this
+ return;
+ }
+
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name')
->from('filecache', 'a')
@@ -86,18 +63,20 @@ class BackgroundCleanupJob extends TimedJob {
$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
))
->where(
- $qb->expr()->isNull('b.fileid')
- )->andWhere(
- $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId()))
- )->andWhere(
- $qb->expr()->like('a.name', $qb->createNamedParameter('__%'))
+ $qb->expr()->andX(
+ $qb->expr()->isNull('b.fileid'),
+ $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())),
+ $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())),
+ $qb->expr()->like('a.name', $qb->createNamedParameter('__%')),
+ $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory')))
+ )
);
if (!$this->isCLI) {
$qb->setMaxResults(10);
}
- $cursor = $qb->execute();
+ $cursor = $qb->executeQuery();
while ($row = $cursor->fetch()) {
yield $row['name'];
@@ -111,7 +90,7 @@ class BackgroundCleanupJob extends TimedJob {
$qb->select('path', 'mimetype')
->from('filecache')
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
- $cursor = $qb->execute();
+ $cursor = $qb->executeQuery();
$data = $cursor->fetch();
$cursor->closeCursor();
@@ -119,6 +98,15 @@ class BackgroundCleanupJob extends TimedJob {
return [];
}
+ if ($this->connection->getShardDefinition('filecache')) {
+ $chunks = $this->getAllPreviewIds($data['path'], 1000);
+ foreach ($chunks as $chunk) {
+ yield from $this->findMissingSources($chunk);
+ }
+
+ return;
+ }
+
/*
* This lovely like is the result of the way the new previews are stored
* We take the md5 of the name (fileid) and split the first 7 chars. That way
@@ -126,6 +114,21 @@ class BackgroundCleanupJob extends TimedJob {
*/
$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
+ /*
+ * Deleting a file will not delete related previews right away.
+ *
+ * A delete request is usually an HTTP request.
+ * The preview deleting is done by a background job to avoid timeouts.
+ *
+ * Previews for a file are stored within a folder in appdata_/preview using the fileid as folder name.
+ * Preview folders in oc_filecache are identified by a.storage, a.path (cf. $like) and a.mimetype.
+ *
+ * To find preview folders to delete, we query oc_filecache for a preview folder in app data, matching the preview folder structure
+ * and use the name to left join oc_filecache on a.name = b.fileid. A left join returns all rows from the left table (a),
+ * even if there are no matches in the right table (b).
+ *
+ * If the related file is deleted, b.fileid will be null and the preview folder can be deleted.
+ */
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name')
->from('filecache', 'a')
@@ -145,7 +148,7 @@ class BackgroundCleanupJob extends TimedJob {
$qb->setMaxResults(10);
}
- $cursor = $qb->execute();
+ $cursor = $qb->executeQuery();
while ($row = $cursor->fetch()) {
yield $row['name'];
@@ -153,4 +156,46 @@ class BackgroundCleanupJob extends TimedJob {
$cursor->closeCursor();
}
+
+ private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterator {
+ // See `getNewPreviewLocations` for some more info about the logic here
+ $like = $this->connection->escapeLikeParameter($previewRoot) . '/_/_/_/_/_/_/_/%';
+
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('name', 'fileid')
+ ->from('filecache')
+ ->where(
+ $qb->expr()->andX(
+ $qb->expr()->eq('storage', $qb->createNamedParameter($this->previewFolder->getStorageId())),
+ $qb->expr()->like('path', $qb->createNamedParameter($like)),
+ $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))),
+ $qb->expr()->gt('fileid', $qb->createParameter('min_id')),
+ )
+ )
+ ->orderBy('fileid', 'ASC')
+ ->setMaxResults($chunkSize);
+
+ $minId = 0;
+ while (true) {
+ $qb->setParameter('min_id', $minId);
+ $rows = $qb->executeQuery()->fetchAll();
+ if (count($rows) > 0) {
+ $minId = $rows[count($rows) - 1]['fileid'];
+ yield array_map(function ($row) {
+ return (int)$row['name'];
+ }, $rows);
+ } else {
+ break;
+ }
+ }
+ }
+
+ private function findMissingSources(array $ids): array {
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('fileid')
+ ->from('filecache')
+ ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)));
+ $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ return array_diff($ids, $found);
+ }
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-15 04:59:03 +0000