diff options
| -rw-r--r-- | lib/composer/composer/autoload_classmap.php | 1 | ||||
| -rw-r--r-- | lib/composer/composer/autoload_static.php | 1 | ||||
| -rw-r--r-- | lib/private/Preview/BackgroundCleanupJob.php | 94 | ||||
| -rw-r--r-- | lib/private/Preview/Storage/Root.php | 71 | ||||
| -rw-r--r-- | lib/private/Server.php | 4 | ||||
| -rw-r--r-- | tests/lib/Preview/BackgroundCleanupJobTest.php | 96 |
6 files changed, 223 insertions, 44 deletions
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 01dcecb0e47..53ffecddd3b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1153,6 +1153,7 @@ return array( 'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', 'OC\\Preview\\Watcher' => $baseDir . '/lib/private/Preview/Watcher.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 7a9bd5652d6..45ab02ded2b 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1182,6 +1182,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', 'OC\\Preview\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Preview/Watcher.php', diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 15bb6bd7188..e0546c4a11c 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -27,8 +27,9 @@ declare(strict_types=1); namespace OC\Preview; use OC\BackgroundJob\TimedJob; -use OC\Files\AppData\Factory; +use OC\Preview\Storage\Root; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IDBConnection; @@ -38,28 +39,47 @@ class BackgroundCleanupJob extends TimedJob { /** @var IDBConnection */ private $connection; - /** @var Factory */ - private $appDataFactory; + /** @var Root */ + private $previewFolder; /** @var bool */ private $isCLI; + /** @var IMimeTypeLoader */ + private $mimeTypeLoader; + public function __construct(IDBConnection $connection, - Factory $appDataFactory, + Root $previewFolder, + IMimeTypeLoader $mimeTypeLoader, bool $isCLI) { // Run at most once an hour $this->setInterval(3600); $this->connection = $connection; - $this->appDataFactory = $appDataFactory; + $this->previewFolder = $previewFolder; $this->isCLI = $isCLI; + $this->mimeTypeLoader = $mimeTypeLoader; } public function run($argument) { - $previews = $this->appDataFactory->get('preview'); + foreach ($this->getDeletedFiles() as $fileId) { + try { + $preview = $this->previewFolder->getFolder((string)$fileId); + $preview->delete(); + } catch (NotFoundException $e) { + // continue + } catch (NotPermittedException $e) { + // continue + } + } + } - $previewFodlerId = $previews->getId(); + private function getDeletedFiles(): \Iterator { + yield from $this->getOldPreviewLocations(); + yield from $this->getNewPreviewLocations(); + } + private function getOldPreviewLocations(): \Iterator { $qb = $this->connection->getQueryBuilder(); $qb->select('a.name') ->from('filecache', 'a') @@ -69,7 +89,9 @@ class BackgroundCleanupJob extends TimedJob { ->where( $qb->expr()->isNull('b.fileid') )->andWhere( - $qb->expr()->eq('a.parent', $qb->createNamedParameter($previewFodlerId)) + $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())) + )->andWhere( + $qb->expr()->like('a.name', $qb->createNamedParameter('__%')) ); if (!$this->isCLI) { @@ -79,14 +101,54 @@ class BackgroundCleanupJob extends TimedJob { $cursor = $qb->execute(); while ($row = $cursor->fetch()) { - try { - $preview = $previews->getFolder($row['name']); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue - } + yield $row['name']; + } + + $cursor->closeCursor(); + } + + private function getNewPreviewLocations(): \Iterator { + $qb = $this->connection->getQueryBuilder(); + $qb->select('path', 'mimetype') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); + $cursor = $qb->execute(); + $data = $cursor->fetch(); + $cursor->closeCursor(); + + if ($data === null) { + return []; + } + + /* + * This lovely like is the result of the way the new previews are stored + * We take the md5 of the name (fileid) and split the first 7 chars. That way + * there are not a gazillion files in the root of the preview appdata. + */ + $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; + + $qb = $this->connection->getQueryBuilder(); + $qb->select('a.name') + ->from('filecache', 'a') + ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( + $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + )) + ->where( + $qb->expr()->andX( + $qb->expr()->isNull('b.fileid'), + $qb->expr()->like('a.path', $qb->createNamedParameter($like)), + $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) + ) + ); + + if (!$this->isCLI) { + $qb->setMaxResults(10); + } + + $cursor = $qb->execute(); + + while ($row = $cursor->fetch()) { + yield $row['name']; } $cursor->closeCursor(); diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php new file mode 100644 index 00000000000..0e9d92a44c2 --- /dev/null +++ b/lib/private/Preview/Storage/Root.php @@ -0,0 +1,71 @@ +<?php + +declare(strict_types=1); +/** + * @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl> + * + * @author Roeland Jago Douma <roeland@famdouma.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Preview\Storage; + +use OC\Files\AppData\AppData; +use OC\SystemConfig; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\SimpleFS\ISimpleFolder; + +class Root extends AppData { + public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) { + parent::__construct($rootFolder, $systemConfig, 'preview'); + } + + + public function getFolder(string $name): ISimpleFolder { + $internalFolder = $this->getInternalFolder($name); + + try { + return parent::getFolder($internalFolder); + } catch (NotFoundException $e) { + /* + * The new folder structure is not found. + * Lets try the old one + */ + } + + return parent::getFolder($name); + } + + public function newFolder(string $name): ISimpleFolder { + $internalFolder = $this->getInternalFolder($name); + return parent::newFolder($internalFolder); + } + + /* + * Do not allow directory listing on this special root + * since it gets to big and time consuming + */ + public function getDirectoryListing(): array { + return []; + } + + private function getInternalFolder(string $name): string { + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index cf4f8cc097b..4fe89e3098e 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -281,7 +281,7 @@ class Server extends ServerContainer implements IServerContainer { return new PreviewManager( $c->getConfig(), $c->getRootFolder(), - $c->getAppDataDir('preview'), + new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview'), $c->getEventDispatcher(), $c->getGeneratorHelper(), $c->getSession()->get('user_id') @@ -291,7 +291,7 @@ class Server extends ServerContainer implements IServerContainer { $this->registerService(\OC\Preview\Watcher::class, function (Server $c) { return new \OC\Preview\Watcher( - $c->getAppDataDir('preview') + new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview') ); }); diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index db2bf09b6e5..cd9f6ef0399 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -22,10 +22,13 @@ namespace Test\Preview; -use OC\Files\AppData\Factory; use OC\Preview\BackgroundCleanupJob; +use OC\Preview\Storage\Root; use OC\PreviewManager; +use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IDBConnection; use Test\Traits\MountProviderTrait; use Test\Traits\UserTrait; @@ -47,9 +50,6 @@ class BackgroundCleanupJobTest extends \Test\TestCase { /** @var bool */ private $trashEnabled; - /** @var Factory */ - private $appDataFactory; - /** @var IDBConnection */ private $connection; @@ -59,6 +59,9 @@ class BackgroundCleanupJobTest extends \Test\TestCase { /** @var IRootFolder */ private $rootFolder; + /** @var IMimeTypeLoader */ + private $mimeTypeLoader; + protected function setUp(): void { parent::setUp(); @@ -76,13 +79,10 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->trashEnabled = $appManager->isEnabledForUser('files_trashbin', $this->userId); $appManager->disableApp('files_trashbin'); - $this->appDataFactory = new Factory( - \OC::$server->getRootFolder(), - \OC::$server->getSystemConfig() - ); $this->connection = \OC::$server->getDatabaseConnection(); $this->previewManager = \OC::$server->getPreviewManager(); $this->rootFolder = \OC::$server->getRootFolder(); + $this->mimeTypeLoader = \OC::$server->getMimeTypeLoader(); } protected function tearDown(): void { @@ -96,6 +96,13 @@ class BackgroundCleanupJobTest extends \Test\TestCase { parent::tearDown(); } + private function getRoot(): Root { + return new Root( + \OC::$server->getRootFolder(), + \OC::$server->getSystemConfig() + ); + } + private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); @@ -110,52 +117,89 @@ class BackgroundCleanupJobTest extends \Test\TestCase { return $files; } + private function countPreviews(Root $previewRoot, array $fileIds): int { + $i = 0; + + foreach ($fileIds as $fileId) { + try { + $previewRoot->getFolder((string)$fileId); + } catch (NotFoundException $e) { + continue; + } + + $i++; + } + + return $i; + } + public function testCleanupSystemCron() { $files = $this->setup11Previews(); + $fileIds = array_map(function (File $f) { + return $f->getId(); + }, $files); - $preview = $this->appDataFactory->get('preview'); - - $previews = $preview->getDirectoryListing(); - $this->assertCount(11, $previews); + $root = $this->getRoot(); - $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, true); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertCount(11, $previews); + $root = $this->getRoot(); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(0, $previews); + $root = $this->getRoot(); + $this->assertSame(0, $this->countPreviews($root, $fileIds)); } public function testCleanupAjax() { $files = $this->setup11Previews(); + $fileIds = array_map(function (File $f) { + return $f->getId(); + }, $files); - $preview = $this->appDataFactory->get('preview'); - - $previews = $preview->getDirectoryListing(); - $this->assertCount(11, $previews); + $root = $this->getRoot(); - $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, false); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertCount(11, $previews); + $root = $this->getRoot(); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(1, $previews); + $root = $this->getRoot(); + $this->assertSame(1, $this->countPreviews($root, $fileIds)); + $job->run([]); + + $root = $this->getRoot(); + $this->assertSame(0, $this->countPreviews($root, $fileIds)); + } + + public function testOldPreviews() { + $appdata = \OC::$server->getAppDataDir('preview'); + + $f1 = $appdata->newFolder('123456781'); + $f1->newFile('foo.jpg', 'foo'); + $f2 = $appdata->newFolder('123456782'); + $f2->newFile('foo.jpg', 'foo'); + + $appdata = \OC::$server->getAppDataDir('preview'); + $this->assertSame(2, count($appdata->getDirectoryListing())); + $job = new BackgroundCleanupJob($this->connection, $this->getRoot(), $this->mimeTypeLoader, true); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(0, $previews); + $appdata = \OC::$server->getAppDataDir('preview'); + $this->assertSame(0, count($appdata->getDirectoryListing())); } } |