From 6c603e8e7d1ac52eeefdda939fa96a0c27b7c3da Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 30 Jan 2020 11:37:01 +0100 Subject: [PATCH] Move to subfolders for preview files Else the number of files can grow very large very quickly in the preview folder. Esp on large systems. This generates the md5 of the fileid. And then creates folders of the first 7 charts. In that folder is then a folder with the fileid. And inside there are the previews. Signed-off-by: Roeland Jago Douma --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Preview/BackgroundCleanupJob.php | 94 ++++++++++++++---- lib/private/Preview/Storage/Root.php | 71 ++++++++++++++ lib/private/Server.php | 4 +- .../lib/Preview/BackgroundCleanupJobTest.php | 96 ++++++++++++++----- 6 files changed, 223 insertions(+), 44 deletions(-) create mode 100644 lib/private/Preview/Storage/Root.php diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 01dcecb0e47..53ffecddd3b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1153,6 +1153,7 @@ return array( 'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', 'OC\\Preview\\Watcher' => $baseDir . '/lib/private/Preview/Watcher.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 7a9bd5652d6..45ab02ded2b 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1182,6 +1182,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', 'OC\\Preview\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Preview/Watcher.php', diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 15bb6bd7188..e0546c4a11c 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -27,8 +27,9 @@ declare(strict_types=1); namespace OC\Preview; use OC\BackgroundJob\TimedJob; -use OC\Files\AppData\Factory; +use OC\Preview\Storage\Root; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IDBConnection; @@ -38,28 +39,47 @@ class BackgroundCleanupJob extends TimedJob { /** @var IDBConnection */ private $connection; - /** @var Factory */ - private $appDataFactory; + /** @var Root */ + private $previewFolder; /** @var bool */ private $isCLI; + /** @var IMimeTypeLoader */ + private $mimeTypeLoader; + public function __construct(IDBConnection $connection, - Factory $appDataFactory, + Root $previewFolder, + IMimeTypeLoader $mimeTypeLoader, bool $isCLI) { // Run at most once an hour $this->setInterval(3600); $this->connection = $connection; - $this->appDataFactory = $appDataFactory; + $this->previewFolder = $previewFolder; $this->isCLI = $isCLI; + $this->mimeTypeLoader = $mimeTypeLoader; } public function run($argument) { - $previews = $this->appDataFactory->get('preview'); + foreach ($this->getDeletedFiles() as $fileId) { + try { + $preview = $this->previewFolder->getFolder((string)$fileId); + $preview->delete(); + } catch (NotFoundException $e) { + // continue + } catch (NotPermittedException $e) { + // continue + } + } + } - $previewFodlerId = $previews->getId(); + private function getDeletedFiles(): \Iterator { + yield from $this->getOldPreviewLocations(); + yield from $this->getNewPreviewLocations(); + } + private function getOldPreviewLocations(): \Iterator { $qb = $this->connection->getQueryBuilder(); $qb->select('a.name') ->from('filecache', 'a') @@ -69,7 +89,9 @@ class BackgroundCleanupJob extends TimedJob { ->where( $qb->expr()->isNull('b.fileid') )->andWhere( - $qb->expr()->eq('a.parent', $qb->createNamedParameter($previewFodlerId)) + $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())) + )->andWhere( + $qb->expr()->like('a.name', $qb->createNamedParameter('__%')) ); if (!$this->isCLI) { @@ -79,14 +101,54 @@ class BackgroundCleanupJob extends TimedJob { $cursor = $qb->execute(); while ($row = $cursor->fetch()) { - try { - $preview = $previews->getFolder($row['name']); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue - } + yield $row['name']; + } + + $cursor->closeCursor(); + } + + private function getNewPreviewLocations(): \Iterator { + $qb = $this->connection->getQueryBuilder(); + $qb->select('path', 'mimetype') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); + $cursor = $qb->execute(); + $data = $cursor->fetch(); + $cursor->closeCursor(); + + if ($data === null) { + return []; + } + + /* + * This lovely like is the result of the way the new previews are stored + * We take the md5 of the name (fileid) and split the first 7 chars. That way + * there are not a gazillion files in the root of the preview appdata. + */ + $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; + + $qb = $this->connection->getQueryBuilder(); + $qb->select('a.name') + ->from('filecache', 'a') + ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( + $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + )) + ->where( + $qb->expr()->andX( + $qb->expr()->isNull('b.fileid'), + $qb->expr()->like('a.path', $qb->createNamedParameter($like)), + $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) + ) + ); + + if (!$this->isCLI) { + $qb->setMaxResults(10); + } + + $cursor = $qb->execute(); + + while ($row = $cursor->fetch()) { + yield $row['name']; } $cursor->closeCursor(); diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php new file mode 100644 index 00000000000..0e9d92a44c2 --- /dev/null +++ b/lib/private/Preview/Storage/Root.php @@ -0,0 +1,71 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Preview\Storage; + +use OC\Files\AppData\AppData; +use OC\SystemConfig; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\SimpleFS\ISimpleFolder; + +class Root extends AppData { + public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) { + parent::__construct($rootFolder, $systemConfig, 'preview'); + } + + + public function getFolder(string $name): ISimpleFolder { + $internalFolder = $this->getInternalFolder($name); + + try { + return parent::getFolder($internalFolder); + } catch (NotFoundException $e) { + /* + * The new folder structure is not found. + * Lets try the old one + */ + } + + return parent::getFolder($name); + } + + public function newFolder(string $name): ISimpleFolder { + $internalFolder = $this->getInternalFolder($name); + return parent::newFolder($internalFolder); + } + + /* + * Do not allow directory listing on this special root + * since it gets to big and time consuming + */ + public function getDirectoryListing(): array { + return []; + } + + private function getInternalFolder(string $name): string { + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index cf4f8cc097b..4fe89e3098e 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -281,7 +281,7 @@ class Server extends ServerContainer implements IServerContainer { return new PreviewManager( $c->getConfig(), $c->getRootFolder(), - $c->getAppDataDir('preview'), + new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview'), $c->getEventDispatcher(), $c->getGeneratorHelper(), $c->getSession()->get('user_id') @@ -291,7 +291,7 @@ class Server extends ServerContainer implements IServerContainer { $this->registerService(\OC\Preview\Watcher::class, function (Server $c) { return new \OC\Preview\Watcher( - $c->getAppDataDir('preview') + new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview') ); }); diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index db2bf09b6e5..cd9f6ef0399 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -22,10 +22,13 @@ namespace Test\Preview; -use OC\Files\AppData\Factory; use OC\Preview\BackgroundCleanupJob; +use OC\Preview\Storage\Root; use OC\PreviewManager; +use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IDBConnection; use Test\Traits\MountProviderTrait; use Test\Traits\UserTrait; @@ -47,9 +50,6 @@ class BackgroundCleanupJobTest extends \Test\TestCase { /** @var bool */ private $trashEnabled; - /** @var Factory */ - private $appDataFactory; - /** @var IDBConnection */ private $connection; @@ -59,6 +59,9 @@ class BackgroundCleanupJobTest extends \Test\TestCase { /** @var IRootFolder */ private $rootFolder; + /** @var IMimeTypeLoader */ + private $mimeTypeLoader; + protected function setUp(): void { parent::setUp(); @@ -76,13 +79,10 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->trashEnabled = $appManager->isEnabledForUser('files_trashbin', $this->userId); $appManager->disableApp('files_trashbin'); - $this->appDataFactory = new Factory( - \OC::$server->getRootFolder(), - \OC::$server->getSystemConfig() - ); $this->connection = \OC::$server->getDatabaseConnection(); $this->previewManager = \OC::$server->getPreviewManager(); $this->rootFolder = \OC::$server->getRootFolder(); + $this->mimeTypeLoader = \OC::$server->getMimeTypeLoader(); } protected function tearDown(): void { @@ -96,6 +96,13 @@ class BackgroundCleanupJobTest extends \Test\TestCase { parent::tearDown(); } + private function getRoot(): Root { + return new Root( + \OC::$server->getRootFolder(), + \OC::$server->getSystemConfig() + ); + } + private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); @@ -110,52 +117,89 @@ class BackgroundCleanupJobTest extends \Test\TestCase { return $files; } + private function countPreviews(Root $previewRoot, array $fileIds): int { + $i = 0; + + foreach ($fileIds as $fileId) { + try { + $previewRoot->getFolder((string)$fileId); + } catch (NotFoundException $e) { + continue; + } + + $i++; + } + + return $i; + } + public function testCleanupSystemCron() { $files = $this->setup11Previews(); + $fileIds = array_map(function (File $f) { + return $f->getId(); + }, $files); - $preview = $this->appDataFactory->get('preview'); - - $previews = $preview->getDirectoryListing(); - $this->assertCount(11, $previews); + $root = $this->getRoot(); - $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, true); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertCount(11, $previews); + $root = $this->getRoot(); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(0, $previews); + $root = $this->getRoot(); + $this->assertSame(0, $this->countPreviews($root, $fileIds)); } public function testCleanupAjax() { $files = $this->setup11Previews(); + $fileIds = array_map(function (File $f) { + return $f->getId(); + }, $files); - $preview = $this->appDataFactory->get('preview'); - - $previews = $preview->getDirectoryListing(); - $this->assertCount(11, $previews); + $root = $this->getRoot(); - $job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, false); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertCount(11, $previews); + $root = $this->getRoot(); + $this->assertSame(11, $this->countPreviews($root, $fileIds)); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(1, $previews); + $root = $this->getRoot(); + $this->assertSame(1, $this->countPreviews($root, $fileIds)); + $job->run([]); + + $root = $this->getRoot(); + $this->assertSame(0, $this->countPreviews($root, $fileIds)); + } + + public function testOldPreviews() { + $appdata = \OC::$server->getAppDataDir('preview'); + + $f1 = $appdata->newFolder('123456781'); + $f1->newFile('foo.jpg', 'foo'); + $f2 = $appdata->newFolder('123456782'); + $f2->newFile('foo.jpg', 'foo'); + + $appdata = \OC::$server->getAppDataDir('preview'); + $this->assertSame(2, count($appdata->getDirectoryListing())); + $job = new BackgroundCleanupJob($this->connection, $this->getRoot(), $this->mimeTypeLoader, true); $job->run([]); - $previews = $preview->getDirectoryListing(); - $this->assertCount(0, $previews); + $appdata = \OC::$server->getAppDataDir('preview'); + $this->assertSame(0, count($appdata->getDirectoryListing())); } } -- 2.39.5