diff options
| author | Maxence Lange <maxence@artificial-owl.com> | 2023-11-07 15:55:24 -0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-11-07 15:55:24 -0100 |
| commit | d9c24f6c7b8b9135817054e00d8b23bfae9c7da2 (patch) | |
| tree | e3a284e7f08599b0e1fd793ac704a40f182a30ca | |
| parent | 3e6642ab0b4d0a95ef4ebff8553ae93a6b9e6d00 (diff) | |
| parent | 363153626175532f471db0dd09e9421d1264765c (diff) | |
| download | nextcloud-server-d9c24f6c7b8b9135817054e00d8b23bfae9c7da2.tar.gz nextcloud-server-d9c24f6c7b8b9135817054e00d8b23bfae9c7da2.zip | |
Merge pull request #40761 from nextcloud/enh/noid/files-metadata
IFilesMetadata
44 files changed, 3723 insertions, 150 deletions
diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 709a4cd68ed..9319327d094 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -7,6 +7,7 @@ * @author Christoph Wurst <christoph@winzerhof-wurst.at> * @author Joas Schilling <coding@schilljs.com> * @author Lukas Reschke <lukas@statuscode.ch> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Michael Jobst <mjobst+github@tecratech.de> * @author Morris Jobke <hey@morrisjobke.de> * @author Robin Appelman <robin@icewind.nl> @@ -49,8 +50,8 @@ use Sabre\DAV\IFile; use Sabre\DAV\INode; use Sabre\DAV\PropFind; use Sabre\DAV\PropPatch; -use Sabre\DAV\ServerPlugin; use Sabre\DAV\Server; +use Sabre\DAV\ServerPlugin; use Sabre\DAV\Tree; use Sabre\HTTP\RequestInterface; use Sabre\HTTP\ResponseInterface; @@ -84,6 +85,7 @@ class FilesPlugin extends ServerPlugin { public const SHARE_NOTE = '{http://nextcloud.org/ns}note'; public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count'; public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count'; + public const FILE_METADATA_PREFIX = '{http://nextcloud.org/ns}metadata-'; public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size'; public const FILE_METADATA_GPS = '{http://nextcloud.org/ns}file-metadata-gps'; @@ -389,6 +391,11 @@ class FilesPlugin extends ServerPlugin { $propFind->handle(self::CREATION_TIME_PROPERTYNAME, function () use ($node) { return $node->getFileInfo()->getCreationTime(); }); + + foreach ($node->getFileInfo()->getMetadata() as $metadataKey => $metadataValue) { + $propFind->handle(self::FILE_METADATA_PREFIX . $metadataKey, $metadataValue); + } + /** * Return file/folder name as displayname. The primary reason to * implement it this way is to avoid costly fallback to diff --git a/apps/dav/lib/Files/FileSearchBackend.php b/apps/dav/lib/Files/FileSearchBackend.php index 524f90e6623..658ad34ec32 100644 --- a/apps/dav/lib/Files/FileSearchBackend.php +++ b/apps/dav/lib/Files/FileSearchBackend.php @@ -4,6 +4,7 @@ * * @author Christian <16852529+cviereck@users.noreply.github.com> * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @@ -42,6 +43,9 @@ use OCP\Files\Node; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IMetadataQuery; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; use OCP\IUser; use OCP\Share\IManager; use Sabre\DAV\Exception\NotFound; @@ -57,37 +61,14 @@ use SearchDAV\Query\Query; class FileSearchBackend implements ISearchBackend { public const OPERATOR_LIMIT = 100; - /** @var CachingTree */ - private $tree; - - /** @var IUser */ - private $user; - - /** @var IRootFolder */ - private $rootFolder; - - /** @var IManager */ - private $shareManager; - - /** @var View */ - private $view; - - /** - * FileSearchBackend constructor. - * - * @param CachingTree $tree - * @param IUser $user - * @param IRootFolder $rootFolder - * @param IManager $shareManager - * @param View $view - * @internal param IRootFolder $rootFolder - */ - public function __construct(CachingTree $tree, IUser $user, IRootFolder $rootFolder, IManager $shareManager, View $view) { - $this->tree = $tree; - $this->user = $user; - $this->rootFolder = $rootFolder; - $this->shareManager = $shareManager; - $this->view = $view; + public function __construct( + private CachingTree $tree, + private IUser $user, + private IRootFolder $rootFolder, + private IManager $shareManager, + private View $view, + private IFilesMetadataManager $filesMetadataManager, + ) { } /** @@ -115,7 +96,7 @@ class FileSearchBackend implements ISearchBackend { // all valid scopes support the same schema //todo dynamically load all propfind properties that are supported - return [ + $props = [ // queryable properties new SearchPropertyDefinition('{DAV:}displayname', true, true, true), new SearchPropertyDefinition('{DAV:}getcontenttype', true, true, true), @@ -137,6 +118,33 @@ class FileSearchBackend implements ISearchBackend { new SearchPropertyDefinition(FilesPlugin::FILE_METADATA_SIZE, true, false, false, SearchPropertyDefinition::DATATYPE_STRING), new SearchPropertyDefinition(FilesPlugin::FILEID_PROPERTYNAME, true, false, false, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER), ]; + + return array_merge($props, $this->getPropertyDefinitionsForMetadata()); + } + + + private function getPropertyDefinitionsForMetadata(): array { + $metadataProps = []; + $metadata = $this->filesMetadataManager->getKnownMetadata(); + $indexes = $metadata->getIndexes(); + foreach ($metadata->getKeys() as $key) { + $isIndex = in_array($key, $indexes); + $type = match ($metadata->getType($key)) { + IMetadataValueWrapper::TYPE_INT => SearchPropertyDefinition::DATATYPE_INTEGER, + IMetadataValueWrapper::TYPE_FLOAT => SearchPropertyDefinition::DATATYPE_DECIMAL, + IMetadataValueWrapper::TYPE_BOOL => SearchPropertyDefinition::DATATYPE_BOOLEAN, + default => SearchPropertyDefinition::DATATYPE_STRING + }; + $metadataProps[] = new SearchPropertyDefinition( + FilesPlugin::FILE_METADATA_PREFIX . $key, + true, + $isIndex, + $isIndex, + $type + ); + } + + return $metadataProps; } /** @@ -300,11 +308,20 @@ class FileSearchBackend implements ISearchBackend { /** * @param Query $query + * * @return ISearchQuery */ private function transformQuery(Query $query): ISearchQuery { + $orders = array_map(function (Order $order): ISearchOrder { + $direction = $order->order === Order::ASC ? ISearchOrder::DIRECTION_ASCENDING : ISearchOrder::DIRECTION_DESCENDING; + if (str_starts_with($order->property->name, FilesPlugin::FILE_METADATA_PREFIX)) { + return new SearchOrder($direction, substr($order->property->name, strlen(FilesPlugin::FILE_METADATA_PREFIX)), IMetadataQuery::EXTRA); + } else { + return new SearchOrder($direction, $this->mapPropertyNameToColumn($order->property)); + } + }, $query->orderBy); + $limit = $query->limit; - $orders = array_map([$this, 'mapSearchOrder'], $query->orderBy); $offset = $limit->firstResult; $limitHome = false; @@ -353,14 +370,6 @@ class FileSearchBackend implements ISearchBackend { } /** - * @param Order $order - * @return ISearchOrder - */ - private function mapSearchOrder(Order $order) { - return new SearchOrder($order->order === Order::ASC ? ISearchOrder::DIRECTION_ASCENDING : ISearchOrder::DIRECTION_DESCENDING, $this->mapPropertyNameToColumn($order->property)); - } - - /** * @param Operator $operator * @return ISearchOperator */ @@ -387,7 +396,16 @@ class FileSearchBackend implements ISearchBackend { if (!($operator->arguments[1] instanceof Literal)) { throw new \InvalidArgumentException('Invalid argument 2 for ' . $trimmedType . ' operation, expected literal'); } - return new SearchComparison($trimmedType, $this->mapPropertyNameToColumn($operator->arguments[0]), $this->castValue($operator->arguments[0], $operator->arguments[1]->value)); + + $property = $operator->arguments[0]; + $value = $this->castValue($property, $operator->arguments[1]->value); + if (str_starts_with($property->name, FilesPlugin::FILE_METADATA_PREFIX)) { + return new SearchComparison($trimmedType, substr($property->name, strlen(FilesPlugin::FILE_METADATA_PREFIX)), $value, IMetadataQuery::EXTRA); + } else { + return new SearchComparison($trimmedType, $this->mapPropertyNameToColumn($property), $value); + } + + // no break case Operator::OPERATION_IS_COLLECTION: return new SearchComparison('eq', 'mimetype', ICacheEntry::DIRECTORY_MIMETYPE); default: diff --git a/apps/dav/lib/Server.php b/apps/dav/lib/Server.php index 603e015fca9..15b97d028cc 100644 --- a/apps/dav/lib/Server.php +++ b/apps/dav/lib/Server.php @@ -11,6 +11,7 @@ * @author Joas Schilling <coding@schilljs.com> * @author John Molakvoæ <skjnldsv@protonmail.com> * @author Lukas Reschke <lukas@statuscode.ch> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> @@ -76,6 +77,7 @@ use OCA\DAV\Upload\ChunkingV2Plugin; use OCP\AppFramework\Http\Response; use OCP\Diagnostics\IEventLogger; use OCP\EventDispatcher\IEventDispatcher; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\ICacheFactory; use OCP\IRequest; use OCP\Profiler\IProfiler; @@ -316,7 +318,8 @@ class Server { $user, \OC::$server->getRootFolder(), \OC::$server->getShareManager(), - $view + $view, + \OCP\Server::get(IFilesMetadataManager::class) )); $this->server->addPlugin( new BulkUploadPlugin( diff --git a/apps/dav/tests/unit/Files/FileSearchBackendTest.php b/apps/dav/tests/unit/Files/FileSearchBackendTest.php index 715130d2fae..ea841140201 100644 --- a/apps/dav/tests/unit/Files/FileSearchBackendTest.php +++ b/apps/dav/tests/unit/Files/FileSearchBackendTest.php @@ -41,6 +41,7 @@ use OCP\Files\IRootFolder; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IUser; use OCP\Share\IManager; use SearchDAV\Backend\SearchPropertyDefinition; @@ -114,7 +115,9 @@ class FileSearchBackendTest extends TestCase { ->method('get') ->willReturn($this->searchFolder); - $this->search = new FileSearchBackend($this->tree, $this->user, $this->rootFolder, $this->shareManager, $this->view); + $filesMetadataManager = $this->createMock(IFilesMetadataManager::class); + + $this->search = new FileSearchBackend($this->tree, $this->user, $this->rootFolder, $this->shareManager, $this->view, $filesMetadataManager); } public function testSearchFilename(): void { diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index f5ac3627196..b1fc25bfe9b 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -11,6 +11,7 @@ * @author Joel S <joel.devbox@protonmail.com> * @author Jörn Friedrich Dreyer <jfd@butonic.de> * @author martin.mattel@diemattels.at <martin.mattel@diemattels.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @author Thomas Müller <thomas.mueller@tmit.eu> @@ -37,17 +38,19 @@ use OC\Core\Command\Base; use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OC\FilesMetadata\FilesMetadataManager; +use OC\ForbiddenException; +use OC\Metadata\MetadataManager; +use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\FileCacheUpdated; use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\NodeRemovedFromCache; use OCP\Files\File; -use OC\ForbiddenException; -use OC\Metadata\MetadataManager; -use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\StorageNotAvailableException; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IUserManager; use Psr\Log\LoggerInterface; use Symfony\Component\Console\Helper\Table; @@ -69,6 +72,7 @@ class Scan extends Base { private IUserManager $userManager, private IRootFolder $rootFolder, private MetadataManager $metadataManager, + private FilesMetadataManager $filesMetadataManager, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, ) { @@ -140,6 +144,11 @@ class Scan extends Base { if ($node instanceof File) { $this->metadataManager->generateMetadata($node, false); } + + $this->filesMetadataManager->refreshMetadata( + $node, + IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND + ); } }); diff --git a/apps/files_trashbin/lib/Trash/TrashItem.php b/apps/files_trashbin/lib/Trash/TrashItem.php index 3bfc905d3a1..99b2d3a1a2c 100644 --- a/apps/files_trashbin/lib/Trash/TrashItem.php +++ b/apps/files_trashbin/lib/Trash/TrashItem.php @@ -2,6 +2,7 @@ /** * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -23,6 +24,7 @@ namespace OCA\Files_Trashbin\Trash; use OCP\Files\FileInfo; +use OCP\FilesMetadata\Model\IFilesMetadata; use OCP\IUser; class TrashItem implements ITrashItem { @@ -190,4 +192,12 @@ class TrashItem implements ITrashItem { public function getParentId(): int { return $this->fileInfo->getParentId(); } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } } diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php new file mode 100644 index 00000000000..99bc167f71d --- /dev/null +++ b/core/Command/FilesMetadata/Get.php @@ -0,0 +1,119 @@ +<?php + +declare(strict_types=1); + +/** + * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Core\Command\FilesMetadata; + +use OC\User\NoUserException; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\IFilesMetadataManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Get extends Command { + public function __construct( + private IRootFolder $rootFolder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct(); + } + + protected function configure(): void { + $this->setName('metadata:get') + ->setDescription('get stored metadata about a file, by its id') + ->addArgument( + 'fileId', + InputArgument::REQUIRED, + 'id of the file document' + ) + ->addArgument( + 'userId', + InputArgument::OPTIONAL, + 'file owner' + ) + ->addOption( + 'as-array', + '', + InputOption::VALUE_NONE, + 'display metadata as a simple key=>value array' + ) + ->addOption( + 'refresh', + '', + InputOption::VALUE_NONE, + 'refresh metadata' + ) + ->addOption( + 'reset', + '', + InputOption::VALUE_NONE, + 'refresh metadata from scratch' + ); + } + + /** + * @throws NotPermittedException + * @throws FilesMetadataNotFoundException + * @throws NoUserException + * @throws NotFoundException + */ + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileId = (int)$input->getArgument('fileId'); + + if ($input->getOption('reset')) { + $this->filesMetadataManager->deleteMetadata($fileId); + if (!$input->getOption('refresh')) { + return self::SUCCESS; + } + } + + if ($input->getOption('refresh')) { + $node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId); + if (count($node) === 0) { + throw new NotFoundException(); + } + $metadata = $this->filesMetadataManager->refreshMetadata( + $node[0], + IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND + ); + } else { + $metadata = $this->filesMetadataManager->getMetadata($fileId); + } + + if ($input->getOption('as-array')) { + $output->writeln(json_encode($metadata->asArray(), JSON_PRETTY_PRINT)); + } else { + $output->writeln(json_encode($metadata, JSON_PRETTY_PRINT)); + } + + return self::SUCCESS; + } +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php new file mode 100644 index 00000000000..3266bf3a440 --- /dev/null +++ b/core/Migrations/Version28000Date20231004103301.php @@ -0,0 +1,83 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version28000Date20231004103301 extends SimpleMigrationStep { + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $updated = false; + + if (!$schema->hasTable('files_metadata')) { + $table = $schema->createTable('files_metadata'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]); + $table->addColumn('json', Types::TEXT); + $table->addColumn('sync_token', Types::STRING, ['length' => 15]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id'], 'files_meta_fileid'); + $updated = true; + } + + if (!$schema->hasTable('files_metadata_index')) { + $table = $schema->createTable('files_metadata_index'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]); + $table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]); + $table->addColumn('meta_value_string', Types::STRING, ['notnull' => false, 'length' => 63]); + $table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['file_id', 'meta_key', 'meta_value_string'], 'f_meta_index'); + $table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i'); + $updated = true; + } + + if (!$updated) { + return null; + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index 975dcf6c05f..27863cfdc8d 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -19,6 +19,7 @@ declare(strict_types=1); * @author Jörn Friedrich Dreyer <jfd@butonic.de> * @author Julius Härtl <jus@bitgrid.net> * @author Lukas Reschke <lukas@statuscode.ch> + * @author Maxence Lange <maxence@artificial-owl.com> * @author michag86 <micha_g@arcor.de> * @author Morris Jobke <hey@morrisjobke.de> * @author Patrik Kernstock <info@pkern.at> @@ -215,6 +216,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\SetupChecks::class)); + $application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class)); } else { $application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class)); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index f6f3c6a3f29..aaba8312c44 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -287,6 +287,17 @@ return array( 'OCP\\Federation\\ICloudId' => $baseDir . '/lib/public/Federation/ICloudId.php', 'OCP\\Federation\\ICloudIdManager' => $baseDir . '/lib/public/Federation/ICloudIdManager.php', 'OCP\\Files' => $baseDir . '/lib/public/Files.php', + 'OCP\\FilesMetadata\\AMetadataEvent' => $baseDir . '/lib/public/FilesMetadata/AMetadataEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataBackgroundEvent' => $baseDir . '/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataLiveEvent' => $baseDir . '/lib/public/FilesMetadata/Event/MetadataLiveEvent.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataKeyFormatException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataKeyFormatException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataNotFoundException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataTypeException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php', + 'OCP\\FilesMetadata\\IFilesMetadataManager' => $baseDir . '/lib/public/FilesMetadata/IFilesMetadataManager.php', + 'OCP\\FilesMetadata\\Model\\IFilesMetadata' => $baseDir . '/lib/public/FilesMetadata/Model/IFilesMetadata.php', + 'OCP\\FilesMetadata\\Model\\IMetadataQuery' => $baseDir . '/lib/public/FilesMetadata/Model/IMetadataQuery.php', + 'OCP\\FilesMetadata\\Model\\IMetadataValueWrapper' => $baseDir . '/lib/public/FilesMetadata/Model/IMetadataValueWrapper.php', 'OCP\\Files\\AlreadyExistsException' => $baseDir . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => $baseDir . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => $baseDir . '/lib/public/Files/Cache/AbstractCacheEvent.php', @@ -1025,6 +1036,7 @@ return array( 'OC\\Core\\Command\\Encryption\\SetDefaultModule' => $baseDir . '/core/Command/Encryption/SetDefaultModule.php', 'OC\\Core\\Command\\Encryption\\ShowKeyStorageRoot' => $baseDir . '/core/Command/Encryption/ShowKeyStorageRoot.php', 'OC\\Core\\Command\\Encryption\\Status' => $baseDir . '/core/Command/Encryption/Status.php', + 'OC\\Core\\Command\\FilesMetadata\\Get' => $baseDir . '/core/Command/FilesMetadata/Get.php', 'OC\\Core\\Command\\Group\\Add' => $baseDir . '/core/Command/Group/Add.php', 'OC\\Core\\Command\\Group\\AddUser' => $baseDir . '/core/Command/Group/AddUser.php', 'OC\\Core\\Command\\Group\\Delete' => $baseDir . '/core/Command/Group/Delete.php', @@ -1198,6 +1210,7 @@ return array( 'OC\\Core\\Migrations\\Version28000Date20230728104802' => $baseDir . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => $baseDir . '/core/Migrations/Version28000Date20230803221055.php', 'OC\\Core\\Migrations\\Version28000Date20230906104802' => $baseDir . '/core/Migrations/Version28000Date20230906104802.php', + 'OC\\Core\\Migrations\\Version28000Date20231004103301' => $baseDir . '/core/Migrations/Version28000Date20231004103301.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php', @@ -1282,6 +1295,15 @@ return array( 'OC\\Federation\\CloudFederationShare' => $baseDir . '/lib/private/Federation/CloudFederationShare.php', 'OC\\Federation\\CloudId' => $baseDir . '/lib/private/Federation/CloudId.php', 'OC\\Federation\\CloudIdManager' => $baseDir . '/lib/private/Federation/CloudIdManager.php', + 'OC\\FilesMetadata\\FilesMetadataManager' => $baseDir . '/lib/private/FilesMetadata/FilesMetadataManager.php', + 'OC\\FilesMetadata\\Job\\UpdateSingleMetadata' => $baseDir . '/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php', + 'OC\\FilesMetadata\\Listener\\MetadataDelete' => $baseDir . '/lib/private/FilesMetadata/Listener/MetadataDelete.php', + 'OC\\FilesMetadata\\Listener\\MetadataUpdate' => $baseDir . '/lib/private/FilesMetadata/Listener/MetadataUpdate.php', + 'OC\\FilesMetadata\\Model\\FilesMetadata' => $baseDir . '/lib/private/FilesMetadata/Model/FilesMetadata.php', + 'OC\\FilesMetadata\\Model\\MetadataQuery' => $baseDir . '/lib/private/FilesMetadata/Model/MetadataQuery.php', + 'OC\\FilesMetadata\\Model\\MetadataValueWrapper' => $baseDir . '/lib/private/FilesMetadata/Model/MetadataValueWrapper.php', + 'OC\\FilesMetadata\\Service\\IndexRequestService' => $baseDir . '/lib/private/FilesMetadata/Service/IndexRequestService.php', + 'OC\\FilesMetadata\\Service\\MetadataRequestService' => $baseDir . '/lib/private/FilesMetadata/Service/MetadataRequestService.php', 'OC\\Files\\AppData\\AppData' => $baseDir . '/lib/private/Files/AppData/AppData.php', 'OC\\Files\\AppData\\Factory' => $baseDir . '/lib/private/Files/AppData/Factory.php', 'OC\\Files\\Cache\\Cache' => $baseDir . '/lib/private/Files/Cache/Cache.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index f632db0e253..6c341797b79 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -320,6 +320,17 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Federation\\ICloudId' => __DIR__ . '/../../..' . '/lib/public/Federation/ICloudId.php', 'OCP\\Federation\\ICloudIdManager' => __DIR__ . '/../../..' . '/lib/public/Federation/ICloudIdManager.php', 'OCP\\Files' => __DIR__ . '/../../..' . '/lib/public/Files.php', + 'OCP\\FilesMetadata\\AMetadataEvent' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/AMetadataEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataBackgroundEvent' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataLiveEvent' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Event/MetadataLiveEvent.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataKeyFormatException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataKeyFormatException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataNotFoundException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataTypeException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php', + 'OCP\\FilesMetadata\\IFilesMetadataManager' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/IFilesMetadataManager.php', + 'OCP\\FilesMetadata\\Model\\IFilesMetadata' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Model/IFilesMetadata.php', + 'OCP\\FilesMetadata\\Model\\IMetadataQuery' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Model/IMetadataQuery.php', + 'OCP\\FilesMetadata\\Model\\IMetadataValueWrapper' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Model/IMetadataValueWrapper.php', 'OCP\\Files\\AlreadyExistsException' => __DIR__ . '/../../..' . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => __DIR__ . '/../../..' . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/AbstractCacheEvent.php', @@ -1058,6 +1069,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Encryption\\SetDefaultModule' => __DIR__ . '/../../..' . '/core/Command/Encryption/SetDefaultModule.php', 'OC\\Core\\Command\\Encryption\\ShowKeyStorageRoot' => __DIR__ . '/../../..' . '/core/Command/Encryption/ShowKeyStorageRoot.php', 'OC\\Core\\Command\\Encryption\\Status' => __DIR__ . '/../../..' . '/core/Command/Encryption/Status.php', + 'OC\\Core\\Command\\FilesMetadata\\Get' => __DIR__ . '/../../..' . '/core/Command/FilesMetadata/Get.php', 'OC\\Core\\Command\\Group\\Add' => __DIR__ . '/../../..' . '/core/Command/Group/Add.php', 'OC\\Core\\Command\\Group\\AddUser' => __DIR__ . '/../../..' . '/core/Command/Group/AddUser.php', 'OC\\Core\\Command\\Group\\Delete' => __DIR__ . '/../../..' . '/core/Command/Group/Delete.php', @@ -1231,6 +1243,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version28000Date20230728104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230803221055.php', 'OC\\Core\\Migrations\\Version28000Date20230906104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230906104802.php', + 'OC\\Core\\Migrations\\Version28000Date20231004103301' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20231004103301.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php', @@ -1315,6 +1328,15 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Federation\\CloudFederationShare' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudFederationShare.php', 'OC\\Federation\\CloudId' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudId.php', 'OC\\Federation\\CloudIdManager' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudIdManager.php', + 'OC\\FilesMetadata\\FilesMetadataManager' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/FilesMetadataManager.php', + 'OC\\FilesMetadata\\Job\\UpdateSingleMetadata' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php', + 'OC\\FilesMetadata\\Listener\\MetadataDelete' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Listener/MetadataDelete.php', + 'OC\\FilesMetadata\\Listener\\MetadataUpdate' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Listener/MetadataUpdate.php', + 'OC\\FilesMetadata\\Model\\FilesMetadata' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/FilesMetadata.php', + 'OC\\FilesMetadata\\Model\\MetadataQuery' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/MetadataQuery.php', + 'OC\\FilesMetadata\\Model\\MetadataValueWrapper' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/MetadataValueWrapper.php', + 'OC\\FilesMetadata\\Service\\IndexRequestService' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Service/IndexRequestService.php', + 'OC\\FilesMetadata\\Service\\MetadataRequestService' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Service/MetadataRequestService.php', 'OC\\Files\\AppData\\AppData' => __DIR__ . '/../../..' . '/lib/private/Files/AppData/AppData.php', 'OC\\Files\\AppData\\Factory' => __DIR__ . '/../../..' . '/lib/private/Files/AppData/Factory.php', 'OC\\Files\\Cache\\Cache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Cache.php', diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 34d2177b84e..f799d6aa72e 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -5,6 +5,7 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2019 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -35,7 +36,7 @@ use Psr\Log\LoggerInterface; * Query builder with commonly used helpers for filecache queries */ class CacheQueryBuilder extends QueryBuilder { - private $alias = null; + private ?string $alias = null; public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) { parent::__construct($connection, $systemConfig, $logger); diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 15c089a0f11..ca54133a243 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @author Tobias Kaminsky <tobias@kaminsky.me> @@ -37,41 +38,24 @@ use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IMetadataQuery; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUser; use Psr\Log\LoggerInterface; class QuerySearchHelper { - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - /** @var IDBConnection */ - private $connection; - /** @var SystemConfig */ - private $systemConfig; - private LoggerInterface $logger; - /** @var SearchBuilder */ - private $searchBuilder; - /** @var QueryOptimizer */ - private $queryOptimizer; - private IGroupManager $groupManager; - public function __construct( - IMimeTypeLoader $mimetypeLoader, - IDBConnection $connection, - SystemConfig $systemConfig, - LoggerInterface $logger, - SearchBuilder $searchBuilder, - QueryOptimizer $queryOptimizer, - IGroupManager $groupManager, + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; - $this->connection = $connection; - $this->systemConfig = $systemConfig; - $this->logger = $logger; - $this->searchBuilder = $searchBuilder; - $this->queryOptimizer = $queryOptimizer; - $this->groupManager = $groupManager; } protected function getQueryBuilder() { @@ -82,7 +66,12 @@ class QuerySearchHelper { ); } - protected function applySearchConstraints(CacheQueryBuilder $query, ISearchQuery $searchQuery, array $caches): void { + protected function applySearchConstraints( + CacheQueryBuilder $query, + ISearchQuery $searchQuery, + array $caches, + ?IMetadataQuery $metadataQuery = null + ): void { $storageFilters = array_values(array_map(function (ICache $cache) { return $cache->getQueryFilterForStorage(); }, $caches)); @@ -90,12 +79,12 @@ class QuerySearchHelper { $filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]); $this->queryOptimizer->processOperator($filter); - $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter); + $searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery); if ($searchExpr) { $query->andWhere($searchExpr); } - $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder()); + $this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery); if ($searchQuery->getLimit()) { $query->setMaxResults($searchQuery->getLimit()); @@ -144,6 +133,20 @@ class QuerySearchHelper { )); } + + /** + * left join metadata and its indexes to the filecache table + * + * @param CacheQueryBuilder $query + * + * @return IMetadataQuery + */ + protected function equipQueryForMetadata(CacheQueryBuilder $query): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'file', 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } + /** * Perform a file system search in multiple caches * @@ -175,6 +178,7 @@ class QuerySearchHelper { $query = $builder->selectFileCache('file', false); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + if (in_array('systemtag', $requestedFields)) { $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } @@ -182,12 +186,14 @@ class QuerySearchHelper { $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } - $this->applySearchConstraints($query, $searchQuery, $caches); + $metadataQuery = $this->equipQueryForMetadata($query); + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); $result = $query->execute(); $files = $result->fetchAll(); - $rawEntries = array_map(function (array $data) { + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return Cache::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php index b9a70bbd39b..1f9a6af931b 100644 --- a/lib/private/Files/Cache/SearchBuilder.php +++ b/lib/private/Files/Cache/SearchBuilder.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> * @author Tobias Kaminsky <tobias@kaminsky.me> @@ -32,6 +33,7 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\Model\IMetadataQuery; /** * Tools for transforming search queries into database queries @@ -76,7 +78,7 @@ class SearchBuilder { return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) { return array_unique(array_merge($fields, $this->extractRequestedFields($operator))); }, []); - } elseif ($operator instanceof ISearchComparison) { + } elseif ($operator instanceof ISearchComparison && !$operator->getExtra()) { return [$operator->getField()]; } return []; @@ -86,13 +88,21 @@ class SearchBuilder { * @param IQueryBuilder $builder * @param ISearchOperator[] $operators */ - public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) { - return array_filter(array_map(function ($operator) use ($builder) { - return $this->searchOperatorToDBExpr($builder, $operator); + public function searchOperatorArrayToDBExprArray( + IQueryBuilder $builder, + array $operators, + ?IMetadataQuery $metadataQuery = null + ) { + return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) { + return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery); }, $operators)); } - public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) { + public function searchOperatorToDBExpr( + IQueryBuilder $builder, + ISearchOperator $operator, + ?IMetadataQuery $metadataQuery = null + ) { $expr = $builder->expr(); if ($operator instanceof ISearchBinaryOperator) { @@ -104,29 +114,37 @@ class SearchBuilder { case ISearchBinaryOperator::OPERATOR_NOT: $negativeOperator = $operator->getArguments()[0]; if ($negativeOperator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap); + return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery); } else { throw new \InvalidArgumentException('Binary operators inside "not" is not supported'); } // no break case ISearchBinaryOperator::OPERATOR_AND: - return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); case ISearchBinaryOperator::OPERATOR_OR: - return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments())); + return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery)); default: throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType()); } } elseif ($operator instanceof ISearchComparison) { - return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap); + return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery); } else { throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator)); } } - private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) { - $this->validateComparison($comparison); + private function searchComparisonToDBExpr( + IQueryBuilder $builder, + ISearchComparison $comparison, + array $operatorMap, + ?IMetadataQuery $metadataQuery = null + ) { + if ($comparison->getExtra()) { + [$field, $value, $type] = $this->getExtraOperatorField($comparison, $metadataQuery); + } else { + [$field, $value, $type] = $this->getOperatorFieldAndValue($comparison); + } - [$field, $value, $type] = $this->getOperatorFieldAndValue($comparison); if (isset($operatorMap[$type])) { $queryOperator = $operatorMap[$type]; return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value)); @@ -136,9 +154,12 @@ class SearchBuilder { } private function getOperatorFieldAndValue(ISearchComparison $operator) { + $this->validateComparison($operator); + $field = $operator->getField(); $value = $operator->getValue(); $type = $operator->getType(); + if ($field === 'mimetype') { $value = (string)$value; if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) { @@ -213,6 +234,24 @@ class SearchBuilder { } } + + private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array { + $field = $operator->getField(); + $value = $operator->getValue(); + $type = $operator->getType(); + + switch($operator->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($field); + break; + default: + throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra()); + } + + return [$field, $value, $type]; + } + private function getParameterForValue(IQueryBuilder $builder, $value) { if ($value instanceof \DateTime) { $value = $value->getTimestamp(); @@ -228,24 +267,32 @@ class SearchBuilder { /** * @param IQueryBuilder $query * @param ISearchOrder[] $orders + * @param IMetadataQuery|null $metadataQuery */ - public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) { + public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void { foreach ($orders as $order) { $field = $order->getField(); - if ($field === 'fileid') { - $field = 'file.fileid'; - } + switch ($order->getExtra()) { + case IMetadataQuery::EXTRA: + $metadataQuery->joinIndex($field); // join index table if not joined yet + $field = $metadataQuery->getMetadataValueField($order->getField()); + break; - // Mysql really likes to pick an index for sorting if it can't fully satisfy the where - // filter with an index, since search queries pretty much never are fully filtered by index - // mysql often picks an index for sorting instead of the much more useful index for filtering. - // - // By changing the order by to an expression, mysql isn't smart enough to see that it could still - // use the index, so it instead picks an index for the filtering - if ($field === 'mtime') { - $field = $query->func()->add($field, $query->createNamedParameter(0)); - } + default: + if ($field === 'fileid') { + $field = 'file.fileid'; + } + // Mysql really likes to pick an index for sorting if it can't fully satisfy the where + // filter with an index, since search queries pretty much never are fully filtered by index + // mysql often picks an index for sorting instead of the much more useful index for filtering. + // + // By changing the order by to an expression, mysql isn't smart enough to see that it could still + // use the index, so it instead picks an index for the filtering + if ($field === 'mtime') { + $field = $query->func()->add($field, $query->createNamedParameter(0)); + } + } $query->addOrderBy($field, $order->getDirection()); } } diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 7800074460b..da0ae7e8d89 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -6,6 +6,7 @@ * @author Joas Schilling <coding@schilljs.com> * @author Julius Härtl <jus@bitgrid.net> * @author Lukas Reschke <lukas@statuscode.ch> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Piotr M <mrow4a@yahoo.com> * @author Robin Appelman <robin@icewind.nl> @@ -416,4 +417,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { public function getParentId(): int { return $this->data['parent'] ?? -1; } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? []; + } } diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php index f13cdc0c4f9..393b3bbeb06 100644 --- a/lib/private/Files/Node/LazyFolder.php +++ b/lib/private/Files/Node/LazyFolder.php @@ -5,6 +5,7 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2020 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -574,4 +575,12 @@ class LazyFolder implements Folder { } return $this->__call(__FUNCTION__, func_get_args()); } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); + } } diff --git a/lib/private/Files/Node/Node.php b/lib/private/Files/Node/Node.php index 385d45f1e3e..1fdedd844ae 100644 --- a/lib/private/Files/Node/Node.php +++ b/lib/private/Files/Node/Node.php @@ -7,6 +7,7 @@ * @author Christoph Wurst <christoph@winzerhof-wurst.at> * @author Joas Schilling <coding@schilljs.com> * @author Julius Härtl <jus@bitgrid.net> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> @@ -43,7 +44,7 @@ use OCP\Files\NotPermittedException; use OCP\Lock\LockedException; use OCP\PreConditionNotMetException; -// FIXME: this class really should be abstract +// FIXME: this class really should be abstract (+1) class Node implements INode { /** * @var \OC\Files\View $view @@ -490,4 +491,12 @@ class Node implements INode { public function getParentId(): int { return $this->fileInfo->getParentId(); } + + /** + * @inheritDoc + * @return array<string, int|string|bool|float|string[]|int[]> + */ + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } } diff --git a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php index 0caa9b12a02..664402f1238 100644 --- a/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php +++ b/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php @@ -4,6 +4,9 @@ declare(strict_types=1); /** * @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> + * @author Robin Appelman <robin@icewind.nl> + * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -48,7 +51,7 @@ class PathPrefixOptimizer extends QueryOptimizerStep { } public function processOperator(ISearchOperator &$operator) { - if (!$this->useHashEq && $operator instanceof ISearchComparison && $operator->getField() === 'path' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { + if (!$this->useHashEq && $operator instanceof ISearchComparison && !$operator->getExtra() && $operator->getField() === 'path' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) { $operator->setQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, false); } @@ -69,7 +72,7 @@ class PathPrefixOptimizer extends QueryOptimizerStep { private function operatorPairIsPathPrefix(ISearchOperator $like, ISearchOperator $equal): bool { return ( $like instanceof ISearchComparison && $equal instanceof ISearchComparison && - $like->getField() === 'path' && $equal->getField() === 'path' && + !$like->getExtra() && !$equal->getExtra() && $like->getField() === 'path' && $equal->getField() === 'path' && $like->getType() === ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE && $equal->getType() === ISearchComparison::COMPARE_EQUAL && $like->getValue() === SearchComparison::escapeLikeParameter($equal->getValue()) . '/%' ); diff --git a/lib/private/Files/Search/SearchComparison.php b/lib/private/Files/Search/SearchComparison.php index 122a1f730b4..4f9ac266397 100644 --- a/lib/private/Files/Search/SearchComparison.php +++ b/lib/private/Files/Search/SearchComparison.php @@ -1,7 +1,10 @@ <?php + +declare(strict_types=1); /** * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -25,48 +28,45 @@ namespace OC\Files\Search; use OCP\Files\Search\ISearchComparison; class SearchComparison implements ISearchComparison { - /** @var string */ - private $type; - /** @var string */ - private $field; - /** @var string|integer|\DateTime */ - private $value; - private $hints = []; + private array $hints = []; - /** - * SearchComparison constructor. - * - * @param string $type - * @param string $field - * @param \DateTime|int|string $value - */ - public function __construct($type, $field, $value) { - $this->type = $type; - $this->field = $field; - $this->value = $value; + public function __construct( + private string $type, + private string $field, + private \DateTime|int|string $value, + private string $extra = '' + ) { } /** * @return string */ - public function getType() { + public function getType(): string { return $this->type; } /** * @return string */ - public function getField() { + public function getField(): string { return $this->field; } /** * @return \DateTime|int|string */ - public function getValue() { + public function getValue(): string|int|\DateTime { return $this->value; } + /** + * @return string + * @since 28.0.0 + */ + public function getExtra(): string { + return $this->extra; + } + public function getQueryHint(string $name, $default) { return $this->hints[$name] ?? $default; } diff --git a/lib/private/Files/Search/SearchOrder.php b/lib/private/Files/Search/SearchOrder.php index 1395a87ac72..de514262bf5 100644 --- a/lib/private/Files/Search/SearchOrder.php +++ b/lib/private/Files/Search/SearchOrder.php @@ -2,6 +2,7 @@ /** * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -26,34 +27,33 @@ use OCP\Files\FileInfo; use OCP\Files\Search\ISearchOrder; class SearchOrder implements ISearchOrder { - /** @var string */ - private $direction; - /** @var string */ - private $field; + public function __construct( + private string $direction, + private string $field, + private string $extra = '' + ) { + } /** - * SearchOrder constructor. - * - * @param string $direction - * @param string $field + * @return string */ - public function __construct($direction, $field) { - $this->direction = $direction; - $this->field = $field; + public function getDirection(): string { + return $this->direction; } /** * @return string */ - public function getDirection() { - return $this->direction; + public function getField(): string { + return $this->field; } /** * @return string + * @since 28.0.0 */ - public function getField() { - return $this->field; + public function getExtra(): string { + return $this->extra; } public function sortFileInfo(FileInfo $a, FileInfo $b): int { diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php new file mode 100644 index 00000000000..54310f934d7 --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -0,0 +1,281 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata; + +use JsonException; +use OC\FilesMetadata\Job\UpdateSingleMetadata; +use OC\FilesMetadata\Listener\MetadataDelete; +use OC\FilesMetadata\Listener\MetadataUpdate; +use OC\FilesMetadata\Model\FilesMetadata; +use OC\FilesMetadata\Model\MetadataQuery; +use OC\FilesMetadata\Service\IndexRequestService; +use OC\FilesMetadata\Service\MetadataRequestService; +use OCP\BackgroundJob\IJobList; +use OCP\DB\Exception; +use OCP\DB\Exception as DBException; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\Files\InvalidPathException; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\FilesMetadata\Event\MetadataBackgroundEvent; +use OCP\FilesMetadata\Event\MetadataLiveEvent; +use OCP\FilesMetadata\Exceptions\FilesMetadataException; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\FilesMetadata\Model\IMetadataQuery; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; +use OCP\IConfig; +use Psr\Log\LoggerInterface; + +/** + * @inheritDoc + * @since 28.0.0 + */ +class FilesMetadataManager implements IFilesMetadataManager { + public const CONFIG_KEY = 'files_metadata'; + private const JSON_MAXSIZE = 100000; + + private ?IFilesMetadata $all = null; + + public function __construct( + private IEventDispatcher $eventDispatcher, + private IJobList $jobList, + private IConfig $config, + private LoggerInterface $logger, + private MetadataRequestService $metadataRequestService, + private IndexRequestService $indexRequestService, + ) { + } + + /** + * @inheritDoc + * + * @param Node $node related node + * @param int $process type of process + * + * @return IFilesMetadata + * @throws FilesMetadataException if metadata are invalid + * @throws InvalidPathException if path to file is not valid + * @throws NotFoundException if file cannot be found + * @see self::PROCESS_BACKGROUND + * @see self::PROCESS_LIVE + * @since 28.0.0 + */ + public function refreshMetadata( + Node $node, + int $process = self::PROCESS_LIVE + ): IFilesMetadata { + try { + $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); + } catch (FilesMetadataNotFoundException) { + $metadata = new FilesMetadata($node->getId()); + } + + // if $process is LIVE, we enforce LIVE + if ((self::PROCESS_LIVE & $process) !== 0) { + $event = new MetadataLiveEvent($node, $metadata); + } else { + $event = new MetadataBackgroundEvent($node, $metadata); + } + + $this->eventDispatcher->dispatchTyped($event); + $this->saveMetadata($event->getMetadata()); + + // if requested, we add a new job for next cron to refresh metadata out of main thread + // if $process was set to LIVE+BACKGROUND, we run background process directly + if ($event instanceof MetadataLiveEvent && $event->isRunAsBackgroundJobRequested()) { + if ((self::PROCESS_BACKGROUND & $process) !== 0) { + return $this->refreshMetadata($node, self::PROCESS_BACKGROUND); + } + + $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]); + } + + return $metadata; + } + + /** + * @param int $fileId file id + * + * @inheritDoc + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException if not found + * @since 28.0.0 + */ + public function getMetadata(int $fileId): IFilesMetadata { + return $this->metadataRequestService->getMetadataFromFileId($fileId); + } + + /** + * @param IFilesMetadata $filesMetadata metadata + * + * @inheritDoc + * @throws FilesMetadataException if metadata seems malformed + * @since 28.0.0 + */ + public function saveMetadata(IFilesMetadata $filesMetadata): void { + if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) { + return; + } + + $json = json_encode($filesMetadata->jsonSerialize()); + if (strlen($json) > self::JSON_MAXSIZE) { + throw new FilesMetadataException('json cannot exceed ' . self::JSON_MAXSIZE . ' characters long'); + } + + try { + if ($filesMetadata->getSyncToken() === '') { + $this->metadataRequestService->store($filesMetadata); + } else { + $this->metadataRequestService->updateMetadata($filesMetadata); + } + } catch (DBException $e) { + // most of the logged exception are the result of race condition + // between 2 simultaneous process trying to create/update metadata + $this->logger->warning('issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata]); + + return; + } + + // update indexes + foreach ($filesMetadata->getIndexes() as $index) { + try { + $this->indexRequestService->updateIndex($filesMetadata, $index); + } catch (DBException $e) { + $this->logger->warning('issue while updateIndex', ['exception' => $e]); + } + } + + // update metadata types list + $current = $this->getKnownMetadata(); + $current->import($filesMetadata->jsonSerialize(true)); + $this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current)); + } + + /** + * @param int $fileId file id + * + * @inheritDoc + * @since 28.0.0 + */ + public function deleteMetadata(int $fileId): void { + try { + $this->metadataRequestService->dropMetadata($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + + try { + $this->indexRequestService->dropIndex($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + } + + /** + * @param IQueryBuilder $qb + * @param string $fileTableAlias alias of the table that contains data about files + * @param string $fileIdField alias of the field that contains file ids + * + * @inheritDoc + * @return IMetadataQuery + * @see IMetadataQuery + * @since 28.0.0 + */ + public function getMetadataQuery( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField + ): IMetadataQuery { + return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField); + } + + /** + * @inheritDoc + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getKnownMetadata(): IFilesMetadata { + if (null !== $this->all) { + return $this->all; + } + $this->all = new FilesMetadata(); + + try { + $data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR); + $this->all->import($data); + } catch (JsonException) { + $this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata'); + } + + return $this->all; + } + + /** + * @param string $key metadata key + * @param string $type metadata type + * @param bool $indexed TRUE if metadata can be search + * + * @inheritDoc + * @since 28.0.0 + * @see IMetadataValueWrapper::TYPE_INT + * @see IMetadataValueWrapper::TYPE_FLOAT + * @see IMetadataValueWrapper::TYPE_BOOL + * @see IMetadataValueWrapper::TYPE_ARRAY + * @see IMetadataValueWrapper::TYPE_STRING_LIST + * @see IMetadataValueWrapper::TYPE_INT_LIST + * @see IMetadataValueWrapper::TYPE_STRING + */ + public function initMetadata(string $key, string $type, bool $indexed): void { + $current = $this->getKnownMetadata(); + try { + if ($current->getType($key) === $type && $indexed === $current->isIndex($key)) { + return; // if key exists, with same type and indexed, we do nothing. + } + } catch (FilesMetadataNotFoundException) { + // if value does not exist, we keep on the writing of course + } + + $current->import([$key => ['type' => $type, 'indexed' => $indexed]]); + $this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current)); + } + + /** + * load listeners + * + * @param IEventDispatcher $eventDispatcher + */ + public static function loadListeners(IEventDispatcher $eventDispatcher): void { + $eventDispatcher->addServiceListener(NodeCreatedEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeDeletedEvent::class, MetadataDelete::class); + } +} diff --git a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php new file mode 100644 index 00000000000..d18c8aa3680 --- /dev/null +++ b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php @@ -0,0 +1,67 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Job; + +use OC\FilesMetadata\FilesMetadataManager; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use OCP\Files\IRootFolder; +use OCP\FilesMetadata\Event\MetadataLiveEvent; +use OCP\FilesMetadata\IFilesMetadataManager; +use Psr\Log\LoggerInterface; + +/** + * Simple background job, created when requested by an app during the + * dispatch of MetadataLiveEvent. + * This background job will re-run the event to refresh metadata on a non-live thread. + * + * @see MetadataLiveEvent::requestBackgroundJob() + * @since 28.0.0 + */ +class UpdateSingleMetadata extends QueuedJob { + public function __construct( + ITimeFactory $time, + private IRootFolder $rootFolder, + private FilesMetadataManager $filesMetadataManager, + private LoggerInterface $logger + ) { + parent::__construct($time); + } + + protected function run($argument) { + [$userId, $fileId] = $argument; + + try { + $node = $this->rootFolder->getUserFolder($userId)->getById($fileId); + if (count($node) > 0) { + $file = array_shift($node); + $this->filesMetadataManager->refreshMetadata($file, IFilesMetadataManager::PROCESS_BACKGROUND); + } + } catch (\Exception $e) { + $this->logger->warning('issue while running UpdateSingleMetadata', ['exception' => $e, 'userId' => $userId, 'fileId' => $fileId]); + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php new file mode 100644 index 00000000000..25c944475a9 --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -0,0 +1,64 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\FilesMetadata\IFilesMetadataManager; +use Psr\Log\LoggerInterface; + +/** + * Handle file deletion event and remove stored metadata related to the deleted file + * + * @template-implements IEventListener<NodeDeletedEvent> + */ +class MetadataDelete implements IEventListener { + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + private LoggerInterface $logger + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeDeletedEvent)) { + return; + } + + try { + $nodeId = (int)$event->getNode()->getId(); + if ($nodeId > 0) { + $this->filesMetadataManager->deleteMetadata($nodeId); + } + } catch (Exception $e) { + $this->logger->warning('issue while running MetadataDelete', ['exception' => $e]); + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php new file mode 100644 index 00000000000..395a852e9e3 --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -0,0 +1,64 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\FilesMetadata\IFilesMetadataManager; +use Psr\Log\LoggerInterface; + +/** + * Handle file creation/modification events and initiate a new event related to the created/edited file. + * The generated new event is broadcast in order to obtain file related metadata from other apps. + * metadata will be stored in database. + * + * @template-implements IEventListener<NodeCreatedEvent|NodeWrittenEvent> + */ +class MetadataUpdate implements IEventListener { + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + private LoggerInterface $logger + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeCreatedEvent) && !($event instanceof NodeWrittenEvent)) { + return; + } + + try { + $this->filesMetadataManager->refreshMetadata($event->getNode()); + } catch (Exception $e) { + $this->logger->warning('issue while running MetadataUpdate', ['exception' => $e]); + } + } +} diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php new file mode 100644 index 00000000000..a94c7a9b6ff --- /dev/null +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -0,0 +1,589 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Model; + +use JsonException; +use OCP\FilesMetadata\Exceptions\FilesMetadataKeyFormatException; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; + +/** + * Model that represent metadata linked to a specific file. + * + * @inheritDoc + * @since 28.0.0 + */ +class FilesMetadata implements IFilesMetadata { + /** @var array<string, MetadataValueWrapper> */ + private array $metadata = []; + private bool $updated = false; + private int $lastUpdate = 0; + private string $syncToken = ''; + + public function __construct( + private int $fileId = 0 + ) { + } + + /** + * @inheritDoc + * @return int related file id + * @since 28.0.0 + */ + public function getFileId(): int { + return $this->fileId; + } + + /** + * @inheritDoc + * @return int timestamp + * @since 28.0.0 + */ + public function lastUpdateTimestamp(): int { + return $this->lastUpdate; + } + + /** + * @inheritDoc + * @return string token + * @since 28.0.0 + */ + public function getSyncToken(): string { + return $this->syncToken; + } + + /** + * @inheritDoc + * @return string[] list of keys + * @since 28.0.0 + */ + public function getKeys(): array { + return array_keys($this->metadata); + } + + /** + * @param string $needle metadata key to search + * + * @inheritDoc + * @return bool TRUE if key exist + * @since 28.0.0 + */ + public function hasKey(string $needle): bool { + return (in_array($needle, $this->getKeys())); + } + + /** + * @inheritDoc + * @return string[] list of indexes + * @since 28.0.0 + */ + public function getIndexes(): array { + $indexes = []; + foreach ($this->getKeys() as $key) { + if ($this->metadata[$key]->isIndexed()) { + $indexes[] = $key; + } + } + + return $indexes; + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return bool TRUE if key exists and is set as indexed + * @since 28.0.0 + */ + public function isIndex(string $key): bool { + return $this->metadata[$key]?->isIndexed() ?? false; + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return string metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function get(string $key): string { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueString(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return int metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getInt(string $key): int { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueInt(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return float metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getFloat(string $key): float { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueFloat(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return bool metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getBool(string $key): bool { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueBool(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return array metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getArray(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueArray(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return string[] metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getStringList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueStringList(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return int[] metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getIntList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueIntList(); + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return string value type + * @throws FilesMetadataNotFoundException + * @see IMetadataValueWrapper::TYPE_STRING + * @see IMetadataValueWrapper::TYPE_INT + * @see IMetadataValueWrapper::TYPE_FLOAT + * @see IMetadataValueWrapper::TYPE_BOOL + * @see IMetadataValueWrapper::TYPE_ARRAY + * @see IMetadataValueWrapper::TYPE_STRING_LIST + * @see IMetadataValueWrapper::TYPE_INT_LIST + * @since 28.0.0 + */ + public function getType(string $key): string { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getType(); + } + + /** + * @param string $key metadata key + * @param string $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function set(string $key, string $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->get($key) === $value && $index === $this->isIndex($key)) { + return $this; // we ignore if value and index have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING); + $this->updated = true; + $this->metadata[$key] = $meta->setValueString($value)->setIndexed($index); + + return $this; + } + + /** + * @param string $key metadata key + * @param int $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setInt(string $key, int $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getInt($key) === $value && $index === $this->isIndex($key)) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_INT); + $this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + /** + * @param string $key metadata key + * @param float $value metadata value + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setFloat(string $key, float $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getFloat($key) === $value && $index === $this->isIndex($key)) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_FLOAT); + $this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + + /** + * @param string $key metadata key + * @param bool $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setBool(string $key, bool $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getBool($key) === $value && $index === $this->isIndex($key)) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_BOOL); + $this->metadata[$key] = $meta->setValueBool($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + + /** + * @param string $key metadata key + * @param array $value metadata value + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setArray(string $key, array $value): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getArray($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_ARRAY); + $this->metadata[$key] = $meta->setValueArray($value); + $this->updated = true; + + return $this; + } + + /** + * @param string $key metadata key + * @param string[] $value metadata value + * @param bool $index set TRUE if each values from the list must be indexed + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setStringList(string $key, array $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getStringList($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $meta->setValueStringList($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + /** + * @param string $key metadata key + * @param int[] $value metadata value + * @param bool $index set TRUE if each values from the list must be indexed + * + * @inheritDoc + * @return self + * @throws FilesMetadataKeyFormatException + * @since 28.0.0 + */ + public function setIntList(string $key, array $value, bool $index = false): IFilesMetadata { + $this->confirmKeyFormat($key); + try { + if ($this->getIntList($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $valueWrapper = new MetadataValueWrapper(IMetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $valueWrapper->setValueIntList($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + /** + * @param string $key metadata key + * + * @inheritDoc + * @return self + * @since 28.0.0 + */ + public function unset(string $key): IFilesMetadata { + if (!array_key_exists($key, $this->metadata)) { + return $this; + } + + unset($this->metadata[$key]); + $this->updated = true; + + return $this; + } + + /** + * @param string $keyPrefix metadata key prefix + * + * @inheritDoc + * @return self + * @since 28.0.0 + */ + public function removeStartsWith(string $keyPrefix): IFilesMetadata { + if ($keyPrefix === '') { + return $this; + } + + foreach ($this->getKeys() as $key) { + if (str_starts_with($key, $keyPrefix)) { + $this->unset($key); + } + } + + return $this; + } + + /** + * @param string $key + * + * @return void + * @throws FilesMetadataKeyFormatException + */ + private function confirmKeyFormat(string $key): void { + $acceptedChars = ['-', '_']; + if (ctype_alnum(str_replace($acceptedChars, '', $key))) { + return; + } + + throw new FilesMetadataKeyFormatException('key can only contains alphanumerical characters, and dash (-)'); + } + + /** + * @inheritDoc + * @return bool TRUE if metadata have been modified + * @since 28.0.0 + */ + public function updated(): bool { + return $this->updated; + } + + public function jsonSerialize(bool $emptyValues = false): array { + $data = []; + foreach ($this->metadata as $metaKey => $metaValueWrapper) { + $data[$metaKey] = $metaValueWrapper->jsonSerialize($emptyValues); + } + + return $data; + } + + /** + * @return array<string, string|int|bool|float|string[]|int[]> + */ + public function asArray(): array { + $data = []; + foreach ($this->metadata as $metaKey => $metaValueWrapper) { + try { + $data[$metaKey] = $metaValueWrapper->getValueAny(); + } catch (FilesMetadataNotFoundException $e) { + // ignore exception + } + } + + return $data; + } + + /** + * @param array $data + * + * @inheritDoc + * @return IFilesMetadata + * @since 28.0.0 + */ + public function import(array $data): IFilesMetadata { + foreach ($data as $k => $v) { + $valueWrapper = new MetadataValueWrapper(); + $this->metadata[$k] = $valueWrapper->import($v); + } + $this->updated = false; + + return $this; + } + + /** + * import data from database to configure this model + * + * @param array $data + * @param string $prefix + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + * @since 28.0.0 + */ + public function importFromDatabase(array $data, string $prefix = ''): IFilesMetadata { + try { + $this->syncToken = $data[$prefix . 'sync_token'] ?? ''; + + return $this->import( + json_decode( + $data[$prefix . 'json'] ?? '[]', + true, + 512, + JSON_THROW_ON_ERROR + ) + ); + } catch (JsonException $e) { + throw new FilesMetadataNotFoundException(); + } + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataQuery.php b/lib/private/FilesMetadata/Model/MetadataQuery.php new file mode 100644 index 00000000000..9ff69e9516e --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataQuery.php @@ -0,0 +1,165 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Model; + +use OC\FilesMetadata\Service\IndexRequestService; +use OC\FilesMetadata\Service\MetadataRequestService; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\FilesMetadata\Model\IMetadataQuery; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; + +/** + * @inheritDoc + * @since 28.0.0 + */ +class MetadataQuery implements IMetadataQuery { + private array $knownJoinedIndex = []; + public function __construct( + private IQueryBuilder $queryBuilder, + private IFilesMetadata $knownMetadata, + private string $fileTableAlias = 'fc', + private string $fileIdField = 'fileid', + private string $alias = 'meta', + private string $aliasIndexPrefix = 'meta_index' + ) { + } + + /** + * @inheritDoc + * @see self::extractMetadata() + * @since 28.0.0 + */ + public function retrieveMetadata(): void { + $this->queryBuilder->selectAlias($this->alias . '.json', 'meta_json'); + $this->queryBuilder->leftJoin( + $this->fileTableAlias, MetadataRequestService::TABLE_METADATA, $this->alias, + $this->queryBuilder->expr()->eq($this->fileTableAlias . '.' . $this->fileIdField, $this->alias . '.file_id') + ); + } + + /** + * @param array $row result row + * + * @inheritDoc + * @return IFilesMetadata metadata + * @see self::retrieveMetadata() + * @since 28.0.0 + */ + public function extractMetadata(array $row): IFilesMetadata { + $fileId = (array_key_exists($this->fileIdField, $row)) ? $row[$this->fileIdField] : 0; + $metadata = new FilesMetadata((int)$fileId); + try { + $metadata->importFromDatabase($row, $this->alias . '_'); + } catch (FilesMetadataNotFoundException) { + // can be ignored as files' metadata are optional and might not exist in database + } + + return $metadata; + } + + /** + * @param string $metadataKey metadata key + * @param bool $enforce limit the request only to existing metadata + * + * @inheritDoc + * @since 28.0.0 + */ + public function joinIndex(string $metadataKey, bool $enforce = false): string { + if (array_key_exists($metadataKey, $this->knownJoinedIndex)) { + return $this->knownJoinedIndex[$metadataKey]; + } + + $aliasIndex = $this->aliasIndexPrefix . '_' . count($this->knownJoinedIndex); + $this->knownJoinedIndex[$metadataKey] = $aliasIndex; + + $expr = $this->queryBuilder->expr(); + $andX = $expr->andX($expr->eq($aliasIndex . '.file_id', $this->fileTableAlias . '.' . $this->fileIdField)); + $andX->add($expr->eq($this->getMetadataKeyField($metadataKey), $this->queryBuilder->createNamedParameter($metadataKey))); + + if ($enforce) { + $this->queryBuilder->rightJoin( + $this->fileTableAlias, + IndexRequestService::TABLE_METADATA_INDEX, + $aliasIndex, + $andX + ); + } else { + $this->queryBuilder->leftJoin( + $this->fileTableAlias, + IndexRequestService::TABLE_METADATA_INDEX, + $aliasIndex, + $andX + ); + } + + return $aliasIndex; + } + + /** + * @throws FilesMetadataNotFoundException + */ + public function joinedTableAlias(string $metadataKey): string { + if (!array_key_exists($metadataKey, $this->knownJoinedIndex)) { + throw new FilesMetadataNotFoundException('table related to ' . $metadataKey . ' not initiated, you need to use leftJoin() first.'); + } + + return $this->knownJoinedIndex[$metadataKey]; + } + + /** + * @inheritDoc + * + * @param string $metadataKey metadata key + * + * @return string table field + * @throws FilesMetadataNotFoundException + * @since 28.0.0 + */ + public function getMetadataKeyField(string $metadataKey): string { + return $this->joinedTableAlias($metadataKey) . '.meta_key'; + } + + /** + * @inheritDoc + * + * @param string $metadataKey metadata key + * + * @return string table field + * @throws FilesMetadataNotFoundException if metadataKey is not known + * @throws FilesMetadataTypeException is metadataKey is not set as indexed + * @since 28.0.0 + */ + public function getMetadataValueField(string $metadataKey): string { + return match ($this->knownMetadata->getType($metadataKey)) { + IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string', + IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int', + default => throw new FilesMetadataTypeException('metadata is not set as indexed'), + }; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php new file mode 100644 index 00000000000..159cd1e6fd1 --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php @@ -0,0 +1,397 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Model; + +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; + +/** + * @inheritDoc + * @see IFilesMetadata + * @since 28.0.0 + */ +class MetadataValueWrapper implements IMetadataValueWrapper { + private string $type; + /** @var string|int|float|bool|array|string[]|int[] */ + private mixed $value = null; + private bool $indexed = false; + + /** + * @param string $type value type + * + * @inheritDoc + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function __construct(string $type = '') { + $this->type = $type; + } + + /** + * @inheritDoc + * @return string value type + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function getType(): string { + return $this->type; + } + + /** + * @param string $type value type + * + * @inheritDoc + * @return bool + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function isType(string $type): bool { + return (strtolower($type) === strtolower($this->type)); + } + + /** + * @param string $type value type + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if type cannot be confirmed + * @see self::TYPE_INT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @see self::TYPE_FLOAT + * @since 28.0.0 + */ + public function assertType(string $type): self { + if (!$this->isType($type)) { + throw new FilesMetadataTypeException('type is \'' . $this->getType() . '\', expecting \'' . $type . '\''); + } + + return $this; + } + + /** + * @param string $value string to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a string + * @since 28.0.0 + */ + public function setValueString(string $value): self { + $this->assertType(self::TYPE_STRING); + $this->value = $value; + + return $this; + } + + /** + * @param int $value int to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an int + * @since 28.0.0 + */ + public function setValueInt(int $value): self { + $this->assertType(self::TYPE_INT); + $this->value = $value; + + return $this; + } + + /** + * @param float $value float to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a float + * @since 28.0.0 + */ + public function setValueFloat(float $value): self { + $this->assertType(self::TYPE_FLOAT); + $this->value = $value; + + return $this; + } + + /** + * @param bool $value bool to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a bool + * @since 28.0.0 + */ + public function setValueBool(bool $value): self { + $this->assertType(self::TYPE_BOOL); + $this->value = $value; + + + return $this; + } + + /** + * @param array $value array to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an array + * @since 28.0.0 + */ + public function setValueArray(array $value): self { + $this->assertType(self::TYPE_ARRAY); + $this->value = $value; + + return $this; + } + + /** + * @param string[] $value string list to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a string list + * @since 28.0.0 + */ + public function setValueStringList(array $value): self { + $this->assertType(self::TYPE_STRING_LIST); + // TODO confirm value is an array or string ? + $this->value = $value; + + return $this; + } + + /** + * @param int[] $value int list to be set as value + * + * @inheritDoc + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an int list + * @since 28.0.0 + */ + public function setValueIntList(array $value): self { + $this->assertType(self::TYPE_INT_LIST); + // TODO confirm value is an array of int ? + $this->value = $value; + + return $this; + } + + + /** + * @inheritDoc + * @return string set value + * @throws FilesMetadataTypeException if wrapper was not set to store a string + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueString(): string { + $this->assertType(self::TYPE_STRING); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (string)$this->value; + } + + /** + * @inheritDoc + * @return int set value + * @throws FilesMetadataTypeException if wrapper was not set to store an int + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueInt(): int { + $this->assertType(self::TYPE_INT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (int)$this->value; + } + + /** + * @inheritDoc + * @return float set value + * @throws FilesMetadataTypeException if wrapper was not set to store a float + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueFloat(): float { + $this->assertType(self::TYPE_FLOAT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (float)$this->value; + } + + /** + * @inheritDoc + * @return bool set value + * @throws FilesMetadataTypeException if wrapper was not set to store a bool + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueBool(): bool { + $this->assertType(self::TYPE_BOOL); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (bool)$this->value; + } + + /** + * @inheritDoc + * @return array set value + * @throws FilesMetadataTypeException if wrapper was not set to store an array + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueArray(): array { + $this->assertType(self::TYPE_ARRAY); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array)$this->value; + } + + /** + * @inheritDoc + * @return string[] set value + * @throws FilesMetadataTypeException if wrapper was not set to store a string list + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueStringList(): array { + $this->assertType(self::TYPE_STRING_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array)$this->value; + } + + /** + * @inheritDoc + * @return int[] set value + * @throws FilesMetadataTypeException if wrapper was not set to store an int list + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueIntList(): array { + $this->assertType(self::TYPE_INT_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array)$this->value; + } + + /** + * @inheritDoc + * @return string|int|float|bool|array|string[]|int[] set value + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueAny(): mixed { + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return $this->value; + } + + /** + * @param bool $indexed TRUE to set the stored value as an indexed value + * + * @inheritDoc + * @return self + * @since 28.0.0 + */ + public function setIndexed(bool $indexed): self { + $this->indexed = $indexed; + + return $this; + } + + /** + * @inheritDoc + * @return bool TRUE if value is an indexed value + * @since 28.0.0 + */ + public function isIndexed(): bool { + return $this->indexed; + } + + /** + * @param array $data serialized version of the object + * + * @inheritDoc + * @return self + * @see jsonSerialize + * @since 28.0.0 + */ + public function import(array $data): self { + $this->value = $data['value'] ?? null; + $this->type = $data['type'] ?? ''; + $this->setIndexed($data['indexed'] ?? false); + + return $this; + } + + public function jsonSerialize(bool $emptyValues = false): array { + return [ + 'value' => ($emptyValues) ? null : $this->value, + 'type' => $this->getType(), + 'indexed' => $this->isIndexed() + ]; + } +} diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php new file mode 100644 index 00000000000..6530dabd1c3 --- /dev/null +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -0,0 +1,195 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Service; + +use OCP\DB\Exception as DbException; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * manage sql request to the metadata_index table + */ +class IndexRequestService { + public const TABLE_METADATA_INDEX = 'files_metadata_index'; + + public function __construct( + private IDBConnection $dbConnection, + private LoggerInterface $logger + ) { + } + + /** + * update the index for a specific metadata key + * + * @param IFilesMetadata $filesMetadata metadata + * @param string $key metadata key to update + * + * @throws DbException + */ + public function updateIndex(IFilesMetadata $filesMetadata, string $key): void { + $fileId = $filesMetadata->getFileId(); + try { + $metadataType = $filesMetadata->getType($key); + } catch (FilesMetadataNotFoundException $e) { + return; + } + + /** + * might look harsh, but a lot simpler than comparing current indexed data, as we can expect + * conflict with a change of types. + * We assume that each time one random metadata were modified we can drop all index for this + * key and recreate them. + * To make it slightly cleaner, we'll use transaction + */ + $this->dbConnection->beginTransaction(); + try { + $this->dropIndex($fileId, $key); + match ($metadataType) { + IMetadataValueWrapper::TYPE_STRING => $this->insertIndexString($fileId, $key, $filesMetadata->get($key)), + IMetadataValueWrapper::TYPE_INT => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)), + IMetadataValueWrapper::TYPE_BOOL => $this->insertIndexBool($fileId, $key, $filesMetadata->getBool($key)), + IMetadataValueWrapper::TYPE_STRING_LIST => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)), + IMetadataValueWrapper::TYPE_INT_LIST => $this->insertIndexIntList($fileId, $key, $filesMetadata->getIntList($key)) + }; + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException|DbException $e) { + $this->dbConnection->rollBack(); + $this->logger->warning('issue while updateIndex', ['exception' => $e, 'fileId' => $fileId, 'key' => $key]); + } + + $this->dbConnection->commit(); + } + + /** + * insert a new entry in the metadata_index table for a string value + * + * @param int $fileId file id + * @param string $key metadata key + * @param string $value metadata value + * + * @throws DbException + */ + private function insertIndexString(int $fileId, string $key, string $value): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_string', $qb->createNamedParameter($value)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } + + /** + * insert a new entry in the metadata_index table for an int value + * + * @param int $fileId file id + * @param string $key metadata key + * @param int $value metadata value + * + * @throws DbException + */ + public function insertIndexInt(int $fileId, string $key, int $value): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } + + /** + * insert a new entry in the metadata_index table for a bool value + * + * @param int $fileId file id + * @param string $key metadata key + * @param bool $value metadata value + * + * @throws DbException + */ + public function insertIndexBool(int $fileId, string $key, bool $value): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter(($value) ? '1' : '0', IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } + + /** + * insert entries in the metadata_index table for list of string + * + * @param int $fileId file id + * @param string $key metadata key + * @param string[] $values metadata values + * + * @throws DbException + */ + public function insertIndexStringList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexString($fileId, $key, $value); + } + } + + /** + * insert entries in the metadata_index table for list of int + * + * @param int $fileId file id + * @param string $key metadata key + * @param int[] $values metadata values + * + * @throws DbException + */ + public function insertIndexIntList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexInt($fileId, $key, $value); + } + } + + /** + * drop indexes related to a file id + * if a key is specified, only drop entries related to it + * + * @param int $fileId file id + * @param string $key metadata key + * + * @throws DbException + */ + public function dropIndex(int $fileId, string $key = ''): void { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + if ($key !== '') { + $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); + } + + $qb->executeStatement(); + } +} diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php new file mode 100644 index 00000000000..85874e92d4a --- /dev/null +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -0,0 +1,160 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\FilesMetadata\Service; + +use OC\FilesMetadata\Model\FilesMetadata; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; + +/** + * manage sql request to the metadata table + */ +class MetadataRequestService { + public const TABLE_METADATA = 'files_metadata'; + + public function __construct( + private IDBConnection $dbConnection, + private LoggerInterface $logger + ) { + } + + /** + * store metadata into database + * + * @param IFilesMetadata $filesMetadata + * + * @throws Exception + */ + public function store(IFilesMetadata $filesMetadata): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken())) + ->setValue('last_update', (string) $qb->createFunction('NOW()')); + $qb->executeStatement(); + } + + /** + * returns metadata for a file id + * + * @param int $fileId file id + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException if no metadata are found in database + */ + public function getMetadataFromFileId(int $fileId): IFilesMetadata { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('json', 'sync_token')->from(self::TABLE_METADATA); + $qb->where( + $qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)) + ); + $result = $qb->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + } catch (Exception $e) { + $this->logger->warning( + 'exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId] + ); + throw new FilesMetadataNotFoundException(); + } + + if ($data === false) { + throw new FilesMetadataNotFoundException(); + } + + $metadata = new FilesMetadata($fileId); + $metadata->importFromDatabase($data); + + return $metadata; + } + + /** + * drop metadata related to a file id + * + * @param int $fileId file id + * + * @return void + * @throws Exception + */ + public function dropMetadata(int $fileId): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } + + /** + * update metadata in the database + * + * @param IFilesMetadata $filesMetadata metadata + * + * @return int number of affected rows + * @throws Exception + */ + public function updateMetadata(IFilesMetadata $filesMetadata): int { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + + $qb->update(self::TABLE_METADATA) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter($this->generateSyncToken())) + ->set('last_update', $qb->createFunction('NOW()')) + ->where( + $expr->andX( + $expr->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)), + $expr->eq('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) + ) + ); + + return $qb->executeStatement(); + } + + /** + * generate a random token + * @return string + */ + private function generateSyncToken(): string { + $chars = 'qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890'; + + $str = ''; + $max = strlen($chars); + for ($i = 0; $i < 7; $i++) { + try { + $str .= $chars[random_int(0, $max - 2)]; + } catch (\Exception $e) { + $this->logger->warning('exception during generateSyncToken', ['exception' => $e]); + } + } + + return $str; + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index 77791c49e8b..31e4a9f79e2 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -102,6 +102,7 @@ use OC\Files\Storage\StorageFactory; use OC\Files\Template\TemplateManager; use OC\Files\Type\Loader; use OC\Files\View; +use OC\FilesMetadata\FilesMetadataManager; use OC\FullTextSearch\FullTextSearchManager; use OC\Http\Client\ClientService; use OC\Http\Client\NegativeDnsCache; @@ -196,6 +197,7 @@ use OCP\Files\Lock\ILockManager; use OCP\Files\Mount\IMountManager; use OCP\Files\Storage\IStorageFactory; use OCP\Files\Template\ITemplateManager; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\GlobalScale\IConfig; use OCP\Group\ISubAdmin; @@ -1401,6 +1403,7 @@ class Server extends ServerContainer implements IServerContainer { $this->registerAlias(\OCP\Dashboard\IManager::class, \OC\Dashboard\Manager::class); $this->registerAlias(IFullTextSearchManager::class, FullTextSearchManager::class); + $this->registerAlias(IFilesMetadataManager::class, FilesMetadataManager::class); $this->registerAlias(ISubAdmin::class, SubAdmin::class); @@ -1480,6 +1483,8 @@ class Server extends ServerContainer implements IServerContainer { $eventDispatcher->addServiceListener(PostLoginEvent::class, UserLoggedInListener::class); $eventDispatcher->addServiceListener(UserChangedEvent::class, UserChangedListener::class); $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); + + FilesMetadataManager::loadListeners($eventDispatcher); } /** diff --git a/lib/public/Files/FileInfo.php b/lib/public/Files/FileInfo.php index da35f7f9028..817b03dfc65 100644 --- a/lib/public/Files/FileInfo.php +++ b/lib/public/Files/FileInfo.php @@ -6,6 +6,7 @@ * @author Felix Heidecke <felix@heidecke.me> * @author Joas Schilling <coding@schilljs.com> * @author Julius Härtl <jus@bitgrid.net> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Morris Jobke <hey@morrisjobke.de> * @author Robin Appelman <robin@icewind.nl> * @author Roeland Jago Douma <roeland@famdouma.nl> @@ -308,4 +309,12 @@ interface FileInfo { * @since 28.0.0 */ public function getParentId(): int; + + /** + * Get the metadata, if available + * + * @return array<string, int|string|bool|float|string[]|int[]> + * @since 28.0.0 + */ + public function getMetadata(): array; } diff --git a/lib/public/Files/Search/ISearchComparison.php b/lib/public/Files/Search/ISearchComparison.php index 8ebaeced304..68924d1e369 100644 --- a/lib/public/Files/Search/ISearchComparison.php +++ b/lib/public/Files/Search/ISearchComparison.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -43,7 +44,7 @@ interface ISearchComparison extends ISearchOperator { * @return string * @since 12.0.0 */ - public function getType(); + public function getType(): string; /** * Get the name of the field to compare with @@ -53,7 +54,15 @@ interface ISearchComparison extends ISearchOperator { * @return string * @since 12.0.0 */ - public function getField(); + public function getField(): string; + + /** + * extra means data are not related to the main files table + * + * @return string + * @since 28.0.0 + */ + public function getExtra(): string; /** * Get the value to compare the field with @@ -61,5 +70,5 @@ interface ISearchComparison extends ISearchOperator { * @return string|integer|\DateTime * @since 12.0.0 */ - public function getValue(); + public function getValue(): string|int|\DateTime; } diff --git a/lib/public/Files/Search/ISearchOrder.php b/lib/public/Files/Search/ISearchOrder.php index 3b9e6e6713a..5b73e7b102c 100644 --- a/lib/public/Files/Search/ISearchOrder.php +++ b/lib/public/Files/Search/ISearchOrder.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> * * @author Christoph Wurst <christoph@winzerhof-wurst.at> + * @author Maxence Lange <maxence@artificial-owl.com> * @author Robin Appelman <robin@icewind.nl> * * @license GNU AGPL version 3 or any later version @@ -38,7 +39,7 @@ interface ISearchOrder { * @return string * @since 12.0.0 */ - public function getDirection(); + public function getDirection(): string; /** * The field to sort on @@ -46,7 +47,15 @@ interface ISearchOrder { * @return string * @since 12.0.0 */ - public function getField(); + public function getField(): string; + + /** + * extra means data are not related to the main files table + * + * @return string + * @since 28.0.0 + */ + public function getExtra(): string; /** * Apply the sorting on 2 FileInfo objects diff --git a/lib/public/FilesMetadata/AMetadataEvent.php b/lib/public/FilesMetadata/AMetadataEvent.php new file mode 100644 index 00000000000..87abc60a983 --- /dev/null +++ b/lib/public/FilesMetadata/AMetadataEvent.php @@ -0,0 +1,68 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata; + +use OCP\EventDispatcher\Event; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +/** + * @since 28.0.0 + */ +abstract class AMetadataEvent extends Event { + /** + * @param Node $node + * @param IFilesMetadata $metadata + * @since 28.0.0 + */ + public function __construct( + private Node $node, + private IFilesMetadata $metadata + ) { + parent::__construct(); + } + + /** + * returns related node + * + * @return Node + * @since 28.0.0 + */ + public function getNode(): Node { + return $this->node; + } + + /** + * returns metadata. if known, it already contains data from the database. + * If the object is modified using its setters, changes are stored in database at the end of the event. + * + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getMetadata(): IFilesMetadata { + return $this->metadata; + } +} diff --git a/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php new file mode 100644 index 00000000000..3d175994c32 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php @@ -0,0 +1,40 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Event; + +use OCP\FilesMetadata\AMetadataEvent; + +/** + * MetadataBackgroundEvent is an event similar to MetadataLiveEvent but dispatched + * on a background thread instead of live thread. Meaning there is no limit to + * the time required for the generation of your metadata. + * + * @see AMetadataEvent::getMetadata() + * @see AMetadataEvent::getNode() + * @since 28.0.0 + */ +class MetadataBackgroundEvent extends AMetadataEvent { +} diff --git a/lib/public/FilesMetadata/Event/MetadataLiveEvent.php b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php new file mode 100644 index 00000000000..a89692fde92 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php @@ -0,0 +1,67 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Event; + +use OCP\FilesMetadata\AMetadataEvent; + +/** + * MetadataLiveEvent is an event initiated when a file is created or updated. + * The app contains the Node related to the created/updated file, and a FilesMetadata that already + * contains the currently known metadata. + * + * Setting new metadata, or modifying already existing metadata with different value, will trigger + * the save of the metadata in the database. + * + * @see AMetadataEvent::getMetadata() + * @see AMetadataEvent::getNode() + * @see MetadataLiveEvent::requestBackgroundJob() + * @since 28.0.0 + */ +class MetadataLiveEvent extends AMetadataEvent { + private bool $runAsBackgroundJob = false; + + /** + * For heavy process, call this method if your app prefers to update metadata on a + * background/cron job, instead of the live process. + * A similar MetadataBackgroundEvent will be broadcast on next cron tick. + * + * @return void + * @since 28.0.0 + */ + public function requestBackgroundJob(): void { + $this->runAsBackgroundJob = true; + } + + /** + * return true if any app that catch this event requested a re-run as background job + * + * @return bool + * @since 28.0.0 + */ + public function isRunAsBackgroundJobRequested(): bool { + return $this->runAsBackgroundJob; + } +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php new file mode 100644 index 00000000000..e3f75a7a7af --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php @@ -0,0 +1,34 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Exceptions; + +use Exception; + +/** + * @since 28.0.0 + */ +class FilesMetadataException extends Exception { +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataKeyFormatException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataKeyFormatException.php new file mode 100644 index 00000000000..0083e985a5e --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataKeyFormatException.php @@ -0,0 +1,32 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Exceptions; + +/** + * @since 28.0.0 + */ +class FilesMetadataKeyFormatException extends FilesMetadataException { +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php new file mode 100644 index 00000000000..9c03c5ba370 --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php @@ -0,0 +1,32 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Exceptions; + +/** + * @since 28.0.0 + */ +class FilesMetadataNotFoundException extends FilesMetadataException { +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php new file mode 100644 index 00000000000..1d134c67ecf --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php @@ -0,0 +1,32 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Exceptions; + +/** + * @since 28.0.0 + */ +class FilesMetadataTypeException extends FilesMetadataException { +} diff --git a/lib/public/FilesMetadata/IFilesMetadataManager.php b/lib/public/FilesMetadata/IFilesMetadataManager.php new file mode 100644 index 00000000000..1cd0fcb4125 --- /dev/null +++ b/lib/public/FilesMetadata/IFilesMetadataManager.php @@ -0,0 +1,139 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\Node; +use OCP\FilesMetadata\Exceptions\FilesMetadataException; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Model\IFilesMetadata; +use OCP\FilesMetadata\Model\IMetadataQuery; + +/** + * Manager for FilesMetadata; manage files' metadata. + * + * @since 28.0.0 + */ +interface IFilesMetadataManager { + /** @since 28.0.0 */ + public const PROCESS_LIVE = 1; + /** @since 28.0.0 */ + public const PROCESS_BACKGROUND = 2; + + /** + * initiate the process of refreshing the metadata in relation to a node + * usually, this process: + * - get current metadata from database, if available, or create a new one + * - dispatch a MetadataLiveEvent, + * - save new metadata in database, if metadata have been changed during the event + * - refresh metadata indexes if needed, + * - prep a new cronjob if an app request it during the event, + * + * @param Node $node related node + * @param int $process type of process + * + * @return IFilesMetadata + * @see self::PROCESS_BACKGROUND + * @see self::PROCESS_LIVE + * @since 28.0.0 + */ + public function refreshMetadata( + Node $node, + int $process = self::PROCESS_LIVE + ): IFilesMetadata; + + /** + * returns metadata from a file id + * + * @param int $fileId file id + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException if not found + * @since 28.0.0 + */ + public function getMetadata(int $fileId): IFilesMetadata; + + /** + * save metadata to database and refresh indexes. + * metadata are saved if new data are available. + * on update, a check on syncToken is done to avoid conflict (race condition) + * + * @param IFilesMetadata $filesMetadata + * + * @throws FilesMetadataException if metadata seems malformed + * @since 28.0.0 + */ + public function saveMetadata(IFilesMetadata $filesMetadata): void; + + /** + * delete metadata and its indexes + * + * @param int $fileId file id + * + * @return void + * @since 28.0.0 + */ + public function deleteMetadata(int $fileId): void; + + /** + * generate and return a MetadataQuery to help building sql queries + * + * @param IQueryBuilder $qb + * @param string $fileTableAlias alias of the table that contains data about files + * @param string $fileIdField alias of the field that contains file ids + * + * @return IMetadataQuery + * @see IMetadataQuery + * @since 28.0.0 + */ + public function getMetadataQuery( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField + ): IMetadataQuery; + + /** + * returns all type of metadata currently available. + * The list is stored in a IFilesMetadata with null values but correct type. + * + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getKnownMetadata(): IFilesMetadata; + + /** + * initiate a metadata key with its type. + * The call is mandatory before using the metadata property in a webdav request. + * It is not needed to only use this method when the app is enabled: the method can be + * called each time during the app loading as the metadata will only be initiated if not known + * + * @param string $key metadata key + * @param string $type metadata type + * @param bool $indexed TRUE if metadata can be search + * @since 28.0.0 + */ + public function initMetadata(string $key, string $type, bool $indexed): void; +} diff --git a/lib/public/FilesMetadata/Model/IFilesMetadata.php b/lib/public/FilesMetadata/Model/IFilesMetadata.php new file mode 100644 index 00000000000..ca1cc898f74 --- /dev/null +++ b/lib/public/FilesMetadata/Model/IFilesMetadata.php @@ -0,0 +1,343 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Model; + +use JsonSerializable; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; + +/** + * Model that represent metadata linked to a specific file. + * + * Example of json stored in the database + * { + * "mymeta": { + * "value": "this is a test", + * "type": "string", + * "indexed": false + * }, + * "myapp-anothermeta": { + * "value": 42, + * "type": "int", + * "indexed": true + * } + * } + * + * @see IMetadataValueWrapper + * @since 28.0.0 + */ +interface IFilesMetadata extends JsonSerializable { + /** + * returns the file id linked to this metadata + * + * @return int related file id + * @since 28.0.0 + */ + public function getFileId(): int; + + /** + * returns last time metadata were updated in the database + * + * @return int timestamp + * @since 28.0.0 + */ + public function lastUpdateTimestamp(): int; + + /** + * returns the token known at the time the metadata were extracted from database + * + * @return string token + * @since 28.0.0 + */ + public function getSyncToken(): string; + + /** + * returns all current metadata keys + * + * @return string[] list of keys + * @since 28.0.0 + */ + public function getKeys(): array; + + /** + * returns true if search metadata key exists + * + * @param string $needle metadata key to search + * + * @return bool TRUE if key exist + * @since 28.0.0 + */ + public function hasKey(string $needle): bool; + + /** + * return the list of metadata keys set as indexed + * + * @return string[] list of indexes + * @since 28.0.0 + */ + public function getIndexes(): array; + + /** + * returns true if key exists and is set as indexed + * + * @param string $key metadata key + * + * @return bool + * @since 28.0.0 + */ + public function isIndex(string $key): bool; + + /** + * returns string value for a metadata key + * + * @param string $key metadata key + * + * @return string metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function get(string $key): string; + + /** + * returns int value for a metadata key + * + * @param string $key metadata key + * + * @return int metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getInt(string $key): int; + + /** + * returns float value for a metadata key + * + * @param string $key metadata key + * + * @return float metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getFloat(string $key): float; + + /** + * returns bool value for a metadata key + * + * @param string $key metadata key + * + * @return bool metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getBool(string $key): bool; + + /** + * returns array for a metadata key + * + * @param string $key metadata key + * + * @return array metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getArray(string $key): array; + + /** + * returns string[] value for a metadata key + * + * @param string $key metadata key + * + * @return string[] metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getStringList(string $key): array; + + /** + * returns int[] value for a metadata key + * + * @param string $key metadata key + * + * @return int[] metadata value + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + * @since 28.0.0 + */ + public function getIntList(string $key): array; + + /** + * returns the value type of the metadata (string, int, ...) + * + * @param string $key metadata key + * + * @return string value type + * @throws FilesMetadataNotFoundException + * @see IMetadataValueWrapper::TYPE_STRING + * @see IMetadataValueWrapper::TYPE_INT + * @see IMetadataValueWrapper::TYPE_FLOAT + * @see IMetadataValueWrapper::TYPE_BOOL + * @see IMetadataValueWrapper::TYPE_ARRAY + * @see IMetadataValueWrapper::TYPE_STRING_LIST + * @see IMetadataValueWrapper::TYPE_INT_LIST + * @since 28.0.0 + */ + public function getType(string $key): string; + + /** + * set a metadata key/value pair for string value + * + * @param string $key metadata key + * @param string $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @return self + * @since 28.0.0 + */ + public function set(string $key, string $value, bool $index = false): self; + + /** + * set a metadata key/value pair for int value + * + * @param string $key metadata key + * @param int $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @return self + * @since 28.0.0 + */ + public function setInt(string $key, int $value, bool $index = false): self; + + /** + * set a metadata key/value pair for float value + * + * @param string $key metadata key + * @param float $value metadata value + * + * @return self + * @since 28.0.0 + */ + public function setFloat(string $key, float $value): self; + + /** + * set a metadata key/value pair for bool value + * + * @param string $key metadata key + * @param bool $value metadata value + * @param bool $index set TRUE if value must be indexed + * + * @return self + * @since 28.0.0 + */ + public function setBool(string $key, bool $value, bool $index = false): self; + + /** + * set a metadata key/value pair for array + * + * @param string $key metadata key + * @param array $value metadata value + * + * @return self + * @since 28.0.0 + */ + public function setArray(string $key, array $value): self; + + /** + * set a metadata key/value pair for list of string + * + * @param string $key metadata key + * @param string[] $value metadata value + * @param bool $index set TRUE if each values from the list must be indexed + * + * @return self + * @since 28.0.0 + */ + public function setStringList(string $key, array $value, bool $index = false): self; + + /** + * set a metadata key/value pair for list of int + * + * @param string $key metadata key + * @param int[] $value metadata value + * @param bool $index set TRUE if each values from the list must be indexed + * + * @return self + * @since 28.0.0 + */ + public function setIntList(string $key, array $value, bool $index = false): self; + + /** + * unset a metadata + * + * @param string $key metadata key + * + * @return self + * @since 28.0.0 + */ + public function unset(string $key): self; + + /** + * unset metadata with key starting with prefix + * + * @param string $keyPrefix metadata key prefix + * + * @return self + * @since 28.0.0 + */ + public function removeStartsWith(string $keyPrefix): self; + + /** + * returns true if object have been updated since last import + * + * @return bool TRUE if metadata have been modified + * @since 28.0.0 + */ + public function updated(): bool; + + /** + * returns metadata in a simple array with METADATA_KEY => METADATA_VALUE + * + * @return array metadata + * @since 28.0.0 + */ + public function asArray(): array; + + /** + * deserialize the object from a json + * + * @param array $data serialized version of the object + * + * @return self + * @see jsonSerialize + * @since 28.0.0 + */ + public function import(array $data): self; +} diff --git a/lib/public/FilesMetadata/Model/IMetadataQuery.php b/lib/public/FilesMetadata/Model/IMetadataQuery.php new file mode 100644 index 00000000000..d3f55ce6cce --- /dev/null +++ b/lib/public/FilesMetadata/Model/IMetadataQuery.php @@ -0,0 +1,90 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Model; + +/** + * Model that help building queries with metadata and metadata indexes + * + * @since 28.0.0 + */ +interface IMetadataQuery { + /** @since 28.0.0 */ + public const EXTRA = 'metadata'; + + /** + * Add metadata linked to file id to the query + * + * @see self::extractMetadata() + * @since 28.0.0 + */ + public function retrieveMetadata(): void; + + /** + * extract metadata from a result row + * + * @param array $row result row + * + * @return IFilesMetadata metadata + * @see self::retrieveMetadata() + * @since 28.0.0 + */ + public function extractMetadata(array $row): IFilesMetadata; + + /** + * join the metadata_index table, based on a metadataKey. + * This will prep the query for condition based on this specific metadataKey. + * If a link to the metadataKey already exists, returns known alias. + * + * TODO: investigate how to support a search done on multiple values for same key (AND). + * + * @param string $metadataKey metadata key + * @param bool $enforce limit the request only to existing metadata + * + * @return string generated table alias + * @since 28.0.0 + */ + public function joinIndex(string $metadataKey, bool $enforce = false): string; + + /** + * returns the name of the field for metadata key to be used in query expressions + * + * @param string $metadataKey metadata key + * + * @return string table field + * @since 28.0.0 + */ + public function getMetadataKeyField(string $metadataKey): string; + + /** + * returns the name of the field for metadata string value to be used in query expressions + * + * @param string $metadataKey metadata key + * + * @return string table field + * @since 28.0.0 + */ + public function getMetadataValueField(string $metadataKey): string; +} diff --git a/lib/public/FilesMetadata/Model/IMetadataValueWrapper.php b/lib/public/FilesMetadata/Model/IMetadataValueWrapper.php new file mode 100644 index 00000000000..577ff874330 --- /dev/null +++ b/lib/public/FilesMetadata/Model/IMetadataValueWrapper.php @@ -0,0 +1,300 @@ +<?php + +declare(strict_types=1); +/** + * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> + * + * @author Maxence Lange <maxence@artificial-owl.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\FilesMetadata\Model; + +use JsonSerializable; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; + +/** + * Model that store the value of a single metadata. + * It stores the value, its type and the index status. + * + * @see IFilesMetadata + * @since 28.0.0 + */ +interface IMetadataValueWrapper extends JsonSerializable { + /** + * @since 28.0.0 + */ + public const TYPE_STRING = 'string'; + public const TYPE_INT = 'int'; + public const TYPE_FLOAT = 'float'; + public const TYPE_BOOL = 'bool'; + public const TYPE_ARRAY = 'array'; + public const TYPE_STRING_LIST = 'string[]'; + public const TYPE_INT_LIST = 'int[]'; + + /** + * Unless a call of import() to deserialize an object is expected, a valid value type is needed here. + * + * @param string $type value type + * + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function __construct(string $type); + + /** + * returns the value type + * + * @return string value type + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function getType(): string; + + /** + * returns if the set value type is the one expected + * + * @param string $type value type + * + * @return bool + * @see self::TYPE_INT + * @see self::TYPE_FLOAT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @since 28.0.0 + */ + public function isType(string $type): bool; + + /** + * throws an exception if the type is not correctly set + * + * @param string $type value type + * + * @return self + * @throws FilesMetadataTypeException if type cannot be confirmed + * @see self::TYPE_INT + * @see self::TYPE_BOOL + * @see self::TYPE_ARRAY + * @see self::TYPE_STRING_LIST + * @see self::TYPE_INT_LIST + * @see self::TYPE_STRING + * @see self::TYPE_FLOAT + * @since 28.0.0 + */ + public function assertType(string $type): self; + + /** + * set a string value + * + * @param string $value string to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a string + * @since 28.0.0 + */ + public function setValueString(string $value): self; + + /** + * set a int value + * + * @param int $value int to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an int + * @since 28.0.0 + */ + public function setValueInt(int $value): self; + + /** + * set a float value + * + * @param float $value float to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a float + * @since 28.0.0 + */ + public function setValueFloat(float $value): self; + + /** + * set a bool value + * + * @param bool $value bool to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a bool + * @since 28.0.0 + */ + public function setValueBool(bool $value): self; + + /** + * set an array value + * + * @param array $value array to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an array + * @since 28.0.0 + */ + public function setValueArray(array $value): self; + + /** + * set a string list value + * + * @param string[] $value string list to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store a string list + * @since 28.0.0 + */ + public function setValueStringList(array $value): self; + + /** + * set an int list value + * + * @param int[] $value int list to be set as value + * + * @return self + * @throws FilesMetadataTypeException if wrapper was not set to store an int list + * @since 28.0.0 + */ + public function setValueIntList(array $value): self; + + + /** + * get stored value + * + * @return string set value + * @throws FilesMetadataTypeException if wrapper was not set to store a string + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueString(): string; + + /** + * get stored value + * + * @return int set value + * @throws FilesMetadataTypeException if wrapper was not set to store an int + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueInt(): int; + + /** + * get stored value + * + * @return float set value + * @throws FilesMetadataTypeException if wrapper was not set to store a float + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueFloat(): float; + + /** + * get stored value + * + * @return bool set value + * @throws FilesMetadataTypeException if wrapper was not set to store a bool + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueBool(): bool; + + /** + * get stored value + * + * @return array set value + * @throws FilesMetadataTypeException if wrapper was not set to store an array + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueArray(): array; + + /** + * get stored value + * + * @return string[] set value + * @throws FilesMetadataTypeException if wrapper was not set to store a string list + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueStringList(): array; + + /** + * get stored value + * + * @return int[] set value + * @throws FilesMetadataTypeException if wrapper was not set to store an int list + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueIntList(): array; + + /** + * get stored value + * + * @return string|int|float|bool|array|string[]|int[] set value + * @throws FilesMetadataNotFoundException if value is not set + * @since 28.0.0 + */ + public function getValueAny(): mixed; + + /** + * @param bool $indexed TRUE to set the stored value as an indexed value + * + * @return self + * @since 28.0.0 + */ + public function setIndexed(bool $indexed): self; + + /** + * returns if value is an indexed value + * + * @return bool TRUE if value is an indexed value + * @since 28.0.0 + */ + public function isIndexed(): bool; + + /** + * deserialize the object from a json + * + * @param array $data serialized version of the object + * + * @return self + * @see jsonSerialize + * @since 28.0.0 + */ + public function import(array $data): self; +} |