enh(metadata): migrate metadata to lazy appconfigpull/44640/head
} elseif (!$data) { | } elseif (!$data) { | ||||
return $data; | return $data; | ||||
} else { | } else { | ||||
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? []; | |||||
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); | |||||
return self::cacheEntryFromData($data, $this->mimetypeLoader); | return self::cacheEntryFromData($data, $this->mimetypeLoader); | ||||
} | } | ||||
} | } | ||||
$result->closeCursor(); | $result->closeCursor(); | ||||
return array_map(function (array $data) use ($metadataQuery) { | return array_map(function (array $data) use ($metadataQuery) { | ||||
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? []; | |||||
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); | |||||
return self::cacheEntryFromData($data, $this->mimetypeLoader); | return self::cacheEntryFromData($data, $this->mimetypeLoader); | ||||
}, $files); | }, $files); | ||||
} | } |
/** | /** | ||||
* join metadata to current query builder and returns an helper | * join metadata to current query builder and returns an helper | ||||
* | * | ||||
* @return IMetadataQuery|null NULL if no metadata have never been generated | |||||
* @return IMetadataQuery | |||||
*/ | */ | ||||
public function selectMetadata(): ?IMetadataQuery { | |||||
public function selectMetadata(): IMetadataQuery { | |||||
$metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); | $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); | ||||
$metadataQuery?->retrieveMetadata(); | |||||
$metadataQuery->retrieveMetadata(); | |||||
return $metadataQuery; | return $metadataQuery; | ||||
} | } | ||||
} | } |
$files = $result->fetchAll(); | $files = $result->fetchAll(); | ||||
$rawEntries = array_map(function (array $data) use ($metadataQuery) { | $rawEntries = array_map(function (array $data) use ($metadataQuery) { | ||||
// migrate to null safe ... | |||||
if ($metadataQuery === null) { | |||||
$data['metadata'] = []; | |||||
} else { | |||||
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); | |||||
} | |||||
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); | |||||
return Cache::cacheEntryFromData($data, $this->mimetypeLoader); | return Cache::cacheEntryFromData($data, $this->mimetypeLoader); | ||||
}, $files); | }, $files); | ||||
use OCP\FilesMetadata\IMetadataQuery; | use OCP\FilesMetadata\IMetadataQuery; | ||||
use OCP\FilesMetadata\Model\IFilesMetadata; | use OCP\FilesMetadata\Model\IFilesMetadata; | ||||
use OCP\FilesMetadata\Model\IMetadataValueWrapper; | use OCP\FilesMetadata\Model\IMetadataValueWrapper; | ||||
use OCP\IConfig; | |||||
use OCP\IDBConnection; | |||||
use OCP\IAppConfig; | |||||
use Psr\Log\LoggerInterface; | use Psr\Log\LoggerInterface; | ||||
/** | /** | ||||
public function __construct( | public function __construct( | ||||
private IEventDispatcher $eventDispatcher, | private IEventDispatcher $eventDispatcher, | ||||
private IJobList $jobList, | private IJobList $jobList, | ||||
private IConfig $config, | |||||
private IAppConfig $appConfig, | |||||
private LoggerInterface $logger, | private LoggerInterface $logger, | ||||
private MetadataRequestService $metadataRequestService, | private MetadataRequestService $metadataRequestService, | ||||
private IndexRequestService $indexRequestService, | private IndexRequestService $indexRequestService, | ||||
// update metadata types list | // update metadata types list | ||||
$current = $this->getKnownMetadata(); | $current = $this->getKnownMetadata(); | ||||
$current->import($filesMetadata->jsonSerialize(true)); | $current->import($filesMetadata->jsonSerialize(true)); | ||||
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current)); | |||||
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true); | |||||
} | } | ||||
/** | /** | ||||
* @param string $fileIdField alias of the field that contains file ids | * @param string $fileIdField alias of the field that contains file ids | ||||
* | * | ||||
* @inheritDoc | * @inheritDoc | ||||
* @return IMetadataQuery|null | |||||
* @return IMetadataQuery | |||||
* @see IMetadataQuery | * @see IMetadataQuery | ||||
* @since 28.0.0 | * @since 28.0.0 | ||||
*/ | */ | ||||
IQueryBuilder $qb, | IQueryBuilder $qb, | ||||
string $fileTableAlias, | string $fileTableAlias, | ||||
string $fileIdField | string $fileIdField | ||||
): ?IMetadataQuery { | |||||
if (!$this->metadataInitiated()) { | |||||
return null; | |||||
} | |||||
return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField); | |||||
): IMetadataQuery { | |||||
return new MetadataQuery($qb, $this, $fileTableAlias, $fileIdField); | |||||
} | } | ||||
/** | /** | ||||
$this->all = new FilesMetadata(); | $this->all = new FilesMetadata(); | ||||
try { | try { | ||||
$data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR); | |||||
$this->all->import($data); | |||||
$this->all->import($this->appConfig->getValueArray('core', self::CONFIG_KEY, lazy: true)); | |||||
} catch (JsonException) { | } catch (JsonException) { | ||||
$this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata'); | $this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata'); | ||||
} | } | ||||
} | } | ||||
$current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]); | $current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]); | ||||
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current)); | |||||
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true); | |||||
$this->all = $current; | $this->all = $current; | ||||
} | } | ||||
$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); | $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); | ||||
$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class); | $eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class); | ||||
} | } | ||||
/** | |||||
* Will confirm that tables were created and store an app value to cache the result. | |||||
* Can be removed in 29 as this is to avoid strange situation when Nextcloud files were | |||||
* replaced but the upgrade was not triggered yet. | |||||
* | |||||
* @return bool | |||||
*/ | |||||
private function metadataInitiated(): bool { | |||||
if ($this->config->getAppValue('core', self::MIGRATION_DONE, '0') === '1') { | |||||
return true; | |||||
} | |||||
$dbConnection = \OCP\Server::get(IDBConnection::class); | |||||
if ($dbConnection->tableExists(MetadataRequestService::TABLE_METADATA)) { | |||||
$this->config->setAppValue('core', self::MIGRATION_DONE, '1'); | |||||
return true; | |||||
} | |||||
return false; | |||||
} | |||||
} | } |
use OCP\DB\QueryBuilder\IQueryBuilder; | use OCP\DB\QueryBuilder\IQueryBuilder; | ||||
use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; | use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; | ||||
use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; | use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException; | ||||
use OCP\FilesMetadata\IFilesMetadataManager; | |||||
use OCP\FilesMetadata\IMetadataQuery; | use OCP\FilesMetadata\IMetadataQuery; | ||||
use OCP\FilesMetadata\Model\IFilesMetadata; | use OCP\FilesMetadata\Model\IFilesMetadata; | ||||
use OCP\FilesMetadata\Model\IMetadataValueWrapper; | use OCP\FilesMetadata\Model\IMetadataValueWrapper; | ||||
use Psr\Log\LoggerInterface; | |||||
/** | /** | ||||
* @inheritDoc | * @inheritDoc | ||||
private array $knownJoinedIndex = []; | private array $knownJoinedIndex = []; | ||||
public function __construct( | public function __construct( | ||||
private IQueryBuilder $queryBuilder, | private IQueryBuilder $queryBuilder, | ||||
private IFilesMetadata $knownMetadata, | |||||
private IFilesMetadata|IFilesMetadataManager $manager, | |||||
private string $fileTableAlias = 'fc', | private string $fileTableAlias = 'fc', | ||||
private string $fileIdField = 'fileid', | private string $fileIdField = 'fileid', | ||||
private string $alias = 'meta', | private string $alias = 'meta', | ||||
private string $aliasIndexPrefix = 'meta_index' | private string $aliasIndexPrefix = 'meta_index' | ||||
) { | ) { | ||||
if ($manager instanceof IFilesMetadata) { | |||||
/** | |||||
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter | |||||
* to not call getKnownMetadata() at the load of this class as it is only needed | |||||
* in {@see getMetadataValueField}. | |||||
* | |||||
* FIXME: remove support for IFilesMetadata | |||||
*/ | |||||
$logger = \OCP\Server::get(LoggerInterface::class); | |||||
$logger->debug('It is deprecated to use IFilesMetadata as second parameter when calling MetadataQuery::__construct()'); | |||||
} | |||||
} | } | ||||
/** | /** | ||||
* @since 28.0.0 | * @since 28.0.0 | ||||
*/ | */ | ||||
public function getMetadataValueField(string $metadataKey): string { | public function getMetadataValueField(string $metadataKey): string { | ||||
return match ($this->knownMetadata->getType($metadataKey)) { | |||||
if ($this->manager instanceof IFilesMetadataManager) { | |||||
/** | |||||
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter | |||||
* to not call getKnownMetadata() at the load of this class as it is only needed | |||||
* in this method. | |||||
* | |||||
* FIXME: keep only this line and remove support for previous IFilesMetadata in constructor | |||||
*/ | |||||
$knownMetadata = $this->manager->getKnownMetadata(); | |||||
} else { | |||||
$knownMetadata = $this->manager; | |||||
} | |||||
return match ($knownMetadata->getType($metadataKey)) { | |||||
IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string', | IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string', | ||||
IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int', | IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int', | ||||
default => throw new FilesMetadataTypeException('metadata is not set as indexed'), | default => throw new FilesMetadataTypeException('metadata is not set as indexed'), |
* @param string $fileTableAlias alias of the table that contains data about files | * @param string $fileTableAlias alias of the table that contains data about files | ||||
* @param string $fileIdField alias of the field that contains file ids | * @param string $fileIdField alias of the field that contains file ids | ||||
* | * | ||||
* @return IMetadataQuery|null NULL if table are not set yet or never used | |||||
* @return IMetadataQuery | |||||
* @see IMetadataQuery | * @see IMetadataQuery | ||||
* @since 28.0.0 | * @since 28.0.0 | ||||
*/ | */ | ||||
IQueryBuilder $qb, | IQueryBuilder $qb, | ||||
string $fileTableAlias, | string $fileTableAlias, | ||||
string $fileIdField | string $fileIdField | ||||
): ?IMetadataQuery; | |||||
): IMetadataQuery; | |||||
/** | /** | ||||
* returns all type of metadata currently available. | * returns all type of metadata currently available. | ||||
* The list is stored in a IFilesMetadata with null values but correct type. | * The list is stored in a IFilesMetadata with null values but correct type. | ||||
* | * | ||||
* Note: this method loads lazy appconfig values. | |||||
* | |||||
* @return IFilesMetadata | * @return IFilesMetadata | ||||
* @since 28.0.0 | * @since 28.0.0 | ||||
*/ | */ | ||||
public function getKnownMetadata(): IFilesMetadata; | public function getKnownMetadata(): IFilesMetadata; | ||||
/** | /** | ||||
* initiate a metadata key with its type. | |||||
* Initiate a metadata key with its type. | |||||
* | |||||
* The call is mandatory before using the metadata property in a webdav request. | * The call is mandatory before using the metadata property in a webdav request. | ||||
* It is not needed to only use this method when the app is enabled: the method can be | |||||
* called each time during the app loading as the metadata will only be initiated if not known | |||||
* The call should be part of a migration/repair step and not be called on app's boot | |||||
* process as it is using lazy-appconfig value | |||||
* | |||||
* Note: this method loads lazy appconfig values. | |||||
* | * | ||||
* @param string $key metadata key | * @param string $key metadata key | ||||
* @param string $type metadata type | * @param string $type metadata type | ||||
* @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION | * @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION | ||||
* @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION | * @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION | ||||
* @since 28.0.0 | * @since 28.0.0 | ||||
* @since 29.0.0 uses lazy config value - do not use this method out of repair steps | |||||
*/ | */ | ||||
public function initMetadata(string $key, string $type, bool $indexed, int $editPermission): void; | public function initMetadata(string $key, string $type, bool $indexed, int $editPermission): void; | ||||
} | } |
/** | /** | ||||
* returns the name of the field for metadata string value to be used in query expressions | * returns the name of the field for metadata string value to be used in query expressions | ||||
* | * | ||||
* Note: this method loads lazy appconfig values. | |||||
* | |||||
* @param string $metadataKey metadata key | * @param string $metadataKey metadata key | ||||
* | * | ||||
* @return string table field | * @return string table field |