diff options
Diffstat (limited to 'lib/private/FilesMetadata')
9 files changed, 97 insertions, 207 deletions
diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index 08c1b4f459c..4846799b3d4 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata; @@ -92,13 +75,16 @@ class FilesMetadataManager implements IFilesMetadataManager { public function refreshMetadata( Node $node, int $process = self::PROCESS_LIVE, - string $namedEvent = '' + string $namedEvent = '', ): IFilesMetadata { + $storageId = $node->getStorage()->getCache()->getNumericStorageId(); try { + /** @var FilesMetadata $metadata */ $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); } catch (FilesMetadataNotFoundException) { $metadata = new FilesMetadata($node->getId()); } + $metadata->setStorageId($storageId); // if $process is LIVE, we enforce LIVE // if $process is NAMED, we go NAMED @@ -121,7 +107,7 @@ class FilesMetadataManager implements IFilesMetadataManager { return $this->refreshMetadata($node, self::PROCESS_BACKGROUND); } - $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]); + $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()?->getUID(), $node->getId()]); } return $metadata; @@ -241,7 +227,7 @@ class FilesMetadataManager implements IFilesMetadataManager { public function getMetadataQuery( IQueryBuilder $qb, string $fileTableAlias, - string $fileIdField + string $fileIdField, ): IMetadataQuery { return new MetadataQuery($qb, $this, $fileTableAlias, $fileIdField); } @@ -290,7 +276,7 @@ class FilesMetadataManager implements IFilesMetadataManager { string $key, string $type, bool $indexed = false, - int $editPermission = IMetadataValueWrapper::EDIT_FORBIDDEN + int $editPermission = IMetadataValueWrapper::EDIT_FORBIDDEN, ): void { $current = $this->getKnownMetadata(); try { diff --git a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php index 3a3b35ce205..5746493ddef 100644 --- a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php +++ b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Job; @@ -46,7 +29,7 @@ class UpdateSingleMetadata extends QueuedJob { ITimeFactory $time, private IRootFolder $rootFolder, private FilesMetadataManager $filesMetadataManager, - private LoggerInterface $logger + private LoggerInterface $logger, ) { parent::__construct($time); } diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php index d950c2cea5f..226bd3bdafa 100644 --- a/lib/private/FilesMetadata/Listener/MetadataDelete.php +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Listener; @@ -40,7 +23,7 @@ use Psr\Log\LoggerInterface; class MetadataDelete implements IEventListener { public function __construct( private IFilesMetadataManager $filesMetadataManager, - private LoggerInterface $logger + private LoggerInterface $logger, ) { } diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php index 9848f079882..4c5c913c740 100644 --- a/lib/private/FilesMetadata/Listener/MetadataUpdate.php +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Listener; @@ -43,7 +26,7 @@ use Psr\Log\LoggerInterface; class MetadataUpdate implements IEventListener { public function __construct( private IFilesMetadataManager $filesMetadataManager, - private LoggerInterface $logger + private LoggerInterface $logger, ) { } diff --git a/lib/private/FilesMetadata/MetadataQuery.php b/lib/private/FilesMetadata/MetadataQuery.php index e7bb0225174..deae433e2fc 100644 --- a/lib/private/FilesMetadata/MetadataQuery.php +++ b/lib/private/FilesMetadata/MetadataQuery.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata; @@ -49,7 +32,7 @@ class MetadataQuery implements IMetadataQuery { private string $fileTableAlias = 'fc', private string $fileIdField = 'fileid', private string $alias = 'meta', - private string $aliasIndexPrefix = 'meta_index' + private string $aliasIndexPrefix = 'meta_index', ) { if ($manager instanceof IFilesMetadata) { /** diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php index 41ede3ef2d3..b66e1fe3711 100644 --- a/lib/private/FilesMetadata/Model/FilesMetadata.php +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Model; @@ -44,9 +27,10 @@ class FilesMetadata implements IFilesMetadata { private bool $updated = false; private int $lastUpdate = 0; private string $syncToken = ''; + private ?int $storageId = null; public function __construct( - private int $fileId = 0 + private int $fileId = 0, ) { } @@ -59,6 +43,22 @@ class FilesMetadata implements IFilesMetadata { return $this->fileId; } + public function getStorageId(): ?int { + return $this->storageId; + } + + /** + * Set which storage the file this metadata belongs to. + * + * This helps with sharded filecache setups to know where to store the metadata + * + * @param int $storageId + * @return void + */ + public function setStorageId(int $storageId): void { + $this->storageId = $storageId; + } + /** * @inheritDoc * @return int timestamp diff --git a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php index be38b234be3..710a8129340 100644 --- a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php +++ b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Model; diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php index 2a23e2c9a67..91bd9f0b11e 100644 --- a/lib/private/FilesMetadata/Service/IndexRequestService.php +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Service; @@ -42,7 +25,7 @@ class IndexRequestService { public function __construct( private IDBConnection $dbConnection, - private LoggerInterface $logger + private LoggerInterface $logger, ) { } @@ -99,9 +82,9 @@ class IndexRequestService { private function insertIndexString(int $fileId, string $key, string $value): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->insert(self::TABLE_METADATA_INDEX) - ->setValue('meta_key', $qb->createNamedParameter($key)) - ->setValue('meta_value_string', $qb->createNamedParameter($value)) - ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_string', $qb->createNamedParameter($value)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); $qb->executeStatement(); } @@ -117,9 +100,9 @@ class IndexRequestService { public function insertIndexInt(int $fileId, string $key, int $value): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->insert(self::TABLE_METADATA_INDEX) - ->setValue('meta_key', $qb->createNamedParameter($key)) - ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) - ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); $qb->executeStatement(); } @@ -135,9 +118,9 @@ class IndexRequestService { public function insertIndexBool(int $fileId, string $key, bool $value): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->insert(self::TABLE_METADATA_INDEX) - ->setValue('meta_key', $qb->createNamedParameter($key)) - ->setValue('meta_value_int', $qb->createNamedParameter(($value) ? '1' : '0', IQueryBuilder::PARAM_INT)) - ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter(($value) ? '1' : '0', IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); $qb->executeStatement(); } @@ -184,7 +167,7 @@ class IndexRequestService { $qb = $this->dbConnection->getQueryBuilder(); $expr = $qb->expr(); $qb->delete(self::TABLE_METADATA_INDEX) - ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); if ($key !== '') { $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php index b6d1b277a00..c308ae1c9c8 100644 --- a/lib/private/FilesMetadata/Service/MetadataRequestService.php +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -2,25 +2,8 @@ declare(strict_types=1); /** - * @copyright 2023 Maxence Lange <maxence@artificial-owl.com> - * - * @author Maxence Lange <maxence@artificial-owl.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\FilesMetadata\Service; @@ -41,10 +24,31 @@ class MetadataRequestService { public function __construct( private IDBConnection $dbConnection, - private LoggerInterface $logger + private LoggerInterface $logger, ) { } + private function getStorageId(IFilesMetadata $filesMetadata): int { + if ($filesMetadata instanceof FilesMetadata) { + $storage = $filesMetadata->getStorageId(); + if ($storage) { + return $storage; + } + } + // all code paths that lead to saving metadata *should* have the storage id set + // this fallback is there just in case + $query = $this->dbConnection->getQueryBuilder(); + $query->select('storage') + ->from('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + $storageId = $query->executeQuery()->fetchColumn(); + + if ($filesMetadata instanceof FilesMetadata) { + $filesMetadata->setStorageId($storageId); + } + return $storageId; + } + /** * store metadata into database * @@ -55,10 +59,11 @@ class MetadataRequestService { public function store(IFilesMetadata $filesMetadata): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->insert(self::TABLE_METADATA) - ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) - ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) - ->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken())) - ->setValue('last_update', (string) $qb->createFunction('NOW()')); + ->hintShardKey('storage', $this->getStorageId($filesMetadata)) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken())) + ->setValue('last_update', (string)$qb->createFunction('NOW()')); $qb->executeStatement(); } @@ -109,7 +114,7 @@ class MetadataRequestService { $list = []; $result = $qb->executeQuery(); while ($data = $result->fetch()) { - $fileId = (int) $data['file_id']; + $fileId = (int)$data['file_id']; $metadata = new FilesMetadata($fileId); try { $metadata->importFromDatabase($data); @@ -134,7 +139,7 @@ class MetadataRequestService { public function dropMetadata(int $fileId): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->delete(self::TABLE_METADATA) - ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); $qb->executeStatement(); } @@ -151,15 +156,16 @@ class MetadataRequestService { $expr = $qb->expr(); $qb->update(self::TABLE_METADATA) - ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) - ->set('sync_token', $qb->createNamedParameter($this->generateSyncToken())) - ->set('last_update', $qb->createFunction('NOW()')) - ->where( - $expr->andX( - $expr->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)), - $expr->eq('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) - ) - ); + ->hintShardKey('files_metadata', $this->getStorageId($filesMetadata)) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter($this->generateSyncToken())) + ->set('last_update', $qb->createFunction('NOW()')) + ->where( + $expr->andX( + $expr->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)), + $expr->eq('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) + ) + ); return $qb->executeStatement(); } |