diff options
6 files changed, 149 insertions, 0 deletions
diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 3855f51c65a..dbd4e522517 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -8,6 +8,7 @@ namespace OCA\DAV\Connector\Sabre; use OC\AppFramework\Http\Request; +use OC\FilesMetadata\Model\FilesMetadata; use OCA\DAV\Connector\Sabre\Exception\InvalidPath; use OCP\Constants; use OCP\Files\ForbiddenException; @@ -575,7 +576,9 @@ class FilesPlugin extends ServerPlugin { $propPatch->handle( $mutation, function (mixed $value) use ($accessRight, $knownMetadata, $node, $mutation, $filesMetadataManager): bool { + /** @var FilesMetadata $metadata */ $metadata = $filesMetadataManager->getMetadata((int)$node->getFileId(), true); + $metadata->setStorageId($node->getNode()->getStorage()->getCache()->getNumericStorageId()); $metadataKey = substr($mutation, strlen(self::FILE_METADATA_PREFIX)); // confirm metadata key is editable via PROPPATCH diff --git a/apps/files/lib/Listener/SyncLivePhotosListener.php b/apps/files/lib/Listener/SyncLivePhotosListener.php index 049d7319b64..34789187644 100644 --- a/apps/files/lib/Listener/SyncLivePhotosListener.php +++ b/apps/files/lib/Listener/SyncLivePhotosListener.php @@ -8,6 +8,7 @@ declare(strict_types=1); namespace OCA\Files\Listener; +use OC\FilesMetadata\Model\FilesMetadata; use OCA\Files\Service\LivePhotosService; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; @@ -154,10 +155,14 @@ class SyncLivePhotosListener implements IEventListener { * We have everything to update metadata and keep the link between the 2 copies. */ $newPeerFile = $peerFile->copy($targetParent->getPath() . '/' . $peerTargetName); + /** @var FilesMetadata $targetMetadata */ $targetMetadata = $this->filesMetadataManager->getMetadata($targetFile->getId(), true); + $targetMetadata->setStorageId($targetFile->getStorage()->getCache()->getNumericStorageId()); $targetMetadata->setString('files-live-photo', (string)$newPeerFile->getId()); $this->filesMetadataManager->saveMetadata($targetMetadata); + /** @var FilesMetadata $peerMetadata */ $peerMetadata = $this->filesMetadataManager->getMetadata($newPeerFile->getId(), true); + $peerMetadata->setStorageId($newPeerFile->getStorage()->getCache()->getNumericStorageId()); $peerMetadata->setString('files-live-photo', (string)$targetFile->getId()); $this->filesMetadataManager->saveMetadata($peerMetadata); } diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index f2ea6cb11a7..4846799b3d4 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -77,11 +77,14 @@ class FilesMetadataManager implements IFilesMetadataManager { int $process = self::PROCESS_LIVE, string $namedEvent = '', ): IFilesMetadata { + $storageId = $node->getStorage()->getCache()->getNumericStorageId(); try { + /** @var FilesMetadata $metadata */ $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); } catch (FilesMetadataNotFoundException) { $metadata = new FilesMetadata($node->getId()); } + $metadata->setStorageId($storageId); // if $process is LIVE, we enforce LIVE // if $process is NAMED, we go NAMED diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php index d338bdbba8f..b66e1fe3711 100644 --- a/lib/private/FilesMetadata/Model/FilesMetadata.php +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -27,6 +27,7 @@ class FilesMetadata implements IFilesMetadata { private bool $updated = false; private int $lastUpdate = 0; private string $syncToken = ''; + private ?int $storageId = null; public function __construct( private int $fileId = 0, @@ -42,6 +43,22 @@ class FilesMetadata implements IFilesMetadata { return $this->fileId; } + public function getStorageId(): ?int { + return $this->storageId; + } + + /** + * Set which storage the file this metadata belongs to. + * + * This helps with sharded filecache setups to know where to store the metadata + * + * @param int $storageId + * @return void + */ + public function setStorageId(int $storageId): void { + $this->storageId = $storageId; + } + /** * @inheritDoc * @return int timestamp diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php index 5fc02c5d00b..c308ae1c9c8 100644 --- a/lib/private/FilesMetadata/Service/MetadataRequestService.php +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -28,6 +28,27 @@ class MetadataRequestService { ) { } + private function getStorageId(IFilesMetadata $filesMetadata): int { + if ($filesMetadata instanceof FilesMetadata) { + $storage = $filesMetadata->getStorageId(); + if ($storage) { + return $storage; + } + } + // all code paths that lead to saving metadata *should* have the storage id set + // this fallback is there just in case + $query = $this->dbConnection->getQueryBuilder(); + $query->select('storage') + ->from('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + $storageId = $query->executeQuery()->fetchColumn(); + + if ($filesMetadata instanceof FilesMetadata) { + $filesMetadata->setStorageId($storageId); + } + return $storageId; + } + /** * store metadata into database * @@ -38,6 +59,7 @@ class MetadataRequestService { public function store(IFilesMetadata $filesMetadata): void { $qb = $this->dbConnection->getQueryBuilder(); $qb->insert(self::TABLE_METADATA) + ->hintShardKey('storage', $this->getStorageId($filesMetadata)) ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) ->setValue('sync_token', $qb->createNamedParameter($this->generateSyncToken())) @@ -134,6 +156,7 @@ class MetadataRequestService { $expr = $qb->expr(); $qb->update(self::TABLE_METADATA) + ->hintShardKey('files_metadata', $this->getStorageId($filesMetadata)) ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) ->set('sync_token', $qb->createNamedParameter($this->generateSyncToken())) ->set('last_update', $qb->createFunction('NOW()')) diff --git a/tests/lib/FilesMetadata/FilesMetadataManagerTest.php b/tests/lib/FilesMetadata/FilesMetadataManagerTest.php new file mode 100644 index 00000000000..2f9edba0015 --- /dev/null +++ b/tests/lib/FilesMetadata/FilesMetadataManagerTest.php @@ -0,0 +1,98 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace Test\FilesMetadata; + +use OC\BackgroundJob\JobList; +use OC\Files\Storage\Temporary; +use OC\FilesMetadata\FilesMetadataManager; +use OC\FilesMetadata\Service\IndexRequestService; +use OC\FilesMetadata\Service\MetadataRequestService; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\FilesMetadata\AMetadataEvent; +use OCP\IAppConfig; +use OCP\IDBConnection; +use OCP\Server; +use Psr\Log\LoggerInterface; +use Test\TestCase; +use Test\Traits\MountProviderTrait; +use Test\Traits\UserTrait; + +/** + * @group DB + */ +class FilesMetadataManagerTest extends TestCase { + use UserTrait; + use MountProviderTrait; + + private IEventDispatcher $eventDispatcher; + private JobList $jobList; + private IAppConfig $appConfig; + private LoggerInterface $logger; + private MetadataRequestService $metadataRequestService; + private IndexRequestService $indexRequestService; + private FilesMetadataManager $manager; + private IDBConnection $connection; + private Folder $userFolder; + private array $metadata = []; + + protected function setUp(): void { + parent::setUp(); + + $this->jobList = $this->createMock(JobList::class); + $this->eventDispatcher = $this->createMock(IEventDispatcher::class); + $this->eventDispatcher->method('dispatchTyped')->willReturnCallback(function (Event $event) { + if ($event instanceof AMetadataEvent) { + $name = $event->getNode()->getName(); + if (isset($this->metadata[$name])) { + $meta = $event->getMetadata(); + foreach ($this->metadata[$name] as $key => $value) { + $meta->setString($key, $value); + } + } + } + }); + $this->appConfig = $this->createMock(IAppConfig::class); + $this->logger = $this->createMock(LoggerInterface::class); + + $this->connection = Server::get(IDBConnection::class); + $this->metadataRequestService = new MetadataRequestService($this->connection, $this->logger); + $this->indexRequestService = new IndexRequestService($this->connection, $this->logger); + $this->manager = new FilesMetadataManager( + $this->eventDispatcher, + $this->jobList, + $this->appConfig, + $this->logger, + $this->metadataRequestService, + $this->indexRequestService, + ); + + $this->createUser('metatest', ''); + $this->registerMount('metatest', new Temporary([]), '/metatest'); + + $rootFolder = Server::get(IRootFolder::class); + $this->userFolder = $rootFolder->getUserFolder('metatest'); + } + + public function testRefreshMetadata(): void { + $this->metadata['test.txt'] = [ + 'istest' => 'yes' + ]; + $file = $this->userFolder->newFile('test.txt', 'test'); + $stored = $this->manager->refreshMetadata($file); + $this->assertEquals($file->getId(), $stored->getFileId()); + $this->assertEquals('yes', $stored->getString('istest')); + + $retrieved = $this->manager->getMetadata($file->getId()); + $this->assertEquals($file->getId(), $retrieved->getFileId()); + $this->assertEquals('yes', $retrieved->getString('istest')); + } +} |