diff options
Diffstat (limited to 'lib/private/Files/Cache/Cache.php')
| -rw-r--r-- | lib/private/Files/Cache/Cache.php | 241 |
1 files changed, 171 insertions, 70 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 3c871fdf4dc..329466e682d 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -9,6 +9,7 @@ namespace OC\Files\Cache; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchQuery; use OC\Files\Storage\Wrapper\Encryption; @@ -54,7 +55,7 @@ class Cache implements ICache { protected array $partial = []; protected string $storageId; protected Storage $storageCache; - protected IMimeTypeLoader$mimetypeLoader; + protected IMimeTypeLoader $mimetypeLoader; protected IDBConnection $connection; protected SystemConfig $systemConfig; protected LoggerInterface $logger; @@ -73,7 +74,7 @@ class Cache implements ICache { $this->storageId = md5($this->storageId); } if (!$dependencies) { - $dependencies = \OC::$server->get(CacheDependencies::class); + $dependencies = \OCP\Server::get(CacheDependencies::class); } $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection()); $this->mimetypeLoader = $dependencies->getMimeTypeLoader(); @@ -87,9 +88,7 @@ class Cache implements ICache { protected function getQueryBuilder() { return new CacheQueryBuilder( - $this->connection, - $this->systemConfig, - $this->logger, + $this->connection->getQueryBuilder(), $this->metadataManager, ); } @@ -110,7 +109,7 @@ class Cache implements ICache { /** * get the stored metadata of a file or folder * - * @param string | int $file either the path of a file or folder or the file id for a file or folder + * @param string|int $file either the path of a file or folder or the file id for a file or folder * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache */ public function get($file) { @@ -122,25 +121,27 @@ class Cache implements ICache { // normalize file $file = $this->normalize($file); - $query->whereStorageId($this->getNumericStorageId()) - ->wherePath($file); + $query->wherePath($file); } else { //file id $query->whereFileId($file); } + $query->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $data = $result->fetch(); $result->closeCursor(); - //merge partial data - if (!$data && is_string($file) && isset($this->partial[$file])) { - return $this->partial[$file]; - } elseif (!$data) { - return $data; - } else { + if ($data !== false) { $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); + } else { + //merge partial data + if (is_string($file) && isset($this->partial[$file])) { + return $this->partial[$file]; + } } + + return false; } /** @@ -169,6 +170,9 @@ class Cache implements ICache { if ($data['storage_mtime'] == 0) { $data['storage_mtime'] = $data['mtime']; } + if (isset($data['f_permissions'])) { + $data['scan_permissions'] = $data['f_permissions']; + } $data['permissions'] = (int)$data['permissions']; if (isset($data['creation_time'])) { $data['creation_time'] = (int)$data['creation_time']; @@ -201,11 +205,12 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->selectFileCache() ->whereParent($fileId) + ->whereStorageId($this->getNumericStorageId()) ->orderBy('name', 'ASC'); $metadataQuery = $query->selectMetadata(); - $result = $query->execute(); + $result = $query->executeQuery(); $files = $result->fetchAll(); $result->closeCursor(); @@ -249,14 +254,14 @@ class Cache implements ICache { $file = $this->normalize($file); if (isset($this->partial[$file])) { //add any saved partial data - $data = array_merge($this->partial[$file], $data); + $data = array_merge($this->partial[$file]->getData(), $data); unset($this->partial[$file]); } $requiredFields = ['size', 'mtime', 'mimetype']; foreach ($requiredFields as $field) { if (!isset($data[$field])) { //data not complete save as partial and return - $this->partial[$file] = $data; + $this->partial[$file] = new CacheEntry($data); return -1; } } @@ -265,6 +270,9 @@ class Cache implements ICache { if (!isset($data['parent'])) { $data['parent'] = $this->getParentId($file); } + if ($data['parent'] === -1 && $file !== '') { + throw new \Exception('Parent folder not in filecache for ' . $file); + } $data['name'] = basename($file); [$values, $extensionValues] = $this->normalizeData($data); @@ -285,12 +293,13 @@ class Cache implements ICache { if (count($extensionValues)) { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $storageId); $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { $query->setValue($column, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId); @@ -339,6 +348,7 @@ class Cache implements ICache { $query->update('filecache') ->whereFileId($id) + ->whereStorageId($this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -350,13 +360,14 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } if (count($extensionValues)) { try { $query = $this->getQueryBuilder(); $query->insert('filecache_extended'); + $query->hintShardKey('storage', $this->getNumericStorageId()); $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)); foreach ($extensionValues as $column => $value) { @@ -368,6 +379,7 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->update('filecache_extended') ->whereFileId($id) + ->hintShardKey('storage', $this->getNumericStorageId()) ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) { return $query->expr()->orX( $query->expr()->neq($key, $query->createNamedParameter($value)), @@ -379,7 +391,7 @@ class Cache implements ICache { $query->set($key, $query->createNamedParameter($value)); } - $query->execute(); + $query->executeStatement(); } } @@ -461,7 +473,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $id = $result->fetchOne(); $result->closeCursor(); @@ -514,13 +526,15 @@ class Cache implements ICache { if ($entry instanceof ICacheEntry) { $query = $this->getQueryBuilder(); $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereFileId($entry->getId()); - $query->execute(); + $query->executeStatement(); $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->whereFileId($entry->getId()); - $query->execute(); + ->whereFileId($entry->getId()) + ->hintShardKey('storage', $this->getNumericStorageId()); + $query->executeStatement(); if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) { $this->removeChildren($entry); @@ -563,11 +577,12 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache_extended') - ->where($query->expr()->in('fileid', $query->createParameter('childIds'))); + ->where($query->expr()->in('fileid', $query->createParameter('childIds'))) + ->hintShardKey('storage', $this->getNumericStorageId()); foreach (array_chunk($childIds, 1000) as $childIdChunk) { $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY); - $query->execute(); + $query->executeStatement(); } /** @var ICacheEntry[] $childFolders */ @@ -585,6 +600,7 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereParentInParameter('parentIds'); // Sorting before chunking allows the db to find the entries close to each @@ -592,7 +608,7 @@ class Cache implements ICache { sort($parentIds, SORT_NUMERIC); foreach (array_chunk($parentIds, 1000) as $parentIdChunk) { $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY); - $query->execute(); + $query->executeStatement(); } foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { @@ -647,7 +663,16 @@ class Cache implements ICache { $sourceData = $sourceCache->get($sourcePath); if (!$sourceData) { - throw new \Exception('Invalid source storage path: ' . $sourcePath); + throw new \Exception('Source path not found in cache: ' . $sourcePath); + } + + $shardDefinition = $this->connection->getShardDefinition('filecache'); + if ( + $shardDefinition + && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId()) + ) { + $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath); + return; } $sourceId = $sourceData['fileid']; @@ -671,7 +696,7 @@ class Cache implements ICache { $childChunks = array_chunk($childIds, 1000); - $query = $this->connection->getQueryBuilder(); + $query = $this->getQueryBuilder(); $fun = $query->func(); $newPathFunction = $fun->concat( @@ -679,12 +704,15 @@ class Cache implements ICache { $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash ); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT)) ->set('path_hash', $fun->md5($newPathFunction)) ->set('path', $newPathFunction) - ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT))) + ->whereStorageId($sourceStorageId) ->andWhere($query->expr()->in('fileid', $query->createParameter('files'))); + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); @@ -726,19 +754,23 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->update('filecache') - ->set('storage', $query->createNamedParameter($targetStorageId)) ->set('path', $query->createNamedParameter($targetPath)) ->set('path_hash', $query->createNamedParameter(md5($targetPath))) ->set('name', $query->createNamedParameter(basename($targetPath))) ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT)) + ->whereStorageId($sourceStorageId) ->whereFileId($sourceId); + if ($sourceStorageId !== $targetStorageId) { + $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT); + } + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) { $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)); } - $query->execute(); + $query->executeStatement(); $this->connection->commit(); @@ -773,12 +805,12 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->delete('filecache') ->whereStorageId($this->getNumericStorageId()); - $query->execute(); + $query->executeStatement(); $query = $this->connection->getQueryBuilder(); $query->delete('storages') ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId))); - $query->execute(); + $query->executeStatement(); } /** @@ -803,7 +835,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->wherePath($file); - $result = $query->execute(); + $result = $query->executeQuery(); $size = $result->fetchOne(); $result->closeCursor(); @@ -837,7 +869,7 @@ class Cache implements ICache { * search for files by mimetype * * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image') - * where it will search for all mimetypes in the group ('image/*') + * where it will search for all mimetypes in the group ('image/*') * @return ICacheEntry[] an array of cache entries where the mimetype matches the search */ public function searchByMime($mimetype) { @@ -849,26 +881,30 @@ class Cache implements ICache { return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null)); } - public function searchQuery(ISearchQuery $searchQuery) { - return current($this->querySearchHelper->searchInCaches($searchQuery, [$this])); + public function searchQuery(ISearchQuery $query) { + return current($this->querySearchHelper->searchInCaches($query, [$this])); } /** * Re-calculate the folder size and the size of all parent folders * - * @param string|boolean $path - * @param array $data (optional) meta data of the folder + * @param array|ICacheEntry|null $data (optional) meta data of the folder */ - public function correctFolderSize($path, $data = null, $isBackgroundScan = false) { + public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void { $this->calculateFolderSize($path, $data); + if ($path !== '') { $parent = dirname($path); if ($parent === '.' || $parent === '/') { $parent = ''; } + if ($isBackgroundScan) { $parentData = $this->get($parent); - if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) { + if ($parentData !== false + && $parentData['size'] !== -1 + && $this->getIncompleteChildrenCount($parentData['fileid']) === 0 + ) { $this->correctFolderSize($parent, $parentData, $isBackgroundScan); } } else { @@ -889,9 +925,10 @@ class Cache implements ICache { $query->select($query->func()->count()) ->from('filecache') ->whereParent($fileId) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))); + ->whereStorageId($this->getNumericStorageId()) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $size = (int)$result->fetchOne(); $result->closeCursor(); @@ -931,12 +968,13 @@ class Cache implements ICache { $query = $this->getQueryBuilder(); $query->select('size', 'unencrypted_size') ->from('filecache') + ->whereStorageId($this->getNumericStorageId()) ->whereParent($id); if ($ignoreUnknown) { $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0))); } - $result = $query->execute(); + $result = $query->executeQuery(); $rows = $result->fetchAll(); $result->closeCursor(); @@ -977,8 +1015,8 @@ class Cache implements ICache { } // only set unencrypted size for a folder if any child entries have it set, or the folder is empty - $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || $entry['unencrypted_size'] > 0; - if ($entry['size'] !== $totalSize || ($entry['unencrypted_size'] !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { + $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0; + if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) { if ($shouldWriteUnEncryptedSize) { // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes if ($unencryptedMax === 0) { @@ -1010,7 +1048,7 @@ class Cache implements ICache { ->from('filecache') ->whereStorageId($this->getNumericStorageId()); - $result = $query->execute(); + $result = $query->executeQuery(); $files = $result->fetchAll(\PDO::FETCH_COLUMN); $result->closeCursor(); @@ -1029,28 +1067,19 @@ class Cache implements ICache { * @return string|false the path of the folder or false when no folder matched */ public function getIncomplete() { - // we select the fileid here first instead of directly selecting the path since this helps mariadb/mysql - // to use the correct index. - // The overhead of this should be minimal since the cost of selecting the path by id should be much lower - // than the cost of finding an item with size < 0 $query = $this->getQueryBuilder(); - $query->select('fileid') + $query->select('path') ->from('filecache') ->whereStorageId($this->getNumericStorageId()) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('fileid', 'DESC') ->setMaxResults(1); - $result = $query->execute(); - $id = $result->fetchOne(); + $result = $query->executeQuery(); + $path = $result->fetchOne(); $result->closeCursor(); - if ($id === false) { - return false; - } - - $path = $this->getPathById($id); - return $path ?? false; + return $path === false ? false : (string)$path; } /** @@ -1066,7 +1095,7 @@ class Cache implements ICache { ->whereStorageId($this->getNumericStorageId()) ->whereFileId($id); - $result = $query->execute(); + $result = $query->executeQuery(); $path = $result->fetchOne(); $result->closeCursor(); @@ -1084,7 +1113,7 @@ class Cache implements ICache { * * @param int $id * @return array first element holding the storage id, second the path - * @deprecated use getPathById() instead + * @deprecated 17.0.0 use getPathById() instead */ public static function getById($id) { $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); @@ -1092,7 +1121,7 @@ class Cache implements ICache { ->from('filecache') ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); - $result = $query->execute(); + $result = $query->executeQuery(); $row = $result->fetch(); $result->closeCursor(); @@ -1130,7 +1159,7 @@ class Cache implements ICache { */ public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int { if ($sourceEntry->getId() < 0) { - throw new \RuntimeException("Invalid source cache entry on copyFromCache"); + throw new \RuntimeException('Invalid source cache entry on copyFromCache'); } $data = $this->cacheEntryToArray($sourceEntry); @@ -1141,7 +1170,7 @@ class Cache implements ICache { $fileId = $this->put($targetPath, $data); if ($fileId <= 0) { - throw new \RuntimeException("Failed to copy to " . $targetPath . " from cache with source data " . json_encode($data) . " "); + throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' '); } if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId()); @@ -1154,7 +1183,7 @@ class Cache implements ICache { } private function cacheEntryToArray(ICacheEntry $entry): array { - return [ + $data = [ 'size' => $entry->getSize(), 'mtime' => $entry->getMTime(), 'storage_mtime' => $entry->getStorageMTime(), @@ -1167,6 +1196,10 @@ class Cache implements ICache { 'upload_time' => $entry->getUploadTime(), 'metadata_etag' => $entry->getMetadataEtag(), ]; + if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) { + $data['permissions'] = $entry['scan_permissions']; + } + return $data; } public function getQueryFilterForStorage(): ISearchOperator { @@ -1180,4 +1213,72 @@ class Cache implements ICache { return null; } } + + private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void { + if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath()); + } else { + $fileIds = []; + } + $fileIds[] = $sourceEntry->getId(); + + $helper = $this->connection->getCrossShardMoveHelper(); + + $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId()); + $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId()); + + $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + + // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark + $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper(); + + $sourcePathLength = strlen($sourceEntry->getPath()); + foreach ($cacheItems as &$cacheItem) { + if ($cacheItem['path'] === $sourceEntry->getPath()) { + $cacheItem['path'] = $targetPath; + $cacheItem['parent'] = $this->getParentId($targetPath); + $cacheItem['name'] = basename($cacheItem['path']); + } else { + $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash + } + $cacheItem['path_hash'] = md5($cacheItem['path']); + $cacheItem['storage'] = $this->getNumericStorageId(); + if ($removeEncryptedFlag) { + $cacheItem['encrypted'] = 0; + } + } + + $targetConnection->beginTransaction(); + + try { + $helper->saveItems($targetConnection, 'filecache', $cacheItems); + $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems); + $helper->saveItems($targetConnection, 'files_metadata', $metadataItems); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + + $sourceConnection->beginTransaction(); + + try { + $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds); + $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds); + } catch (\Exception $e) { + $targetConnection->rollback(); + $sourceConnection->rollBack(); + throw $e; + } + + try { + $sourceConnection->commit(); + } catch (\Exception $e) { + $targetConnection->rollback(); + throw $e; + } + $targetConnection->commit(); + } } |