aboutsummaryrefslogtreecommitdiffstats
path: root/lib/private/Files/Cache/Cache.php
diff options
context:
space:
mode:
Diffstat (limited to 'lib/private/Files/Cache/Cache.php')
-rw-r--r--lib/private/Files/Cache/Cache.php533
1 files changed, 353 insertions, 180 deletions
diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php
index 6440bf05a1d..329466e682d 100644
--- a/lib/private/Files/Cache/Cache.php
+++ b/lib/private/Files/Cache/Cache.php
@@ -1,54 +1,25 @@
<?php
+
/**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author Andreas Fischer <bantu@owncloud.com>
- * @author Ari Selseng <ari@selseng.net>
- * @author Artem Kochnev <MrJeos@gmail.com>
- * @author Björn Schießle <bjoern@schiessle.org>
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Daniel Kesselberg <mail@danielkesselberg.de>
- * @author Florin Peter <github@florin-peter.de>
- * @author Frédéric Fortier <frederic.fortier@oronospolytechnique.com>
- * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch>
- * @author Joas Schilling <coding@schilljs.com>
- * @author John Molakvoæ <skjnldsv@protonmail.com>
- * @author Jörn Friedrich Dreyer <jfd@butonic.de>
- * @author Lukas Reschke <lukas@statuscode.ch>
- * @author Michael Gapczynski <GapczynskiM@gmail.com>
- * @author Morris Jobke <hey@morrisjobke.de>
- * @author Robin Appelman <robin@icewind.nl>
- * @author Robin McCorkell <robin@mccorkell.me.uk>
- * @author Roeland Jago Douma <roeland@famdouma.nl>
- * @author Vincent Petry <vincent@nextcloud.com>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
*/
-
namespace OC\Files\Cache;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
+use OC\DB\Exceptions\DbalException;
+use OC\DB\QueryBuilder\Sharded\ShardDefinition;
use OC\Files\Search\SearchComparison;
use OC\Files\Search\SearchQuery;
+use OC\Files\Storage\Wrapper\Encryption;
+use OC\SystemConfig;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Cache\CacheEntryInsertedEvent;
+use OCP\Files\Cache\CacheEntryRemovedEvent;
use OCP\Files\Cache\CacheEntryUpdatedEvent;
use OCP\Files\Cache\CacheInsertEvent;
-use OCP\Files\Cache\CacheEntryRemovedEvent;
use OCP\Files\Cache\CacheUpdateEvent;
use OCP\Files\Cache\ICache;
use OCP\Files\Cache\ICacheEntry;
@@ -58,7 +29,9 @@ use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
use OCP\Files\Search\ISearchQuery;
use OCP\Files\Storage\IStorage;
+use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\IDBConnection;
+use OCP\Util;
use Psr\Log\LoggerInterface;
/**
@@ -79,61 +52,51 @@ class Cache implements ICache {
/**
* @var array partial data for the cache
*/
- protected $partial = [];
-
- /**
- * @var string
- */
- protected $storageId;
-
- private $storage;
-
- /**
- * @var Storage $storageCache
- */
- protected $storageCache;
-
- /** @var IMimeTypeLoader */
- protected $mimetypeLoader;
-
- /**
- * @var IDBConnection
- */
- protected $connection;
-
- /**
- * @var IEventDispatcher
- */
- protected $eventDispatcher;
-
- /** @var QuerySearchHelper */
- protected $querySearchHelper;
-
- /**
- * @param IStorage $storage
- */
- public function __construct(IStorage $storage) {
+ protected array $partial = [];
+ protected string $storageId;
+ protected Storage $storageCache;
+ protected IMimeTypeLoader $mimetypeLoader;
+ protected IDBConnection $connection;
+ protected SystemConfig $systemConfig;
+ protected LoggerInterface $logger;
+ protected QuerySearchHelper $querySearchHelper;
+ protected IEventDispatcher $eventDispatcher;
+ protected IFilesMetadataManager $metadataManager;
+
+ public function __construct(
+ private IStorage $storage,
+ // this constructor is used in to many pleases to easily do proper di
+ // so instead we group it all together
+ ?CacheDependencies $dependencies = null,
+ ) {
$this->storageId = $storage->getId();
- $this->storage = $storage;
if (strlen($this->storageId) > 64) {
$this->storageId = md5($this->storageId);
}
-
- $this->storageCache = new Storage($storage);
- $this->mimetypeLoader = \OC::$server->getMimeTypeLoader();
- $this->connection = \OC::$server->getDatabaseConnection();
- $this->eventDispatcher = \OC::$server->get(IEventDispatcher::class);
- $this->querySearchHelper = \OC::$server->query(QuerySearchHelper::class);
+ if (!$dependencies) {
+ $dependencies = \OCP\Server::get(CacheDependencies::class);
+ }
+ $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection());
+ $this->mimetypeLoader = $dependencies->getMimeTypeLoader();
+ $this->connection = $dependencies->getConnection();
+ $this->systemConfig = $dependencies->getSystemConfig();
+ $this->logger = $dependencies->getLogger();
+ $this->querySearchHelper = $dependencies->getQuerySearchHelper();
+ $this->eventDispatcher = $dependencies->getEventDispatcher();
+ $this->metadataManager = $dependencies->getMetadataManager();
}
protected function getQueryBuilder() {
return new CacheQueryBuilder(
- $this->connection,
- \OC::$server->getSystemConfig(),
- \OC::$server->get(LoggerInterface::class)
+ $this->connection->getQueryBuilder(),
+ $this->metadataManager,
);
}
+ public function getStorageCache(): Storage {
+ return $this->storageCache;
+ }
+
/**
* Get the numeric storage id for this cache's storage
*
@@ -146,35 +109,39 @@ class Cache implements ICache {
/**
* get the stored metadata of a file or folder
*
- * @param string | int $file either the path of a file or folder or the file id for a file or folder
- * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache
+ * @param string|int $file either the path of a file or folder or the file id for a file or folder
+ * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache
*/
public function get($file) {
$query = $this->getQueryBuilder();
$query->selectFileCache();
+ $metadataQuery = $query->selectMetadata();
if (is_string($file) || $file == '') {
// normalize file
$file = $this->normalize($file);
- $query->whereStorageId($this->getNumericStorageId())
- ->wherePath($file);
+ $query->wherePath($file);
} else { //file id
$query->whereFileId($file);
}
+ $query->whereStorageId($this->getNumericStorageId());
- $result = $query->execute();
+ $result = $query->executeQuery();
$data = $result->fetch();
$result->closeCursor();
- //merge partial data
- if (!$data && is_string($file) && isset($this->partial[$file])) {
- return $this->partial[$file];
- } elseif (!$data) {
- return $data;
- } else {
+ if ($data !== false) {
+ $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
+ } else {
+ //merge partial data
+ if (is_string($file) && isset($this->partial[$file])) {
+ return $this->partial[$file];
+ }
}
+
+ return false;
}
/**
@@ -190,8 +157,8 @@ class Cache implements ICache {
$data['path'] = (string)$data['path'];
$data['fileid'] = (int)$data['fileid'];
$data['parent'] = (int)$data['parent'];
- $data['size'] = 0 + $data['size'];
- $data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0);
+ $data['size'] = Util::numericToNumber($data['size']);
+ $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0);
$data['mtime'] = (int)$data['mtime'];
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encryptedVersion'] = (int)$data['encrypted'];
@@ -203,6 +170,9 @@ class Cache implements ICache {
if ($data['storage_mtime'] == 0) {
$data['storage_mtime'] = $data['mtime'];
}
+ if (isset($data['f_permissions'])) {
+ $data['scan_permissions'] = $data['f_permissions'];
+ }
$data['permissions'] = (int)$data['permissions'];
if (isset($data['creation_time'])) {
$data['creation_time'] = (int)$data['creation_time'];
@@ -235,13 +205,17 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->selectFileCache()
->whereParent($fileId)
+ ->whereStorageId($this->getNumericStorageId())
->orderBy('name', 'ASC');
- $result = $query->execute();
+ $metadataQuery = $query->selectMetadata();
+
+ $result = $query->executeQuery();
$files = $result->fetchAll();
$result->closeCursor();
- return array_map(function (array $data) {
+ return array_map(function (array $data) use ($metadataQuery) {
+ $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
}, $files);
}
@@ -280,14 +254,14 @@ class Cache implements ICache {
$file = $this->normalize($file);
if (isset($this->partial[$file])) { //add any saved partial data
- $data = array_merge($this->partial[$file], $data);
+ $data = array_merge($this->partial[$file]->getData(), $data);
unset($this->partial[$file]);
}
$requiredFields = ['size', 'mtime', 'mimetype'];
foreach ($requiredFields as $field) {
if (!isset($data[$field])) { //data not complete save as partial and return
- $this->partial[$file] = $data;
+ $this->partial[$file] = new CacheEntry($data);
return -1;
}
}
@@ -296,6 +270,9 @@ class Cache implements ICache {
if (!isset($data['parent'])) {
$data['parent'] = $this->getParentId($file);
}
+ if ($data['parent'] === -1 && $file !== '') {
+ throw new \Exception('Parent folder not in filecache for ' . $file);
+ }
$data['name'] = basename($file);
[$values, $extensionValues] = $this->normalizeData($data);
@@ -316,12 +293,13 @@ class Cache implements ICache {
if (count($extensionValues)) {
$query = $this->getQueryBuilder();
$query->insert('filecache_extended');
+ $query->hintShardKey('storage', $storageId);
$query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
foreach ($extensionValues as $column => $value) {
$query->setValue($column, $query->createNamedParameter($value));
}
- $query->execute();
+ $query->executeStatement();
}
$event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId);
@@ -370,6 +348,7 @@ class Cache implements ICache {
$query->update('filecache')
->whereFileId($id)
+ ->whereStorageId($this->getNumericStorageId())
->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
return $query->expr()->orX(
$query->expr()->neq($key, $query->createNamedParameter($value)),
@@ -381,13 +360,14 @@ class Cache implements ICache {
$query->set($key, $query->createNamedParameter($value));
}
- $query->execute();
+ $query->executeStatement();
}
if (count($extensionValues)) {
try {
$query = $this->getQueryBuilder();
$query->insert('filecache_extended');
+ $query->hintShardKey('storage', $this->getNumericStorageId());
$query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT));
foreach ($extensionValues as $column => $value) {
@@ -399,6 +379,7 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->update('filecache_extended')
->whereFileId($id)
+ ->hintShardKey('storage', $this->getNumericStorageId())
->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
return $query->expr()->orX(
$query->expr()->neq($key, $query->createNamedParameter($value)),
@@ -410,7 +391,7 @@ class Cache implements ICache {
$query->set($key, $query->createNamedParameter($value));
}
- $query->execute();
+ $query->executeStatement();
}
}
@@ -445,7 +426,7 @@ class Cache implements ICache {
$params = [];
$extensionParams = [];
foreach ($data as $name => $value) {
- if (array_search($name, $fields) !== false) {
+ if (in_array($name, $fields)) {
if ($name === 'path') {
$params['path_hash'] = md5($value);
} elseif ($name === 'mimetype') {
@@ -465,7 +446,7 @@ class Cache implements ICache {
}
$params[$name] = $value;
}
- if (array_search($name, $extensionFields) !== false) {
+ if (in_array($name, $extensionFields)) {
$extensionParams[$name] = $value;
}
}
@@ -492,7 +473,7 @@ class Cache implements ICache {
->whereStorageId($this->getNumericStorageId())
->wherePath($file);
- $result = $query->execute();
+ $result = $query->executeQuery();
$id = $result->fetchOne();
$result->closeCursor();
@@ -545,13 +526,15 @@ class Cache implements ICache {
if ($entry instanceof ICacheEntry) {
$query = $this->getQueryBuilder();
$query->delete('filecache')
+ ->whereStorageId($this->getNumericStorageId())
->whereFileId($entry->getId());
- $query->execute();
+ $query->executeStatement();
$query = $this->getQueryBuilder();
$query->delete('filecache_extended')
- ->whereFileId($entry->getId());
- $query->execute();
+ ->whereFileId($entry->getId())
+ ->hintShardKey('storage', $this->getNumericStorageId());
+ $query->executeStatement();
if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
$this->removeChildren($entry);
@@ -562,20 +545,7 @@ class Cache implements ICache {
}
/**
- * Get all sub folders of a folder
- *
- * @param ICacheEntry $entry the cache entry of the folder to get the subfolders for
- * @return ICacheEntry[] the cache entries for the subfolders
- */
- private function getSubFolders(ICacheEntry $entry) {
- $children = $this->getFolderContentsById($entry->getId());
- return array_filter($children, function ($child) {
- return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER;
- });
- }
-
- /**
- * Recursively remove all children of a folder
+ * Remove all children of a folder
*
* @param ICacheEntry $entry the cache entry of the folder to remove the children of
* @throws \OC\DatabaseException
@@ -583,6 +553,8 @@ class Cache implements ICache {
private function removeChildren(ICacheEntry $entry) {
$parentIds = [$entry->getId()];
$queue = [$entry->getId()];
+ $deletedIds = [];
+ $deletedPaths = [];
// we walk depth first through the file tree, removing all filecache_extended attributes while we walk
// and collecting all folder ids to later use to delete the filecache entries
@@ -591,20 +563,35 @@ class Cache implements ICache {
$childIds = array_map(function (ICacheEntry $cacheEntry) {
return $cacheEntry->getId();
}, $children);
+ $childPaths = array_map(function (ICacheEntry $cacheEntry) {
+ return $cacheEntry->getPath();
+ }, $children);
+
+ foreach ($childIds as $childId) {
+ $deletedIds[] = $childId;
+ }
+
+ foreach ($childPaths as $childPath) {
+ $deletedPaths[] = $childPath;
+ }
$query = $this->getQueryBuilder();
$query->delete('filecache_extended')
- ->where($query->expr()->in('fileid', $query->createParameter('childIds')));
+ ->where($query->expr()->in('fileid', $query->createParameter('childIds')))
+ ->hintShardKey('storage', $this->getNumericStorageId());
foreach (array_chunk($childIds, 1000) as $childIdChunk) {
$query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
- $query->execute();
+ $query->executeStatement();
}
/** @var ICacheEntry[] $childFolders */
- $childFolders = array_filter($children, function ($child) {
- return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER;
- });
+ $childFolders = [];
+ foreach ($children as $child) {
+ if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
+ $childFolders[] = $child;
+ }
+ }
foreach ($childFolders as $folder) {
$parentIds[] = $folder->getId();
$queue[] = $folder->getId();
@@ -613,11 +600,25 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->delete('filecache')
+ ->whereStorageId($this->getNumericStorageId())
->whereParentInParameter('parentIds');
+ // Sorting before chunking allows the db to find the entries close to each
+ // other in the index
+ sort($parentIds, SORT_NUMERIC);
foreach (array_chunk($parentIds, 1000) as $parentIdChunk) {
$query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
- $query->execute();
+ $query->executeStatement();
+ }
+
+ foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
+ $cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
+ $this->storage,
+ $filePath,
+ $fileId,
+ $this->getNumericStorageId()
+ );
+ $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
}
}
@@ -641,6 +642,10 @@ class Cache implements ICache {
return [$this->getNumericStorageId(), $path];
}
+ protected function hasEncryptionWrapper(): bool {
+ return $this->storage->instanceOfStorage(Encryption::class);
+ }
+
/**
* Move a file or folder in the cache
*
@@ -657,8 +662,17 @@ class Cache implements ICache {
$targetPath = $this->normalize($targetPath);
$sourceData = $sourceCache->get($sourcePath);
- if ($sourceData === false) {
- throw new \Exception('Invalid source storage path: ' . $sourcePath);
+ if (!$sourceData) {
+ throw new \Exception('Source path not found in cache: ' . $sourcePath);
+ }
+
+ $shardDefinition = $this->connection->getShardDefinition('filecache');
+ if (
+ $shardDefinition
+ && $shardDefinition->getShardForKey($sourceCache->getNumericStorageId()) !== $shardDefinition->getShardForKey($this->getNumericStorageId())
+ ) {
+ $this->moveFromStorageSharded($shardDefinition, $sourceCache, $sourceData, $targetPath);
+ return;
}
$sourceId = $sourceData['fileid'];
@@ -674,11 +688,15 @@ class Cache implements ICache {
throw new \Exception('Invalid target storage id: ' . $targetStorageId);
}
- $this->connection->beginTransaction();
if ($sourceData['mimetype'] === 'httpd/unix-directory') {
//update all child entries
$sourceLength = mb_strlen($sourcePath);
- $query = $this->connection->getQueryBuilder();
+
+ $childIds = $this->getChildIds($sourceStorageId, $sourcePath);
+
+ $childChunks = array_chunk($childIds, 1000);
+
+ $query = $this->getQueryBuilder();
$fun = $query->func();
$newPathFunction = $fun->concat(
@@ -686,29 +704,73 @@ class Cache implements ICache {
$fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash
);
$query->update('filecache')
- ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT))
->set('path_hash', $fun->md5($newPathFunction))
->set('path', $newPathFunction)
- ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT)))
- ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%')));
-
- try {
- $query->execute();
- } catch (\OC\DatabaseException $e) {
- $this->connection->rollBack();
- throw $e;
+ ->whereStorageId($sourceStorageId)
+ ->andWhere($query->expr()->in('fileid', $query->createParameter('files')));
+
+ if ($sourceStorageId !== $targetStorageId) {
+ $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT);
+ }
+
+ // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
+ if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
+ $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
}
+
+ // Retry transaction in case of RetryableException like deadlocks.
+ // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend
+ $retryLimit = 4;
+ for ($i = 1; $i <= $retryLimit; $i++) {
+ try {
+ $this->connection->beginTransaction();
+ foreach ($childChunks as $chunk) {
+ $query->setParameter('files', $chunk, IQueryBuilder::PARAM_INT_ARRAY);
+ $query->executeStatement();
+ }
+ break;
+ } catch (\OC\DatabaseException $e) {
+ $this->connection->rollBack();
+ throw $e;
+ } catch (DbalException $e) {
+ $this->connection->rollBack();
+
+ if (!$e->isRetryable()) {
+ throw $e;
+ }
+
+ // Simply throw if we already retried 4 times.
+ if ($i === $retryLimit) {
+ throw $e;
+ }
+
+ // Sleep a bit to give some time to the other transaction to finish.
+ usleep(100 * 1000 * $i);
+ }
+ }
+ } else {
+ $this->connection->beginTransaction();
}
$query = $this->getQueryBuilder();
$query->update('filecache')
- ->set('storage', $query->createNamedParameter($targetStorageId))
->set('path', $query->createNamedParameter($targetPath))
->set('path_hash', $query->createNamedParameter(md5($targetPath)))
->set('name', $query->createNamedParameter(basename($targetPath)))
->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT))
+ ->whereStorageId($sourceStorageId)
->whereFileId($sourceId);
- $query->execute();
+
+ if ($sourceStorageId !== $targetStorageId) {
+ $query->set('storage', $query->createNamedParameter($targetStorageId), IQueryBuilder::PARAM_INT);
+ }
+
+ // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
+ if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
+ $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
+ }
+
+ $query->executeStatement();
$this->connection->commit();
@@ -727,6 +789,15 @@ class Cache implements ICache {
}
}
+ private function getChildIds(int $storageId, string $path): array {
+ $query = $this->connection->getQueryBuilder();
+ $query->select('fileid')
+ ->from('filecache')
+ ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($path) . '/%')));
+ return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ }
+
/**
* remove all entries for files that are stored on the storage from the cache
*/
@@ -734,12 +805,12 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->delete('filecache')
->whereStorageId($this->getNumericStorageId());
- $query->execute();
+ $query->executeStatement();
$query = $this->connection->getQueryBuilder();
$query->delete('storages')
->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId)));
- $query->execute();
+ $query->executeStatement();
}
/**
@@ -764,7 +835,7 @@ class Cache implements ICache {
->whereStorageId($this->getNumericStorageId())
->wherePath($file);
- $result = $query->execute();
+ $result = $query->executeQuery();
$size = $result->fetchOne();
$result->closeCursor();
@@ -798,11 +869,11 @@ class Cache implements ICache {
* search for files by mimetype
*
* @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image')
- * where it will search for all mimetypes in the group ('image/*')
+ * where it will search for all mimetypes in the group ('image/*')
* @return ICacheEntry[] an array of cache entries where the mimetype matches the search
*/
public function searchByMime($mimetype) {
- if (strpos($mimetype, '/') === false) {
+ if (!str_contains($mimetype, '/')) {
$operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%');
} else {
$operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype);
@@ -810,26 +881,30 @@ class Cache implements ICache {
return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
}
- public function searchQuery(ISearchQuery $searchQuery) {
- return current($this->querySearchHelper->searchInCaches($searchQuery, [$this]));
+ public function searchQuery(ISearchQuery $query) {
+ return current($this->querySearchHelper->searchInCaches($query, [$this]));
}
/**
* Re-calculate the folder size and the size of all parent folders
*
- * @param string|boolean $path
- * @param array $data (optional) meta data of the folder
+ * @param array|ICacheEntry|null $data (optional) meta data of the folder
*/
- public function correctFolderSize($path, $data = null, $isBackgroundScan = false) {
+ public function correctFolderSize(string $path, $data = null, bool $isBackgroundScan = false): void {
$this->calculateFolderSize($path, $data);
+
if ($path !== '') {
$parent = dirname($path);
if ($parent === '.' || $parent === '/') {
$parent = '';
}
+
if ($isBackgroundScan) {
$parentData = $this->get($parent);
- if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) {
+ if ($parentData !== false
+ && $parentData['size'] !== -1
+ && $this->getIncompleteChildrenCount($parentData['fileid']) === 0
+ ) {
$this->correctFolderSize($parent, $parentData, $isBackgroundScan);
}
} else {
@@ -850,9 +925,10 @@ class Cache implements ICache {
$query->select($query->func()->count())
->from('filecache')
->whereParent($fileId)
- ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
+ ->whereStorageId($this->getNumericStorageId())
+ ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));
- $result = $query->execute();
+ $result = $query->executeQuery();
$size = (int)$result->fetchOne();
$result->closeCursor();
@@ -865,10 +941,23 @@ class Cache implements ICache {
* calculate the size of a folder and set it in the cache
*
* @param string $path
- * @param array $entry (optional) meta data of the folder
- * @return int
+ * @param array|null|ICacheEntry $entry (optional) meta data of the folder
+ * @return int|float
*/
public function calculateFolderSize($path, $entry = null) {
+ return $this->calculateFolderSizeInner($path, $entry);
+ }
+
+
+ /**
+ * inner function because we can't add new params to the public function without breaking any child classes
+ *
+ * @param string $path
+ * @param array|null|ICacheEntry $entry (optional) meta data of the folder
+ * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
+ * @return int|float
+ */
+ protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) {
$totalSize = 0;
if (is_null($entry) || !isset($entry['fileid'])) {
$entry = $this->get($path);
@@ -879,21 +968,25 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->select('size', 'unencrypted_size')
->from('filecache')
+ ->whereStorageId($this->getNumericStorageId())
->whereParent($id);
+ if ($ignoreUnknown) {
+ $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0)));
+ }
- $result = $query->execute();
+ $result = $query->executeQuery();
$rows = $result->fetchAll();
$result->closeCursor();
if ($rows) {
$sizes = array_map(function (array $row) {
- return (int)$row['size'];
+ return Util::numericToNumber($row['size']);
}, $rows);
$unencryptedOnlySizes = array_map(function (array $row) {
- return (int)$row['unencrypted_size'];
+ return Util::numericToNumber($row['unencrypted_size']);
}, $rows);
$unencryptedSizes = array_map(function (array $row) {
- return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
+ return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
}, $rows);
$sum = array_sum($sizes);
@@ -920,9 +1013,16 @@ class Cache implements ICache {
$unencryptedTotal = 0;
$unencryptedMax = 0;
}
- if ($entry['size'] !== $totalSize) {
- // only set unencrypted size for a folder if any child entries have it set, or the folder is empty
- if ($unencryptedMax > 0 || $totalSize === 0) {
+
+ // only set unencrypted size for a folder if any child entries have it set, or the folder is empty
+ $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0;
+ if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
+ if ($shouldWriteUnEncryptedSize) {
+ // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
+ if ($unencryptedMax === 0) {
+ $unencryptedTotal = 0;
+ }
+
$this->update($id, [
'size' => $totalSize,
'unencrypted_size' => $unencryptedTotal,
@@ -948,7 +1048,7 @@ class Cache implements ICache {
->from('filecache')
->whereStorageId($this->getNumericStorageId());
- $result = $query->execute();
+ $result = $query->executeQuery();
$files = $result->fetchAll(\PDO::FETCH_COLUMN);
$result->closeCursor();
@@ -971,20 +1071,15 @@ class Cache implements ICache {
$query->select('path')
->from('filecache')
->whereStorageId($this->getNumericStorageId())
- ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+ ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
->orderBy('fileid', 'DESC')
->setMaxResults(1);
- $result = $query->execute();
+ $result = $query->executeQuery();
$path = $result->fetchOne();
$result->closeCursor();
- if ($path === false) {
- return false;
- }
-
- // Make sure Oracle does not continue with null for empty strings
- return (string)$path;
+ return $path === false ? false : (string)$path;
}
/**
@@ -1000,7 +1095,7 @@ class Cache implements ICache {
->whereStorageId($this->getNumericStorageId())
->whereFileId($id);
- $result = $query->execute();
+ $result = $query->executeQuery();
$path = $result->fetchOne();
$result->closeCursor();
@@ -1018,7 +1113,7 @@ class Cache implements ICache {
*
* @param int $id
* @return array first element holding the storage id, second the path
- * @deprecated use getPathById() instead
+ * @deprecated 17.0.0 use getPathById() instead
*/
public static function getById($id) {
$query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
@@ -1026,7 +1121,7 @@ class Cache implements ICache {
->from('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
- $result = $query->execute();
+ $result = $query->executeQuery();
$row = $result->fetch();
$result->closeCursor();
@@ -1064,12 +1159,18 @@ class Cache implements ICache {
*/
public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int {
if ($sourceEntry->getId() < 0) {
- throw new \RuntimeException("Invalid source cache entry on copyFromCache");
+ throw new \RuntimeException('Invalid source cache entry on copyFromCache');
}
$data = $this->cacheEntryToArray($sourceEntry);
+
+ // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
+ if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
+ $data['encrypted'] = 0;
+ }
+
$fileId = $this->put($targetPath, $data);
if ($fileId <= 0) {
- throw new \RuntimeException("Failed to copy to " . $targetPath . " from cache with source data " . json_encode($data) . " ");
+ throw new \RuntimeException('Failed to copy to ' . $targetPath . ' from cache with source data ' . json_encode($data) . ' ');
}
if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
$folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId());
@@ -1082,7 +1183,7 @@ class Cache implements ICache {
}
private function cacheEntryToArray(ICacheEntry $entry): array {
- return [
+ $data = [
'size' => $entry->getSize(),
'mtime' => $entry->getMTime(),
'storage_mtime' => $entry->getStorageMTime(),
@@ -1095,6 +1196,10 @@ class Cache implements ICache {
'upload_time' => $entry->getUploadTime(),
'metadata_etag' => $entry->getMetadataEtag(),
];
+ if ($entry instanceof CacheEntry && isset($entry['scan_permissions'])) {
+ $data['permissions'] = $entry['scan_permissions'];
+ }
+ return $data;
}
public function getQueryFilterForStorage(): ISearchOperator {
@@ -1108,4 +1213,72 @@ class Cache implements ICache {
return null;
}
}
+
+ private function moveFromStorageSharded(ShardDefinition $shardDefinition, ICache $sourceCache, ICacheEntry $sourceEntry, $targetPath): void {
+ if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
+ $fileIds = $this->getChildIds($sourceCache->getNumericStorageId(), $sourceEntry->getPath());
+ } else {
+ $fileIds = [];
+ }
+ $fileIds[] = $sourceEntry->getId();
+
+ $helper = $this->connection->getCrossShardMoveHelper();
+
+ $sourceConnection = $helper->getConnection($shardDefinition, $sourceCache->getNumericStorageId());
+ $targetConnection = $helper->getConnection($shardDefinition, $this->getNumericStorageId());
+
+ $cacheItems = $helper->loadItems($sourceConnection, 'filecache', 'fileid', $fileIds);
+ $extendedItems = $helper->loadItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds);
+ $metadataItems = $helper->loadItems($sourceConnection, 'files_metadata', 'file_id', $fileIds);
+
+ // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
+ $removeEncryptedFlag = ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper()) && !$this->hasEncryptionWrapper();
+
+ $sourcePathLength = strlen($sourceEntry->getPath());
+ foreach ($cacheItems as &$cacheItem) {
+ if ($cacheItem['path'] === $sourceEntry->getPath()) {
+ $cacheItem['path'] = $targetPath;
+ $cacheItem['parent'] = $this->getParentId($targetPath);
+ $cacheItem['name'] = basename($cacheItem['path']);
+ } else {
+ $cacheItem['path'] = $targetPath . '/' . substr($cacheItem['path'], $sourcePathLength + 1); // +1 for the leading slash
+ }
+ $cacheItem['path_hash'] = md5($cacheItem['path']);
+ $cacheItem['storage'] = $this->getNumericStorageId();
+ if ($removeEncryptedFlag) {
+ $cacheItem['encrypted'] = 0;
+ }
+ }
+
+ $targetConnection->beginTransaction();
+
+ try {
+ $helper->saveItems($targetConnection, 'filecache', $cacheItems);
+ $helper->saveItems($targetConnection, 'filecache_extended', $extendedItems);
+ $helper->saveItems($targetConnection, 'files_metadata', $metadataItems);
+ } catch (\Exception $e) {
+ $targetConnection->rollback();
+ throw $e;
+ }
+
+ $sourceConnection->beginTransaction();
+
+ try {
+ $helper->deleteItems($sourceConnection, 'filecache', 'fileid', $fileIds);
+ $helper->deleteItems($sourceConnection, 'filecache_extended', 'fileid', $fileIds);
+ $helper->deleteItems($sourceConnection, 'files_metadata', 'file_id', $fileIds);
+ } catch (\Exception $e) {
+ $targetConnection->rollback();
+ $sourceConnection->rollBack();
+ throw $e;
+ }
+
+ try {
+ $sourceConnection->commit();
+ } catch (\Exception $e) {
+ $targetConnection->rollback();
+ throw $e;
+ }
+ $targetConnection->commit();
+ }
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-15 04:21:34 +0000