From 8238582e59b7b6ec03318bcf81bf47cce54af320 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 13 Apr 2022 16:05:45 +0200 Subject: [PATCH] store unencrypted size in the unencrypted_size column Signed-off-by: Robin Appelman --- lib/private/Files/Cache/Cache.php | 35 ++++++- lib/private/Files/Cache/CacheEntry.php | 8 ++ lib/private/Files/Cache/CacheQueryBuilder.php | 2 +- lib/private/Files/Cache/Propagator.php | 15 +++ lib/private/Files/FileInfo.php | 27 ++++- .../Files/Storage/Wrapper/Encryption.php | 98 +++++++++++-------- lib/public/Files/Cache/ICacheEntry.php | 10 ++ .../Files/Storage/Wrapper/EncryptionTest.php | 7 +- tests/lib/HelperStorageTest.php | 3 + 9 files changed, 153 insertions(+), 52 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 949079dfa22..c6cadf14f86 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -37,6 +37,7 @@ * along with this program. If not, see * */ + namespace OC\Files\Cache; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; @@ -188,6 +189,7 @@ class Cache implements ICache { $data['fileid'] = (int)$data['fileid']; $data['parent'] = (int)$data['parent']; $data['size'] = 0 + $data['size']; + $data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0); $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; $data['encryptedVersion'] = (int)$data['encrypted']; @@ -428,7 +430,7 @@ class Cache implements ICache { protected function normalizeData(array $data): array { $fields = [ 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', - 'etag', 'permissions', 'checksum', 'storage']; + 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; $extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; $doNotCopyStorageMTime = false; @@ -873,8 +875,16 @@ class Cache implements ICache { $id = $entry['fileid']; $query = $this->getQueryBuilder(); - $query->selectAlias($query->func()->sum('size'), 'f1') - ->selectAlias($query->func()->min('size'), 'f2') + $query->selectAlias($query->func()->sum('size'), 'size_sum') + ->selectAlias($query->func()->min('size'), 'size_min') + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + ->selectAlias($query->func()->sum( + $query->func()->case([ + ['when' => $query->expr()->eq('unencrypted_size', $query->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size'], + ], 'unencrypted_size') + ), 'unencrypted_sum') + ->selectAlias($query->func()->min('unencrypted_size'), 'unencrypted_min') + ->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max') ->from('filecache') ->whereStorageId($this->getNumericStorageId()) ->whereParent($id); @@ -884,7 +894,7 @@ class Cache implements ICache { $result->closeCursor(); if ($row) { - [$sum, $min] = array_values($row); + ['size_sum' => $sum, 'size_min' => $min, 'unencrypted_sum' => $unencryptedSum, 'unencrypted_min' => $unencryptedMin, 'unencrypted_max' => $unencryptedMax] = $row; $sum = 0 + $sum; $min = 0 + $min; if ($min === -1) { @@ -892,8 +902,23 @@ class Cache implements ICache { } else { $totalSize = $sum; } + if ($unencryptedMin === -1 || $min === -1) { + $unencryptedTotal = $unencryptedMin; + } else { + $unencryptedTotal = $unencryptedSum; + } if ($entry['size'] !== $totalSize) { - $this->update($id, ['size' => $totalSize]); + // only set unencrypted size for a folder if any child entries have it set + if ($unencryptedMax > 0) { + $this->update($id, [ + 'size' => $totalSize, + 'unencrypted_size' => $unencryptedTotal, + ]); + } else { + $this->update($id, [ + 'size' => $totalSize, + ]); + } } } } diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index 12f0273fb6e..8ac76acf6d1 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -132,4 +132,12 @@ class CacheEntry implements ICacheEntry { public function __clone() { $this->data = array_merge([], $this->data); } + + public function getUnencryptedSize(): int { + if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return $this->data['size']; + } + } } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index b448424c1a8..b92e79aa229 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -44,7 +44,7 @@ class CacheQueryBuilder extends QueryBuilder { public function selectFileCache(string $alias = null) { $name = $alias ? $alias : 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', - 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time') + 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size') ->from('filecache', $name) ->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index 270b2b013f5..afff2fb51ff 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -24,6 +24,7 @@ namespace OC\Files\Cache; +use OC\Files\Storage\Wrapper\Encryption; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IPropagator; use OCP\Files\Storage\IReliableEtagStorage; @@ -113,6 +114,20 @@ class Propagator implements IPropagator { ->andWhere($builder->expr()->in('path_hash', $hashParams)) ->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + if ($this->storage->instanceOfStorage(Encryption::class)) { + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + $builder->set('unencrypted_size', $builder->func()->greatest( + $builder->func()->add( + $builder->func()->case([ + ['when' => $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size'] + ], 'unencrypted_size'), + $builder->createNamedParameter($sizeDifference) + ), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + )); + } + + $a = $builder->getSQL(); $builder->execute(); } } diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 6389544184f..5912eefcf9c 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -101,7 +101,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { $this->data = $data; $this->mount = $mount; $this->owner = $owner; - $this->rawSize = $this->data['size'] ?? 0; + if (isset($this->data['unencrypted_size'])) { + $this->rawSize = $this->data['unencrypted_size']; + } else { + $this->rawSize = $this->data['size'] ?? 0; + } } public function offsetSet($offset, $value): void { @@ -208,7 +212,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { public function getSize($includeMounts = true) { if ($includeMounts) { $this->updateEntryfromSubMounts(); - return isset($this->data['size']) ? 0 + $this->data['size'] : 0; + + if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return isset($this->data['size']) ? 0 + $this->data['size'] : 0; + } } else { return $this->rawSize; } @@ -386,7 +395,19 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @param string $entryPath full path of the child entry */ public function addSubEntry($data, $entryPath) { - $this->data['size'] += isset($data['size']) ? $data['size'] : 0; + if (!$data) { + return; + } + $hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0; + if ($hasUnencryptedSize) { + $subSize = $data['unencrypted_size']; + } else { + $subSize = $data['size'] ?: 0; + } + $this->data['size'] += $subSize; + if ($hasUnencryptedSize) { + $this->data['unencrypted_size'] += $subSize; + } if (isset($data['mtime'])) { $this->data['mtime'] = max($this->data['mtime'], $data['mtime']); } diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php index 4cfe932cc9f..d5bf929101f 100644 --- a/lib/private/Files/Storage/Wrapper/Encryption.php +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -33,6 +33,7 @@ * along with this program. If not, see * */ + namespace OC\Files\Storage\Wrapper; use OC\Encryption\Exceptions\ModuleDoesNotExistsException; @@ -41,6 +42,7 @@ use OC\Encryption\Util; use OC\Files\Cache\CacheEntry; use OC\Files\Filesystem; use OC\Files\Mount\Manager; +use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\Storage\LocalTempFileTrait; use OC\Memcache\ArrayCache; use OCP\Encryption\Exceptions\GenericEncryptionException; @@ -139,28 +141,36 @@ class Encryption extends Wrapper { $size = $this->unencryptedSize[$fullPath]; // update file cache if ($info instanceof ICacheEntry) { - $info = $info->getData(); $info['encrypted'] = $info['encryptedVersion']; } else { if (!is_array($info)) { $info = []; } $info['encrypted'] = true; + $info = new CacheEntry($info); } - $info['size'] = $size; - $this->getCache()->put($path, $info); + if ($size !== $info->getUnencryptedSize()) { + $this->getCache()->update($info->getId(), [ + 'unencrypted_size' => $size + ]); + } return $size; } if (isset($info['fileid']) && $info['encrypted']) { - return $this->verifyUnencryptedSize($path, $info['size']); + return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); } return $this->storage->filesize($path); } + /** + * @param string $path + * @param array $data + * @return array + */ private function modifyMetaData(string $path, array $data): array { $fullPath = $this->getFullPath($path); $info = $this->getCache()->get($path); @@ -170,7 +180,7 @@ class Encryption extends Wrapper { $data['size'] = $this->unencryptedSize[$fullPath]; } else { if (isset($info['fileid']) && $info['encrypted']) { - $data['size'] = $this->verifyUnencryptedSize($path, $info['size']); + $data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); $data['encrypted'] = true; } } @@ -478,7 +488,7 @@ class Encryption extends Wrapper { * * @return int unencrypted size */ - protected function verifyUnencryptedSize($path, $unencryptedSize) { + protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int { $size = $this->storage->filesize($path); $result = $unencryptedSize; @@ -510,7 +520,7 @@ class Encryption extends Wrapper { * * @return int calculated unencrypted size */ - protected function fixUnencryptedSize($path, $size, $unencryptedSize) { + protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int { $headerSize = $this->getHeaderSize($path); $header = $this->getHeader($path); $encryptionModule = $this->getEncryptionModule($path); @@ -581,7 +591,9 @@ class Encryption extends Wrapper { $cache = $this->storage->getCache(); if ($cache) { $entry = $cache->get($path); - $cache->update($entry['fileid'], ['size' => $newUnencryptedSize]); + $cache->update($entry['fileid'], [ + 'unencrypted_size' => $newUnencryptedSize + ]); } return $newUnencryptedSize; @@ -621,7 +633,12 @@ class Encryption extends Wrapper { * @param bool $preserveMtime * @return bool */ - public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) { + public function moveFromStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime = true + ) { if ($sourceStorage === $this) { return $this->rename($sourceInternalPath, $targetInternalPath); } @@ -656,7 +673,13 @@ class Encryption extends Wrapper { * @param bool $isRename * @return bool */ - public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) { + public function copyFromStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime = false, + $isRename = false + ) { // TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed: // - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage @@ -676,7 +699,13 @@ class Encryption extends Wrapper { * @param bool $isRename * @param bool $keepEncryptionVersion */ - private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) { + private function updateEncryptedVersion( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $isRename, + $keepEncryptionVersion + ) { $isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath); $cacheInformation = [ 'encrypted' => $isEncrypted, @@ -725,7 +754,13 @@ class Encryption extends Wrapper { * @return bool * @throws \Exception */ - private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) { + private function copyBetweenStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime, + $isRename + ) { // for versions we have nothing to do, because versions should always use the // key from the original file. Just create a 1:1 copy and done @@ -743,7 +778,7 @@ class Encryption extends Wrapper { if (isset($info['encrypted']) && $info['encrypted'] === true) { $this->updateUnencryptedSize( $this->getFullPath($targetInternalPath), - $info['size'] + $info->getUnencryptedSize() ); } $this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true); @@ -808,13 +843,6 @@ class Encryption extends Wrapper { return (bool)$result; } - /** - * get the path to a local version of the file. - * The local version of the file can be temporary and doesn't have to be persistent across requests - * - * @param string $path - * @return string - */ public function getLocalFile($path) { if ($this->encryptionManager->isEnabled()) { $cachedFile = $this->getCachedFile($path); @@ -825,11 +853,6 @@ class Encryption extends Wrapper { return $this->storage->getLocalFile($path); } - /** - * Returns the wrapped storage's value for isLocal() - * - * @return bool wrapped storage's isLocal() value - */ public function isLocal() { if ($this->encryptionManager->isEnabled()) { return false; @@ -837,15 +860,11 @@ class Encryption extends Wrapper { return $this->storage->isLocal(); } - /** - * see https://www.php.net/manual/en/function.stat.php - * only the following keys are required in the result: size and mtime - * - * @param string $path - * @return array - */ public function stat($path) { $stat = $this->storage->stat($path); + if (!$stat) { + return false; + } $fileSize = $this->filesize($path); $stat['size'] = $fileSize; $stat[7] = $fileSize; @@ -853,14 +872,6 @@ class Encryption extends Wrapper { return $stat; } - /** - * see https://www.php.net/manual/en/function.hash.php - * - * @param string $type - * @param string $path - * @param bool $raw - * @return string - */ public function hash($type, $path, $raw = false) { $fh = $this->fopen($path, 'rb'); $ctx = hash_init($type); @@ -1068,6 +1079,13 @@ class Encryption extends Wrapper { [$count, $result] = \OC_Helper::streamCopy($stream, $target); fclose($stream); fclose($target); + + // object store, stores the size after write and doesn't update this during scan + // manually store the unencrypted size + if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) { + $this->getCache()->put($path, ['unencrypted_size' => $count]); + } + return $count; } } diff --git a/lib/public/Files/Cache/ICacheEntry.php b/lib/public/Files/Cache/ICacheEntry.php index 17eecf89ddb..e1e8129394c 100644 --- a/lib/public/Files/Cache/ICacheEntry.php +++ b/lib/public/Files/Cache/ICacheEntry.php @@ -162,4 +162,14 @@ interface ICacheEntry extends ArrayAccess { * @since 18.0.0 */ public function getUploadTime(): ?int; + + /** + * Get the unencrypted size + * + * This might be different from the result of getSize + * + * @return int + * @since 25.0.0 + */ + public function getUnencryptedSize(): int; } diff --git a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php index d26e5c499e7..ebb97a25c77 100644 --- a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php +++ b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php @@ -5,6 +5,7 @@ namespace Test\Files\Storage\Wrapper; use OC\Encryption\Exceptions\ModuleDoesNotExistsException; use OC\Encryption\Update; use OC\Encryption\Util; +use OC\Files\Cache\CacheEntry; use OC\Files\Storage\Temporary; use OC\Files\Storage\Wrapper\Encryption; use OC\Files\View; @@ -259,7 +260,7 @@ class EncryptionTest extends Storage { ->method('get') ->willReturnCallback( function ($path) use ($encrypted) { - return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]; + return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]); } ); @@ -332,7 +333,7 @@ class EncryptionTest extends Storage { ->disableOriginalConstructor()->getMock(); $cache->expects($this->any()) ->method('get') - ->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]); + ->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1])); $this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption') ->setConstructorArgs( @@ -910,7 +911,7 @@ class EncryptionTest extends Storage { if ($copyResult) { $cache->expects($this->once())->method('get') ->with($sourceInternalPath) - ->willReturn(['encrypted' => $encrypted, 'size' => 42]); + ->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42])); if ($encrypted) { $instance->expects($this->once())->method('updateUnencryptedSize') ->with($mountPoint . $targetInternalPath, 42); diff --git a/tests/lib/HelperStorageTest.php b/tests/lib/HelperStorageTest.php index 6d7ea513d3f..d3f480502b2 100644 --- a/tests/lib/HelperStorageTest.php +++ b/tests/lib/HelperStorageTest.php @@ -104,6 +104,9 @@ class HelperStorageTest extends \Test\TestCase { $extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq'); $extStorage->getScanner()->scan(''); // update root size + $config = \OC::$server->getConfig(); + $config->setSystemValue('quota_include_external_storage', false); + \OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext'); $storageInfo = \OC_Helper::getStorageInfo(''); -- 2.39.5