Signed-off-by: Robin Appelman <robin@icewind.nl>tags/v25.0.0beta1
@@ -37,6 +37,7 @@ | |||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||
* | |||
*/ | |||
namespace OC\Files\Cache; | |||
use Doctrine\DBAL\Exception\UniqueConstraintViolationException; | |||
@@ -188,6 +189,7 @@ class Cache implements ICache { | |||
$data['fileid'] = (int)$data['fileid']; | |||
$data['parent'] = (int)$data['parent']; | |||
$data['size'] = 0 + $data['size']; | |||
$data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0); | |||
$data['mtime'] = (int)$data['mtime']; | |||
$data['storage_mtime'] = (int)$data['storage_mtime']; | |||
$data['encryptedVersion'] = (int)$data['encrypted']; | |||
@@ -428,7 +430,7 @@ class Cache implements ICache { | |||
protected function normalizeData(array $data): array { | |||
$fields = [ | |||
'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', | |||
'etag', 'permissions', 'checksum', 'storage']; | |||
'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; | |||
$extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; | |||
$doNotCopyStorageMTime = false; | |||
@@ -873,8 +875,16 @@ class Cache implements ICache { | |||
$id = $entry['fileid']; | |||
$query = $this->getQueryBuilder(); | |||
$query->selectAlias($query->func()->sum('size'), 'f1') | |||
->selectAlias($query->func()->min('size'), 'f2') | |||
$query->selectAlias($query->func()->sum('size'), 'size_sum') | |||
->selectAlias($query->func()->min('size'), 'size_min') | |||
// in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size | |||
->selectAlias($query->func()->sum( | |||
$query->func()->case([ | |||
['when' => $query->expr()->eq('unencrypted_size', $query->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size'], | |||
], 'unencrypted_size') | |||
), 'unencrypted_sum') | |||
->selectAlias($query->func()->min('unencrypted_size'), 'unencrypted_min') | |||
->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max') | |||
->from('filecache') | |||
->whereStorageId($this->getNumericStorageId()) | |||
->whereParent($id); | |||
@@ -884,7 +894,7 @@ class Cache implements ICache { | |||
$result->closeCursor(); | |||
if ($row) { | |||
[$sum, $min] = array_values($row); | |||
['size_sum' => $sum, 'size_min' => $min, 'unencrypted_sum' => $unencryptedSum, 'unencrypted_min' => $unencryptedMin, 'unencrypted_max' => $unencryptedMax] = $row; | |||
$sum = 0 + $sum; | |||
$min = 0 + $min; | |||
if ($min === -1) { | |||
@@ -892,8 +902,23 @@ class Cache implements ICache { | |||
} else { | |||
$totalSize = $sum; | |||
} | |||
if ($unencryptedMin === -1 || $min === -1) { | |||
$unencryptedTotal = $unencryptedMin; | |||
} else { | |||
$unencryptedTotal = $unencryptedSum; | |||
} | |||
if ($entry['size'] !== $totalSize) { | |||
$this->update($id, ['size' => $totalSize]); | |||
// only set unencrypted size for a folder if any child entries have it set | |||
if ($unencryptedMax > 0) { | |||
$this->update($id, [ | |||
'size' => $totalSize, | |||
'unencrypted_size' => $unencryptedTotal, | |||
]); | |||
} else { | |||
$this->update($id, [ | |||
'size' => $totalSize, | |||
]); | |||
} | |||
} | |||
} | |||
} |
@@ -132,4 +132,12 @@ class CacheEntry implements ICacheEntry { | |||
public function __clone() { | |||
$this->data = array_merge([], $this->data); | |||
} | |||
public function getUnencryptedSize(): int { | |||
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { | |||
return $this->data['unencrypted_size']; | |||
} else { | |||
return $this->data['size']; | |||
} | |||
} | |||
} |
@@ -44,7 +44,7 @@ class CacheQueryBuilder extends QueryBuilder { | |||
public function selectFileCache(string $alias = null) { | |||
$name = $alias ? $alias : 'filecache'; | |||
$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime', | |||
'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time') | |||
'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size') | |||
->from('filecache', $name) | |||
->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); | |||
@@ -24,6 +24,7 @@ | |||
namespace OC\Files\Cache; | |||
use OC\Files\Storage\Wrapper\Encryption; | |||
use OCP\DB\QueryBuilder\IQueryBuilder; | |||
use OCP\Files\Cache\IPropagator; | |||
use OCP\Files\Storage\IReliableEtagStorage; | |||
@@ -113,6 +114,20 @@ class Propagator implements IPropagator { | |||
->andWhere($builder->expr()->in('path_hash', $hashParams)) | |||
->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT))); | |||
if ($this->storage->instanceOfStorage(Encryption::class)) { | |||
// in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size | |||
$builder->set('unencrypted_size', $builder->func()->greatest( | |||
$builder->func()->add( | |||
$builder->func()->case([ | |||
['when' => $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size'] | |||
], 'unencrypted_size'), | |||
$builder->createNamedParameter($sizeDifference) | |||
), | |||
$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) | |||
)); | |||
} | |||
$a = $builder->getSQL(); | |||
$builder->execute(); | |||
} | |||
} |
@@ -101,7 +101,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { | |||
$this->data = $data; | |||
$this->mount = $mount; | |||
$this->owner = $owner; | |||
$this->rawSize = $this->data['size'] ?? 0; | |||
if (isset($this->data['unencrypted_size'])) { | |||
$this->rawSize = $this->data['unencrypted_size']; | |||
} else { | |||
$this->rawSize = $this->data['size'] ?? 0; | |||
} | |||
} | |||
public function offsetSet($offset, $value): void { | |||
@@ -208,7 +212,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { | |||
public function getSize($includeMounts = true) { | |||
if ($includeMounts) { | |||
$this->updateEntryfromSubMounts(); | |||
return isset($this->data['size']) ? 0 + $this->data['size'] : 0; | |||
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { | |||
return $this->data['unencrypted_size']; | |||
} else { | |||
return isset($this->data['size']) ? 0 + $this->data['size'] : 0; | |||
} | |||
} else { | |||
return $this->rawSize; | |||
} | |||
@@ -386,7 +395,19 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { | |||
* @param string $entryPath full path of the child entry | |||
*/ | |||
public function addSubEntry($data, $entryPath) { | |||
$this->data['size'] += isset($data['size']) ? $data['size'] : 0; | |||
if (!$data) { | |||
return; | |||
} | |||
$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0; | |||
if ($hasUnencryptedSize) { | |||
$subSize = $data['unencrypted_size']; | |||
} else { | |||
$subSize = $data['size'] ?: 0; | |||
} | |||
$this->data['size'] += $subSize; | |||
if ($hasUnencryptedSize) { | |||
$this->data['unencrypted_size'] += $subSize; | |||
} | |||
if (isset($data['mtime'])) { | |||
$this->data['mtime'] = max($this->data['mtime'], $data['mtime']); | |||
} |
@@ -33,6 +33,7 @@ | |||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||
* | |||
*/ | |||
namespace OC\Files\Storage\Wrapper; | |||
use OC\Encryption\Exceptions\ModuleDoesNotExistsException; | |||
@@ -41,6 +42,7 @@ use OC\Encryption\Util; | |||
use OC\Files\Cache\CacheEntry; | |||
use OC\Files\Filesystem; | |||
use OC\Files\Mount\Manager; | |||
use OC\Files\ObjectStore\ObjectStoreStorage; | |||
use OC\Files\Storage\LocalTempFileTrait; | |||
use OC\Memcache\ArrayCache; | |||
use OCP\Encryption\Exceptions\GenericEncryptionException; | |||
@@ -139,28 +141,36 @@ class Encryption extends Wrapper { | |||
$size = $this->unencryptedSize[$fullPath]; | |||
// update file cache | |||
if ($info instanceof ICacheEntry) { | |||
$info = $info->getData(); | |||
$info['encrypted'] = $info['encryptedVersion']; | |||
} else { | |||
if (!is_array($info)) { | |||
$info = []; | |||
} | |||
$info['encrypted'] = true; | |||
$info = new CacheEntry($info); | |||
} | |||
$info['size'] = $size; | |||
$this->getCache()->put($path, $info); | |||
if ($size !== $info->getUnencryptedSize()) { | |||
$this->getCache()->update($info->getId(), [ | |||
'unencrypted_size' => $size | |||
]); | |||
} | |||
return $size; | |||
} | |||
if (isset($info['fileid']) && $info['encrypted']) { | |||
return $this->verifyUnencryptedSize($path, $info['size']); | |||
return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); | |||
} | |||
return $this->storage->filesize($path); | |||
} | |||
/** | |||
* @param string $path | |||
* @param array $data | |||
* @return array | |||
*/ | |||
private function modifyMetaData(string $path, array $data): array { | |||
$fullPath = $this->getFullPath($path); | |||
$info = $this->getCache()->get($path); | |||
@@ -170,7 +180,7 @@ class Encryption extends Wrapper { | |||
$data['size'] = $this->unencryptedSize[$fullPath]; | |||
} else { | |||
if (isset($info['fileid']) && $info['encrypted']) { | |||
$data['size'] = $this->verifyUnencryptedSize($path, $info['size']); | |||
$data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); | |||
$data['encrypted'] = true; | |||
} | |||
} | |||
@@ -478,7 +488,7 @@ class Encryption extends Wrapper { | |||
* | |||
* @return int unencrypted size | |||
*/ | |||
protected function verifyUnencryptedSize($path, $unencryptedSize) { | |||
protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int { | |||
$size = $this->storage->filesize($path); | |||
$result = $unencryptedSize; | |||
@@ -510,7 +520,7 @@ class Encryption extends Wrapper { | |||
* | |||
* @return int calculated unencrypted size | |||
*/ | |||
protected function fixUnencryptedSize($path, $size, $unencryptedSize) { | |||
protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int { | |||
$headerSize = $this->getHeaderSize($path); | |||
$header = $this->getHeader($path); | |||
$encryptionModule = $this->getEncryptionModule($path); | |||
@@ -581,7 +591,9 @@ class Encryption extends Wrapper { | |||
$cache = $this->storage->getCache(); | |||
if ($cache) { | |||
$entry = $cache->get($path); | |||
$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]); | |||
$cache->update($entry['fileid'], [ | |||
'unencrypted_size' => $newUnencryptedSize | |||
]); | |||
} | |||
return $newUnencryptedSize; | |||
@@ -621,7 +633,12 @@ class Encryption extends Wrapper { | |||
* @param bool $preserveMtime | |||
* @return bool | |||
*/ | |||
public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) { | |||
public function moveFromStorage( | |||
Storage\IStorage $sourceStorage, | |||
$sourceInternalPath, | |||
$targetInternalPath, | |||
$preserveMtime = true | |||
) { | |||
if ($sourceStorage === $this) { | |||
return $this->rename($sourceInternalPath, $targetInternalPath); | |||
} | |||
@@ -656,7 +673,13 @@ class Encryption extends Wrapper { | |||
* @param bool $isRename | |||
* @return bool | |||
*/ | |||
public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) { | |||
public function copyFromStorage( | |||
Storage\IStorage $sourceStorage, | |||
$sourceInternalPath, | |||
$targetInternalPath, | |||
$preserveMtime = false, | |||
$isRename = false | |||
) { | |||
// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed: | |||
// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage | |||
@@ -676,7 +699,13 @@ class Encryption extends Wrapper { | |||
* @param bool $isRename | |||
* @param bool $keepEncryptionVersion | |||
*/ | |||
private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) { | |||
private function updateEncryptedVersion( | |||
Storage\IStorage $sourceStorage, | |||
$sourceInternalPath, | |||
$targetInternalPath, | |||
$isRename, | |||
$keepEncryptionVersion | |||
) { | |||
$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath); | |||
$cacheInformation = [ | |||
'encrypted' => $isEncrypted, | |||
@@ -725,7 +754,13 @@ class Encryption extends Wrapper { | |||
* @return bool | |||
* @throws \Exception | |||
*/ | |||
private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) { | |||
private function copyBetweenStorage( | |||
Storage\IStorage $sourceStorage, | |||
$sourceInternalPath, | |||
$targetInternalPath, | |||
$preserveMtime, | |||
$isRename | |||
) { | |||
// for versions we have nothing to do, because versions should always use the | |||
// key from the original file. Just create a 1:1 copy and done | |||
@@ -743,7 +778,7 @@ class Encryption extends Wrapper { | |||
if (isset($info['encrypted']) && $info['encrypted'] === true) { | |||
$this->updateUnencryptedSize( | |||
$this->getFullPath($targetInternalPath), | |||
$info['size'] | |||
$info->getUnencryptedSize() | |||
); | |||
} | |||
$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true); | |||
@@ -808,13 +843,6 @@ class Encryption extends Wrapper { | |||
return (bool)$result; | |||
} | |||
/** | |||
* get the path to a local version of the file. | |||
* The local version of the file can be temporary and doesn't have to be persistent across requests | |||
* | |||
* @param string $path | |||
* @return string | |||
*/ | |||
public function getLocalFile($path) { | |||
if ($this->encryptionManager->isEnabled()) { | |||
$cachedFile = $this->getCachedFile($path); | |||
@@ -825,11 +853,6 @@ class Encryption extends Wrapper { | |||
return $this->storage->getLocalFile($path); | |||
} | |||
/** | |||
* Returns the wrapped storage's value for isLocal() | |||
* | |||
* @return bool wrapped storage's isLocal() value | |||
*/ | |||
public function isLocal() { | |||
if ($this->encryptionManager->isEnabled()) { | |||
return false; | |||
@@ -837,15 +860,11 @@ class Encryption extends Wrapper { | |||
return $this->storage->isLocal(); | |||
} | |||
/** | |||
* see https://www.php.net/manual/en/function.stat.php | |||
* only the following keys are required in the result: size and mtime | |||
* | |||
* @param string $path | |||
* @return array | |||
*/ | |||
public function stat($path) { | |||
$stat = $this->storage->stat($path); | |||
if (!$stat) { | |||
return false; | |||
} | |||
$fileSize = $this->filesize($path); | |||
$stat['size'] = $fileSize; | |||
$stat[7] = $fileSize; | |||
@@ -853,14 +872,6 @@ class Encryption extends Wrapper { | |||
return $stat; | |||
} | |||
/** | |||
* see https://www.php.net/manual/en/function.hash.php | |||
* | |||
* @param string $type | |||
* @param string $path | |||
* @param bool $raw | |||
* @return string | |||
*/ | |||
public function hash($type, $path, $raw = false) { | |||
$fh = $this->fopen($path, 'rb'); | |||
$ctx = hash_init($type); | |||
@@ -1068,6 +1079,13 @@ class Encryption extends Wrapper { | |||
[$count, $result] = \OC_Helper::streamCopy($stream, $target); | |||
fclose($stream); | |||
fclose($target); | |||
// object store, stores the size after write and doesn't update this during scan | |||
// manually store the unencrypted size | |||
if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) { | |||
$this->getCache()->put($path, ['unencrypted_size' => $count]); | |||
} | |||
return $count; | |||
} | |||
} |
@@ -162,4 +162,14 @@ interface ICacheEntry extends ArrayAccess { | |||
* @since 18.0.0 | |||
*/ | |||
public function getUploadTime(): ?int; | |||
/** | |||
* Get the unencrypted size | |||
* | |||
* This might be different from the result of getSize | |||
* | |||
* @return int | |||
* @since 25.0.0 | |||
*/ | |||
public function getUnencryptedSize(): int; | |||
} |
@@ -5,6 +5,7 @@ namespace Test\Files\Storage\Wrapper; | |||
use OC\Encryption\Exceptions\ModuleDoesNotExistsException; | |||
use OC\Encryption\Update; | |||
use OC\Encryption\Util; | |||
use OC\Files\Cache\CacheEntry; | |||
use OC\Files\Storage\Temporary; | |||
use OC\Files\Storage\Wrapper\Encryption; | |||
use OC\Files\View; | |||
@@ -259,7 +260,7 @@ class EncryptionTest extends Storage { | |||
->method('get') | |||
->willReturnCallback( | |||
function ($path) use ($encrypted) { | |||
return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]; | |||
return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]); | |||
} | |||
); | |||
@@ -332,7 +333,7 @@ class EncryptionTest extends Storage { | |||
->disableOriginalConstructor()->getMock(); | |||
$cache->expects($this->any()) | |||
->method('get') | |||
->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]); | |||
->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1])); | |||
$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption') | |||
->setConstructorArgs( | |||
@@ -910,7 +911,7 @@ class EncryptionTest extends Storage { | |||
if ($copyResult) { | |||
$cache->expects($this->once())->method('get') | |||
->with($sourceInternalPath) | |||
->willReturn(['encrypted' => $encrypted, 'size' => 42]); | |||
->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42])); | |||
if ($encrypted) { | |||
$instance->expects($this->once())->method('updateUnencryptedSize') | |||
->with($mountPoint . $targetInternalPath, 42); |
@@ -104,6 +104,9 @@ class HelperStorageTest extends \Test\TestCase { | |||
$extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq'); | |||
$extStorage->getScanner()->scan(''); // update root size | |||
$config = \OC::$server->getConfig(); | |||
$config->setSystemValue('quota_include_external_storage', false); | |||
\OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext'); | |||
$storageInfo = \OC_Helper::getStorageInfo(''); |