* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
+
namespace OC\Files\Cache;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
$data['fileid'] = (int)$data['fileid'];
$data['parent'] = (int)$data['parent'];
$data['size'] = 0 + $data['size'];
+ $data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0);
$data['mtime'] = (int)$data['mtime'];
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encryptedVersion'] = (int)$data['encrypted'];
protected function normalizeData(array $data): array {
$fields = [
'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
- 'etag', 'permissions', 'checksum', 'storage'];
+ 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
$doNotCopyStorageMTime = false;
$id = $entry['fileid'];
$query = $this->getQueryBuilder();
- $query->selectAlias($query->func()->sum('size'), 'f1')
- ->selectAlias($query->func()->min('size'), 'f2')
+ $query->selectAlias($query->func()->sum('size'), 'size_sum')
+ ->selectAlias($query->func()->min('size'), 'size_min')
+ // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
+ ->selectAlias($query->func()->sum(
+ $query->func()->case([
+ ['when' => $query->expr()->eq('unencrypted_size', $query->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size'],
+ ], 'unencrypted_size')
+ ), 'unencrypted_sum')
+ ->selectAlias($query->func()->min('unencrypted_size'), 'unencrypted_min')
+ ->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max')
->from('filecache')
->whereStorageId($this->getNumericStorageId())
->whereParent($id);
$result->closeCursor();
if ($row) {
- [$sum, $min] = array_values($row);
+ ['size_sum' => $sum, 'size_min' => $min, 'unencrypted_sum' => $unencryptedSum, 'unencrypted_min' => $unencryptedMin, 'unencrypted_max' => $unencryptedMax] = $row;
$sum = 0 + $sum;
$min = 0 + $min;
if ($min === -1) {
} else {
$totalSize = $sum;
}
+ if ($unencryptedMin === -1 || $min === -1) {
+ $unencryptedTotal = $unencryptedMin;
+ } else {
+ $unencryptedTotal = $unencryptedSum;
+ }
if ($entry['size'] !== $totalSize) {
- $this->update($id, ['size' => $totalSize]);
+ // only set unencrypted size for a folder if any child entries have it set
+ if ($unencryptedMax > 0) {
+ $this->update($id, [
+ 'size' => $totalSize,
+ 'unencrypted_size' => $unencryptedTotal,
+ ]);
+ } else {
+ $this->update($id, [
+ 'size' => $totalSize,
+ ]);
+ }
}
}
}
public function __clone() {
$this->data = array_merge([], $this->data);
}
+
+ public function getUnencryptedSize(): int {
+ if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+ return $this->data['unencrypted_size'];
+ } else {
+ return $this->data['size'];
+ }
+ }
}
public function selectFileCache(string $alias = null) {
$name = $alias ? $alias : 'filecache';
$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", "$name.name", 'mimetype', 'mimepart', 'size', 'mtime',
- 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time')
+ 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size')
->from('filecache', $name)
->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid'));
namespace OC\Files\Cache;
+use OC\Files\Storage\Wrapper\Encryption;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\IPropagator;
use OCP\Files\Storage\IReliableEtagStorage;
->andWhere($builder->expr()->in('path_hash', $hashParams))
->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
+ if ($this->storage->instanceOfStorage(Encryption::class)) {
+ // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
+ $builder->set('unencrypted_size', $builder->func()->greatest(
+ $builder->func()->add(
+ $builder->func()->case([
+ ['when' => $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)), 'then' => 'size']
+ ], 'unencrypted_size'),
+ $builder->createNamedParameter($sizeDifference)
+ ),
+ $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
+ ));
+ }
+
+ $a = $builder->getSQL();
$builder->execute();
}
}
$this->data = $data;
$this->mount = $mount;
$this->owner = $owner;
- $this->rawSize = $this->data['size'] ?? 0;
+ if (isset($this->data['unencrypted_size'])) {
+ $this->rawSize = $this->data['unencrypted_size'];
+ } else {
+ $this->rawSize = $this->data['size'] ?? 0;
+ }
}
public function offsetSet($offset, $value): void {
public function getSize($includeMounts = true) {
if ($includeMounts) {
$this->updateEntryfromSubMounts();
- return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+
+ if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+ return $this->data['unencrypted_size'];
+ } else {
+ return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+ }
} else {
return $this->rawSize;
}
* @param string $entryPath full path of the child entry
*/
public function addSubEntry($data, $entryPath) {
- $this->data['size'] += isset($data['size']) ? $data['size'] : 0;
+ if (!$data) {
+ return;
+ }
+ $hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
+ if ($hasUnencryptedSize) {
+ $subSize = $data['unencrypted_size'];
+ } else {
+ $subSize = $data['size'] ?: 0;
+ }
+ $this->data['size'] += $subSize;
+ if ($hasUnencryptedSize) {
+ $this->data['unencrypted_size'] += $subSize;
+ }
if (isset($data['mtime'])) {
$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
}
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
+
namespace OC\Files\Storage\Wrapper;
use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
use OC\Files\Cache\CacheEntry;
use OC\Files\Filesystem;
use OC\Files\Mount\Manager;
+use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\Storage\LocalTempFileTrait;
use OC\Memcache\ArrayCache;
use OCP\Encryption\Exceptions\GenericEncryptionException;
$size = $this->unencryptedSize[$fullPath];
// update file cache
if ($info instanceof ICacheEntry) {
- $info = $info->getData();
$info['encrypted'] = $info['encryptedVersion'];
} else {
if (!is_array($info)) {
$info = [];
}
$info['encrypted'] = true;
+ $info = new CacheEntry($info);
}
- $info['size'] = $size;
- $this->getCache()->put($path, $info);
+ if ($size !== $info->getUnencryptedSize()) {
+ $this->getCache()->update($info->getId(), [
+ 'unencrypted_size' => $size
+ ]);
+ }
return $size;
}
if (isset($info['fileid']) && $info['encrypted']) {
- return $this->verifyUnencryptedSize($path, $info['size']);
+ return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
}
return $this->storage->filesize($path);
}
+ /**
+ * @param string $path
+ * @param array $data
+ * @return array
+ */
private function modifyMetaData(string $path, array $data): array {
$fullPath = $this->getFullPath($path);
$info = $this->getCache()->get($path);
$data['size'] = $this->unencryptedSize[$fullPath];
} else {
if (isset($info['fileid']) && $info['encrypted']) {
- $data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
+ $data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
$data['encrypted'] = true;
}
}
*
* @return int unencrypted size
*/
- protected function verifyUnencryptedSize($path, $unencryptedSize) {
+ protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int {
$size = $this->storage->filesize($path);
$result = $unencryptedSize;
*
* @return int calculated unencrypted size
*/
- protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
+ protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int {
$headerSize = $this->getHeaderSize($path);
$header = $this->getHeader($path);
$encryptionModule = $this->getEncryptionModule($path);
$cache = $this->storage->getCache();
if ($cache) {
$entry = $cache->get($path);
- $cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
+ $cache->update($entry['fileid'], [
+ 'unencrypted_size' => $newUnencryptedSize
+ ]);
}
return $newUnencryptedSize;
* @param bool $preserveMtime
* @return bool
*/
- public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) {
+ public function moveFromStorage(
+ Storage\IStorage $sourceStorage,
+ $sourceInternalPath,
+ $targetInternalPath,
+ $preserveMtime = true
+ ) {
if ($sourceStorage === $this) {
return $this->rename($sourceInternalPath, $targetInternalPath);
}
* @param bool $isRename
* @return bool
*/
- public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) {
+ public function copyFromStorage(
+ Storage\IStorage $sourceStorage,
+ $sourceInternalPath,
+ $targetInternalPath,
+ $preserveMtime = false,
+ $isRename = false
+ ) {
// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed:
// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage
* @param bool $isRename
* @param bool $keepEncryptionVersion
*/
- private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) {
+ private function updateEncryptedVersion(
+ Storage\IStorage $sourceStorage,
+ $sourceInternalPath,
+ $targetInternalPath,
+ $isRename,
+ $keepEncryptionVersion
+ ) {
$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath);
$cacheInformation = [
'encrypted' => $isEncrypted,
* @return bool
* @throws \Exception
*/
- private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) {
+ private function copyBetweenStorage(
+ Storage\IStorage $sourceStorage,
+ $sourceInternalPath,
+ $targetInternalPath,
+ $preserveMtime,
+ $isRename
+ ) {
// for versions we have nothing to do, because versions should always use the
// key from the original file. Just create a 1:1 copy and done
if (isset($info['encrypted']) && $info['encrypted'] === true) {
$this->updateUnencryptedSize(
$this->getFullPath($targetInternalPath),
- $info['size']
+ $info->getUnencryptedSize()
);
}
$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true);
return (bool)$result;
}
- /**
- * get the path to a local version of the file.
- * The local version of the file can be temporary and doesn't have to be persistent across requests
- *
- * @param string $path
- * @return string
- */
public function getLocalFile($path) {
if ($this->encryptionManager->isEnabled()) {
$cachedFile = $this->getCachedFile($path);
return $this->storage->getLocalFile($path);
}
- /**
- * Returns the wrapped storage's value for isLocal()
- *
- * @return bool wrapped storage's isLocal() value
- */
public function isLocal() {
if ($this->encryptionManager->isEnabled()) {
return false;
return $this->storage->isLocal();
}
- /**
- * see https://www.php.net/manual/en/function.stat.php
- * only the following keys are required in the result: size and mtime
- *
- * @param string $path
- * @return array
- */
public function stat($path) {
$stat = $this->storage->stat($path);
+ if (!$stat) {
+ return false;
+ }
$fileSize = $this->filesize($path);
$stat['size'] = $fileSize;
$stat[7] = $fileSize;
return $stat;
}
- /**
- * see https://www.php.net/manual/en/function.hash.php
- *
- * @param string $type
- * @param string $path
- * @param bool $raw
- * @return string
- */
public function hash($type, $path, $raw = false) {
$fh = $this->fopen($path, 'rb');
$ctx = hash_init($type);
[$count, $result] = \OC_Helper::streamCopy($stream, $target);
fclose($stream);
fclose($target);
+
+ // object store, stores the size after write and doesn't update this during scan
+ // manually store the unencrypted size
+ if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) {
+ $this->getCache()->put($path, ['unencrypted_size' => $count]);
+ }
+
return $count;
}
}
* @since 18.0.0
*/
public function getUploadTime(): ?int;
+
+ /**
+ * Get the unencrypted size
+ *
+ * This might be different from the result of getSize
+ *
+ * @return int
+ * @since 25.0.0
+ */
+ public function getUnencryptedSize(): int;
}
use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
use OC\Encryption\Update;
use OC\Encryption\Util;
+use OC\Files\Cache\CacheEntry;
use OC\Files\Storage\Temporary;
use OC\Files\Storage\Wrapper\Encryption;
use OC\Files\View;
->method('get')
->willReturnCallback(
function ($path) use ($encrypted) {
- return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1];
+ return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]);
}
);
->disableOriginalConstructor()->getMock();
$cache->expects($this->any())
->method('get')
- ->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]);
+ ->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]));
$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
->setConstructorArgs(
if ($copyResult) {
$cache->expects($this->once())->method('get')
->with($sourceInternalPath)
- ->willReturn(['encrypted' => $encrypted, 'size' => 42]);
+ ->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42]));
if ($encrypted) {
$instance->expects($this->once())->method('updateUnencryptedSize')
->with($mountPoint . $targetInternalPath, 42);
$extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq');
$extStorage->getScanner()->scan(''); // update root size
+ $config = \OC::$server->getConfig();
+ $config->setSystemValue('quota_include_external_storage', false);
+
\OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext');
$storageInfo = \OC_Helper::getStorageInfo('');
projects / nextcloud-server.git / commitdiff