Merge pull request #22627 from owncloud/fix_broken_unencrypted_size_8.1

[stable8.1] Heal unencrypted file sizes at download time

2 files changed, 286 insertions, 4 deletions

diff --git a/lib/private/files/storage/wrapper/encryption.php b/lib/private/files/storage/wrapper/encryption.php
index 4ba9b21ddb4..a1843980f98 100644
--- a/lib/private/files/storage/wrapper/encryption.php
+++ b/lib/private/files/storage/wrapper/encryption.php
@@ -59,7 +59,7 @@ class Encryption extends Wrapper {
 	private $uid;
 
 	/** @var array */
-	private $unencryptedSize;
+	protected $unencryptedSize;
 
 	/** @var \OCP\Encryption\IFile */
 	private $fileHelper;
@@ -77,6 +77,9 @@ class Encryption extends Wrapper {
 	/** @var Manager */
 	private $mountManager;
 
+	/** @var array remember for which path we execute the repair step to avoid recursions */
+	private $fixUnencryptedSizeOf = array();
+
 	/**
 	 * @param array $parameters
 	 * @param IManager $encryptionManager
@@ -136,8 +139,9 @@ class Encryption extends Wrapper {
 		}
 
 		if (isset($info['fileid']) && $info['encrypted']) {
-			return $info['size'];
+			return $this->verifyUnencryptedSize($path, $info['size']);
 		}
+
 		return $this->storage->filesize($path);
 	}
 
@@ -158,8 +162,8 @@ class Encryption extends Wrapper {
 		} else {
 			$info = $this->getCache()->get($path);
 			if (isset($info['fileid']) && $info['encrypted']) {
+				$data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
 				$data['encrypted'] = true;
-				$data['size'] = $info['size'];
 			}
 		}
 
@@ -431,6 +435,128 @@ class Encryption extends Wrapper {
 		return $this->storage->fopen($path, $mode);
 	}
 
+
+	/**
+	 * perform some plausibility checks if the the unencrypted size is correct.
+	 * If not, we calculate the correct unencrypted size and return it
+	 *
+	 * @param string $path internal path relative to the storage root
+	 * @param int $unencryptedSize size of the unencrypted file
+	 *
+	 * @return int unencrypted size
+	 */
+	protected function verifyUnencryptedSize($path, $unencryptedSize) {
+
+		$size = $this->storage->filesize($path);
+		$result = $unencryptedSize;
+
+		if ($unencryptedSize < 0 ||
+			($size > 0 && $unencryptedSize === $size)
+		) {
+			// check if we already calculate the unencrypted size for the
+			// given path to avoid recursions
+			if (isset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]) === false) {
+				$this->fixUnencryptedSizeOf[$this->getFullPath($path)] = true;
+				try {
+					$result = $this->fixUnencryptedSize($path, $size, $unencryptedSize);
+				} catch (\Exception $e) {
+					$this->logger->error('Couldn\'t re-calculate unencrypted size for '. $path);
+					$this->logger->error($e->getMessage());
+				}
+				unset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]);
+			}
+		}
+
+		return $result;
+	}
+
+	/**
+	 * calculate the unencrypted size
+	 *
+	 * @param string $path internal path relative to the storage root
+	 * @param int $size size of the physical file
+	 * @param int $unencryptedSize size of the unencrypted file
+	 *
+	 * @return int calculated unencrypted size
+	 */
+	protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
+
+		$headerSize = $this->getHeaderSize($path);
+		$header = $this->getHeader($path);
+		$encryptionModule = $this->getEncryptionModule($path);
+
+		$stream = $this->storage->fopen($path, 'r');
+
+		// if we couldn't open the file we return the old unencrypted size
+		if (!is_resource($stream)) {
+			$this->logger->error('Could not open ' . $path . '. Recalculation of unencrypted size aborted.');
+			return $unencryptedSize;
+		}
+
+		$newUnencryptedSize = 0;
+		$size -= $headerSize;
+		$blockSize = $this->util->getBlockSize();
+
+		// if a header exists we skip it
+		if ($headerSize > 0) {
+			fread($stream, $headerSize);
+		}
+
+		// fast path, else the calculation for $lastChunkNr is bogus
+		if ($size === 0) {
+			return 0;
+		}
+
+		$signed = (isset($header['signed']) && $header['signed'] === 'true') ? true : false;
+		$unencryptedBlockSize = $encryptionModule->getUnencryptedBlockSize($signed);
+
+		// calculate last chunk nr
+		// next highest is end of chunks, one subtracted is last one
+		// we have to read the last chunk, we can't just calculate it (because of padding etc)
+
+		$lastChunkNr = ceil($size/ $blockSize)-1;
+		// calculate last chunk position
+		$lastChunkPos = ($lastChunkNr * $blockSize);
+		// try to fseek to the last chunk, if it fails we have to read the whole file
+		if (@fseek($stream, $lastChunkPos, SEEK_CUR) === 0) {
+			$newUnencryptedSize += $lastChunkNr * $unencryptedBlockSize;
+		}
+
+		$lastChunkContentEncrypted='';
+		$count = $blockSize;
+
+		while ($count > 0) {
+			$data=fread($stream, $blockSize);
+			$count=strlen($data);
+			$lastChunkContentEncrypted .= $data;
+			if(strlen($lastChunkContentEncrypted) > $blockSize) {
+				$newUnencryptedSize += $unencryptedBlockSize;
+				$lastChunkContentEncrypted=substr($lastChunkContentEncrypted, $blockSize);
+			}
+		}
+
+		fclose($stream);
+
+		// we have to decrypt the last chunk to get it actual size
+		$encryptionModule->begin($this->getFullPath($path), $this->uid, 'r', $header, []);
+		$decryptedLastChunk = $encryptionModule->decrypt($lastChunkContentEncrypted);
+		$decryptedLastChunk .= $encryptionModule->end($this->getFullPath($path));
+
+		// calc the real file size with the size of the last chunk
+		$newUnencryptedSize += strlen($decryptedLastChunk);
+
+		$this->updateUnencryptedSize($this->getFullPath($path), $newUnencryptedSize);
+
+		// write to cache if applicable
+		$cache = $this->storage->getCache();
+		if ($cache) {
+			$entry = $cache->get($path);
+			$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
+		}
+
+		return $newUnencryptedSize;
+	}
+
 	/**
 	 * @param Storage $sourceStorage
 	 * @param string $sourceInternalPath
diff --git a/tests/lib/files/storage/wrapper/encryption.php b/tests/lib/files/storage/wrapper/encryption.php
index c49e6bb0d1f..c3ed0123f83 100644
--- a/tests/lib/files/storage/wrapper/encryption.php
+++ b/tests/lib/files/storage/wrapper/encryption.php
@@ -5,8 +5,9 @@ namespace Test\Files\Storage\Wrapper;
 use OC\Encryption\Util;
 use OC\Files\Storage\Temporary;
 use OC\Files\View;
+use Test\Files\Storage\Storage;
 
-class Encryption extends \Test\Files\Storage\Storage {
+class Encryption extends Storage {
 
 	/**
 	 * block size will always be 8192 for a PHP stream
@@ -211,6 +212,161 @@ class Encryption extends \Test\Files\Storage\Storage {
 	}
 
 	/**
+	 * @dataProvider dataTestGetMetaData
+	 *
+	 * @param string $path
+	 * @param array $metaData
+	 * @param bool $encrypted
+	 * @param bool $unencryptedSizeSet
+	 * @param int $storedUnencryptedSize
+	 * @param array $expected
+	 */
+	public function testGetMetaData($path, $metaData, $encrypted, $unencryptedSizeSet, $storedUnencryptedSize, $expected) {
+
+		$sourceStorage = $this->getMockBuilder('\OC\Files\Storage\Storage')
+			->disableOriginalConstructor()->getMock();
+
+		$cache = $this->getMockBuilder('\OC\Files\Cache\Cache')
+			->disableOriginalConstructor()->getMock();
+		$cache->expects($this->any())
+			->method('get')
+			->willReturnCallback(
+				function($path) use ($encrypted) {
+					return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1];
+				}
+			);
+
+		$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
+			->setConstructorArgs(
+				[
+					[
+						'storage' => $sourceStorage,
+						'root' => 'foo',
+						'mountPoint' => '/',
+						'mount' => $this->mount
+					],
+					$this->encryptionManager, $this->util, $this->logger, $this->file, null, $this->keyStore, $this->update, $this->mountManager
+				]
+			)
+			->setMethods(['getCache', 'verifyUnencryptedSize'])
+			->getMock();
+
+		if($unencryptedSizeSet) {
+			$this->invokePrivate($this->instance, 'unencryptedSize', [[$path => $storedUnencryptedSize]]);
+		}
+
+
+		$sourceStorage->expects($this->once())->method('getMetaData')->with($path)
+			->willReturn($metaData);
+
+		$this->instance->expects($this->any())->method('getCache')->willReturn($cache);
+		$this->instance->expects($this->any())->method('verifyUnencryptedSize')
+			->with($path, 0)->willReturn($expected['size']);
+
+		$result = $this->instance->getMetaData($path);
+		$this->assertSame($expected['encrypted'], $result['encrypted']);
+		$this->assertSame($expected['size'], $result['size']);
+	}
+
+	public function dataTestGetMetaData() {
+		return [
+			['/test.txt', ['size' => 42, 'encrypted' => false], true, true, 12, ['size' => 12, 'encrypted' => true]],
+			['/test.txt', null, true, true, 12, null],
+			['/test.txt', ['size' => 42, 'encrypted' => false], false, false, 12, ['size' => 42, 'encrypted' => false]],
+			['/test.txt', ['size' => 42, 'encrypted' => false], true, false, 12, ['size' => 12, 'encrypted' => true]]
+		];
+	}
+
+	public function testFilesize() {
+		$cache = $this->getMockBuilder('\OC\Files\Cache\Cache')
+			->disableOriginalConstructor()->getMock();
+		$cache->expects($this->any())
+			->method('get')
+			->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]);
+
+		$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
+			->setConstructorArgs(
+				[
+					[
+						'storage' => $this->sourceStorage,
+						'root' => 'foo',
+						'mountPoint' => '/',
+						'mount' => $this->mount
+					],
+					$this->encryptionManager, $this->util, $this->logger, $this->file, null, $this->keyStore, $this->update, $this->mountManager
+				]
+			)
+			->setMethods(['getCache', 'verifyUnencryptedSize'])
+			->getMock();
+
+		$this->instance->expects($this->any())->method('getCache')->willReturn($cache);
+		$this->instance->expects($this->any())->method('verifyUnencryptedSize')
+			->willReturn(42);
+
+
+		$this->assertSame(42,
+			$this->instance->filesize('/test.txt')
+		);
+
+	}
+
+	/**
+	 * @dataProvider dataTestVerifyUnencryptedSize
+	 *
+	 * @param int $encryptedSize
+	 * @param int $unencryptedSize
+	 * @param bool $failure
+	 * @param int $expected
+	 */
+	public function testVerifyUnencryptedSize($encryptedSize, $unencryptedSize, $failure, $expected) {
+		$sourceStorage = $this->getMockBuilder('\OC\Files\Storage\Storage')
+			->disableOriginalConstructor()->getMock();
+
+		$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
+			->setConstructorArgs(
+				[
+					[
+						'storage' => $sourceStorage,
+						'root' => 'foo',
+						'mountPoint' => '/',
+						'mount' => $this->mount
+					],
+					$this->encryptionManager, $this->util, $this->logger, $this->file, null, $this->keyStore, $this->update, $this->mountManager
+				]
+			)
+			->setMethods(['fixUnencryptedSize'])
+			->getMock();
+
+		$sourceStorage->expects($this->once())->method('filesize')->willReturn($encryptedSize);
+
+		$this->instance->expects($this->any())->method('fixUnencryptedSize')
+			->with('/test.txt', $encryptedSize, $unencryptedSize)
+			->willReturnCallback(
+				function() use ($failure, $expected) {
+					if ($failure) {
+						throw new \Exception();
+					} else {
+						return $expected;
+					}
+				}
+			);
+
+		$this->assertSame(
+			$expected,
+			$this->invokePrivate($this->instance, 'verifyUnencryptedSize', ['/test.txt', $unencryptedSize])
+		);
+	}
+
+	public function dataTestVerifyUnencryptedSize() {
+		return [
+			[120, 80, false, 80],
+			[120, 120, false, 80],
+			[120, -1, false, 80],
+			[120, -1, true, -1]
+		];
+	}
+
+	/**
 	 * @dataProvider dataTestCopyAndRename
 	 *
 	 * @param string $source