diff options
author | Vincent Petry <pvince81@owncloud.com> | 2016-04-29 12:42:37 +0200 |
---|---|---|
committer | Vincent Petry <pvince81@owncloud.com> | 2016-05-20 09:33:59 +0200 |
commit | 63bbbf29f4b8fc49faf8aafd7ebf27a12e892a06 (patch) | |
tree | 16d385b238441e0b0c1ee1605e6899a0c2b099be | |
parent | 2ef751b1ec28f7b5c7113af60ec8c9fa0ae1cf87 (diff) | |
download | nextcloud-server-63bbbf29f4b8fc49faf8aafd7ebf27a12e892a06.tar.gz nextcloud-server-63bbbf29f4b8fc49faf8aafd7ebf27a12e892a06.zip |
Add wrapper for NFD encoding workaround
-rw-r--r-- | apps/files_trashbin/lib/Storage.php | 2 | ||||
-rw-r--r-- | lib/private/Files/Storage/Wrapper/Encoding.php | 568 | ||||
-rw-r--r-- | lib/private/legacy/util.php | 9 | ||||
-rw-r--r-- | tests/lib/files/storage/wrapper/Encoding.php | 154 |
4 files changed, 732 insertions, 1 deletions
diff --git a/apps/files_trashbin/lib/Storage.php b/apps/files_trashbin/lib/Storage.php index c4c523810ac..b621c511db7 100644 --- a/apps/files_trashbin/lib/Storage.php +++ b/apps/files_trashbin/lib/Storage.php @@ -150,7 +150,7 @@ class Storage extends Wrapper { return false; } - $normalized = Filesystem::normalizePath($this->mountPoint . '/' . $path); + $normalized = Filesystem::normalizePath($this->mountPoint . '/' . $path, true, false, true); $result = true; $view = Filesystem::getView(); if (!isset($this->deletedFiles[$normalized]) && $view instanceof View) { diff --git a/lib/private/Files/Storage/Wrapper/Encoding.php b/lib/private/Files/Storage/Wrapper/Encoding.php new file mode 100644 index 00000000000..0696f97368e --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Encoding.php @@ -0,0 +1,568 @@ +<?php +/** + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Files\Storage\Wrapper; + +use OCP\ICache; +use OC\Cache\CappedMemoryCache; + +/** + * Encoding wrapper that deals with file names that use unsupported encodings like NFD. + * + * When applied and a UTF-8 path name was given, the wrapper will first attempt to access + * the actual given name and then try its NFD form. + */ +class Encoding extends Wrapper { + + /** + * @var ICache + */ + private $namesCache; + + /** + * @param array $parameters + */ + public function __construct($parameters) { + $this->storage = $parameters['storage']; + $this->namesCache = new CappedMemoryCache(); + } + + /** + * Returns whether the given string is only made of ASCII characters + * + * @param string $str string + * + * @return bool true if the string is all ASCII, false otherwise + */ + private function isAscii($str) { + foreach (str_split($str) as $s) { + if (ord($s) > 127) { + return false; + } + } + return true; + } + + /** + * Checks whether the given path exists in NFC or NFD form after checking + * each form for each path section and returns the correct form. + * If no existing path found, returns the path as it was given. + * + * @param string $fullPath path to check + * + * @return string original or converted path + */ + private function findPathToUse($fullPath) { + $cachedPath = $this->namesCache[$fullPath]; + if ($cachedPath !== null) { + return $cachedPath; + } + + $sections = explode('/', $fullPath); + $path = ''; + foreach ($sections as $section) { + $path .= $section; + $convertedPath = $this->findPathToUseLastSection($path); + if ($convertedPath === null) { + // no point in continuing if the section was not found, use original path + return $fullPath; + } + $path = $convertedPath . '/'; + } + $path = rtrim($path, '/'); + $this->namesCache[$fullPath] = $path; + return $path; + } + + /** + * Checks whether the last path section of the given path exists in NFC or NFD form + * and returns the correct form. If no existing path found, returns null. + * + * @param string $fullPath path to check + * + * @return string|null original or converted path, or null if none of the forms was found + */ + private function findPathToUseLastSection($fullPath) { + $cachedPath = $this->namesCache[$fullPath]; + if ($cachedPath !== null) { + return $cachedPath; + } + + $sections = explode('/', $fullPath); + $lastSection = array_pop($sections); + $basePath = ltrim(implode('/', $sections) . '/', '/'); + + if ($lastSection === '' || $this->isAscii($lastSection)) { + $this->namesCache[$fullPath] = $fullPath; + return $fullPath; + } + + $result = $this->storage->file_exists($fullPath); + if ($result) { + $this->namesCache[$fullPath] = $fullPath; + return $fullPath; + } + // swap encoding + if (\Normalizer::isNormalized($lastSection, \Normalizer::FORM_C)) { + $otherFormPath = \Normalizer::normalize($lastSection, \Normalizer::FORM_D); + } else { + $otherFormPath = \Normalizer::normalize($lastSection, \Normalizer::FORM_C); + } + if ($this->storage->file_exists($basePath . $otherFormPath)) { + $this->namesCache[$fullPath] = $basePath . $otherFormPath; + return $basePath . $otherFormPath; + } + + // return original path, file did not exist at all + $this->namesCache[$fullPath] = $fullPath; + return null; + } + + /** + * see http://php.net/manual/en/function.mkdir.php + * + * @param string $path + * @return bool + */ + public function mkdir($path) { + // note: no conversion here, method should not be called with non-NFC names! + $result = $this->storage->mkdir($path); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.rmdir.php + * + * @param string $path + * @return bool + */ + public function rmdir($path) { + $result = $this->storage->rmdir($this->findPathToUse($path)); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.opendir.php + * + * @param string $path + * @return resource + */ + public function opendir($path) { + return $this->storage->opendir($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.is_dir.php + * + * @param string $path + * @return bool + */ + public function is_dir($path) { + return $this->storage->is_dir($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.is_file.php + * + * @param string $path + * @return bool + */ + public function is_file($path) { + return $this->storage->is_file($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.stat.php + * only the following keys are required in the result: size and mtime + * + * @param string $path + * @return array + */ + public function stat($path) { + return $this->storage->stat($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.filetype.php + * + * @param string $path + * @return bool + */ + public function filetype($path) { + return $this->storage->filetype($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.filesize.php + * The result for filesize when called on a folder is required to be 0 + * + * @param string $path + * @return int + */ + public function filesize($path) { + return $this->storage->filesize($this->findPathToUse($path)); + } + + /** + * check if a file can be created in $path + * + * @param string $path + * @return bool + */ + public function isCreatable($path) { + return $this->storage->isCreatable($this->findPathToUse($path)); + } + + /** + * check if a file can be read + * + * @param string $path + * @return bool + */ + public function isReadable($path) { + return $this->storage->isReadable($this->findPathToUse($path)); + } + + /** + * check if a file can be written to + * + * @param string $path + * @return bool + */ + public function isUpdatable($path) { + return $this->storage->isUpdatable($this->findPathToUse($path)); + } + + /** + * check if a file can be deleted + * + * @param string $path + * @return bool + */ + public function isDeletable($path) { + return $this->storage->isDeletable($this->findPathToUse($path)); + } + + /** + * check if a file can be shared + * + * @param string $path + * @return bool + */ + public function isSharable($path) { + return $this->storage->isSharable($this->findPathToUse($path)); + } + + /** + * get the full permissions of a path. + * Should return a combination of the PERMISSION_ constants defined in lib/public/constants.php + * + * @param string $path + * @return int + */ + public function getPermissions($path) { + return $this->storage->getPermissions($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.file_exists.php + * + * @param string $path + * @return bool + */ + public function file_exists($path) { + return $this->storage->file_exists($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.filemtime.php + * + * @param string $path + * @return int + */ + public function filemtime($path) { + return $this->storage->filemtime($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.file_get_contents.php + * + * @param string $path + * @return string + */ + public function file_get_contents($path) { + return $this->storage->file_get_contents($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.file_put_contents.php + * + * @param string $path + * @param string $data + * @return bool + */ + public function file_put_contents($path, $data) { + $result = $this->storage->file_put_contents($this->findPathToUse($path), $data); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.unlink.php + * + * @param string $path + * @return bool + */ + public function unlink($path) { + $result = $this->storage->unlink($this->findPathToUse($path)); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.rename.php + * + * @param string $path1 + * @param string $path2 + * @return bool + */ + public function rename($path1, $path2) { + // second name always NFC + $result = $this->storage->rename($this->findPathToUse($path1), $path2); + if ($result) { + unset($this->namesCache[$path1]); + unset($this->namesCache[$path2]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.copy.php + * + * @param string $path1 + * @param string $path2 + * @return bool + */ + public function copy($path1, $path2) { + $result = $this->storage->copy($this->findPathToUse($path1), $path2); + if ($result) { + unset($this->namesCache[$path2]); + } + return $result; + } + + /** + * see http://php.net/manual/en/function.fopen.php + * + * @param string $path + * @param string $mode + * @return resource + */ + public function fopen($path, $mode) { + $result = $this->storage->fopen($this->findPathToUse($path), $mode); + if ($result && $mode !== 'r' && $mode !== 'rb') { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * get the mimetype for a file or folder + * The mimetype for a folder is required to be "httpd/unix-directory" + * + * @param string $path + * @return string + */ + public function getMimeType($path) { + return $this->storage->getMimeType($this->findPathToUse($path)); + } + + /** + * see http://php.net/manual/en/function.hash.php + * + * @param string $type + * @param string $path + * @param bool $raw + * @return string + */ + public function hash($type, $path, $raw = false) { + return $this->storage->hash($type, $this->findPathToUse($path), $raw); + } + + /** + * see http://php.net/manual/en/function.free_space.php + * + * @param string $path + * @return int + */ + public function free_space($path) { + return $this->storage->free_space($this->findPathToUse($path)); + } + + /** + * search for occurrences of $query in file names + * + * @param string $query + * @return array + */ + public function search($query) { + return $this->storage->search($query); + } + + /** + * see http://php.net/manual/en/function.touch.php + * If the backend does not support the operation, false should be returned + * + * @param string $path + * @param int $mtime + * @return bool + */ + public function touch($path, $mtime = null) { + $result = $this->storage->touch($this->findPathToUse($path), $mtime); + if ($result) { + unset($this->namesCache[$path]); + } + return $result; + } + + /** + * get the path to a local version of the file. + * The local version of the file can be temporary and doesn't have to be persistent across requests + * + * @param string $path + * @return string + */ + public function getLocalFile($path) { + return $this->storage->getLocalFile($this->findPathToUse($path)); + } + + /** + * check if a file or folder has been updated since $time + * + * @param string $path + * @param int $time + * @return bool + * + * hasUpdated for folders should return at least true if a file inside the folder is add, removed or renamed. + * returning true for other changes in the folder is optional + */ + public function hasUpdated($path, $time) { + return $this->storage->hasUpdated($this->findPathToUse($path), $time); + } + + /** + * get a cache instance for the storage + * + * @param string $path + * @param \OC\Files\Storage\Storage (optional) the storage to pass to the cache + * @return \OC\Files\Cache\Cache + */ + public function getCache($path = '', $storage = null) { + if (!$storage) { + $storage = $this; + } + return $this->storage->getCache($this->findPathToUse($path), $storage); + } + + /** + * get a scanner instance for the storage + * + * @param string $path + * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner + * @return \OC\Files\Cache\Scanner + */ + public function getScanner($path = '', $storage = null) { + if (!$storage) { + $storage = $this; + } + return $this->storage->getScanner($this->findPathToUse($path), $storage); + } + + /** + * get the ETag for a file or folder + * + * @param string $path + * @return string + */ + public function getETag($path) { + return $this->storage->getETag($this->findPathToUse($path)); + } + + /** + * @param \OCP\Files\Storage $sourceStorage + * @param string $sourceInternalPath + * @param string $targetInternalPath + * @return bool + */ + public function copyFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) { + if ($sourceStorage === $this) { + return $this->copy($this->findPathToUse($sourceInternalPath), $targetInternalPath); + } + + $result = $this->storage->copyFromStorage($sourceStorage, $this->findPathToUse($sourceInternalPath), $targetInternalPath); + if ($result) { + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + /** + * @param \OCP\Files\Storage $sourceStorage + * @param string $sourceInternalPath + * @param string $targetInternalPath + * @return bool + */ + public function moveFromStorage(\OCP\Files\Storage $sourceStorage, $sourceInternalPath, $targetInternalPath) { + if ($sourceStorage === $this) { + $result = $this->rename($this->findPathToUse($sourceInternalPath), $targetInternalPath); + if ($result) { + unset($this->namesCache[$sourceInternalPath]); + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + $result = $this->storage->moveFromStorage($sourceStorage, $this->findPathToUse($sourceInternalPath), $targetInternalPath); + if ($result) { + unset($this->namesCache[$sourceInternalPath]); + unset($this->namesCache[$targetInternalPath]); + } + return $result; + } + + /** + * @param string $path + * @return array + */ + public function getMetaData($path) { + return $this->storage->getMetaData($this->findPathToUse($path)); + } +} diff --git a/lib/private/legacy/util.php b/lib/private/legacy/util.php index 4f7a8668dfc..4196aa6637c 100644 --- a/lib/private/legacy/util.php +++ b/lib/private/legacy/util.php @@ -172,6 +172,15 @@ class OC_Util { return $storage; }); + // install storage availability wrapper, before most other wrappers + \OC\Files\Filesystem::addStorageWrapper('oc_encoding', function ($mountPoint, $storage) { + // TODO: only do this opt-in if the mount option is specified + if (!$storage->instanceOfStorage('\OC\Files\Storage\Shared') && !$storage->isLocal()) { + return new \OC\Files\Storage\Wrapper\Encoding(['storage' => $storage]); + } + return $storage; + }); + \OC\Files\Filesystem::addStorageWrapper('oc_quota', function ($mountPoint, $storage) { // set up quota for home storages, even for other users // which can happen when using sharing diff --git a/tests/lib/files/storage/wrapper/Encoding.php b/tests/lib/files/storage/wrapper/Encoding.php new file mode 100644 index 00000000000..db0dfa66652 --- /dev/null +++ b/tests/lib/files/storage/wrapper/Encoding.php @@ -0,0 +1,154 @@ +<?php +/** + * Copyright (c) 2016 Vincent Petry <pvince81@owncloud.com> + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\Files\Storage\Wrapper; + +class Encoding extends \Test\Files\Storage\Storage { + + const NFD_NAME = 'ümlaut'; + const NFC_NAME = 'ümlaut'; + + /** + * @var \OC\Files\Storage\Temporary + */ + private $sourceStorage; + + public function setUp() { + parent::setUp(); + $this->sourceStorage = new \OC\Files\Storage\Temporary([]); + $this->instance = new \OC\Files\Storage\Wrapper\Encoding([ + 'storage' => $this->sourceStorage + ]); + } + + public function tearDown() { + $this->sourceStorage->cleanUp(); + parent::tearDown(); + } + + public function directoryProvider() { + $a = parent::directoryProvider(); + $a[] = [self::NFD_NAME]; + return $a; + } + + public function fileNameProvider() { + $a = parent::fileNameProvider(); + $a[] = [self::NFD_NAME . '.txt']; + return $a; + } + + public function copyAndMoveProvider() { + $a = parent::copyAndMoveProvider(); + $a[] = [self::NFD_NAME . '.txt', self::NFC_NAME . '-renamed.txt']; + return $a; + } + + public function accessNameProvider() { + return [ + [self::NFD_NAME], + [self::NFC_NAME], + ]; + } + + /** + * @dataProvider accessNameProvider + */ + public function testFputEncoding($accessName) { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'bar'); + $this->assertEquals('bar', $this->instance->file_get_contents($accessName)); + } + + /** + * @dataProvider accessNameProvider + */ + public function testFopenReadEncoding($accessName) { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'bar'); + $fh = $this->instance->fopen($accessName, 'r'); + $data = fgets($fh); + fclose($fh); + $this->assertEquals('bar', $data); + } + + /** + * @dataProvider accessNameProvider + */ + public function testFopenOverwriteEncoding($accessName) { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'bar'); + $fh = $this->instance->fopen($accessName, 'w'); + $data = fputs($fh, 'test'); + fclose($fh); + $data = $this->sourceStorage->file_get_contents(self::NFD_NAME); + $this->assertEquals('test', $data); + $this->assertFalse($this->sourceStorage->file_exists(self::NFC_NAME)); + } + + /** + * @dataProvider accessNameProvider + */ + public function testFileExistsEncoding($accessName) { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'bar'); + $this->assertTrue($this->instance->file_exists($accessName)); + } + + /** + * @dataProvider accessNameProvider + */ + public function testUnlinkEncoding($accessName) { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'bar'); + $this->assertTrue($this->instance->unlink($accessName)); + $this->assertFalse($this->sourceStorage->file_exists(self::NFC_NAME)); + $this->assertFalse($this->sourceStorage->file_exists(self::NFD_NAME)); + } + + public function testNfcHigherPriority() { + $this->sourceStorage->file_put_contents(self::NFC_NAME, 'nfc'); + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'nfd'); + $this->assertEquals('nfc', $this->instance->file_get_contents(self::NFC_NAME)); + } + + public function encodedDirectoriesProvider() { + return [ + [self::NFD_NAME, self::NFC_NAME], + [self::NFD_NAME . '/' . self::NFD_NAME, self::NFC_NAME . '/' . self::NFC_NAME], + [self::NFD_NAME . '/' . self::NFC_NAME . '/' .self::NFD_NAME, self::NFC_NAME . '/' . self::NFC_NAME . '/' . self::NFC_NAME], + ]; + } + + /** + * @dataProvider encodedDirectoriesProvider + */ + public function testOperationInsideDirectory($sourceDir, $accessDir) { + $this->sourceStorage->mkdir($sourceDir); + $this->instance->file_put_contents($accessDir . '/test.txt', 'bar'); + $this->assertTrue($this->instance->file_exists($accessDir . '/test.txt')); + $this->assertEquals('bar', $this->instance->file_get_contents($accessDir . '/test.txt')); + + $this->sourceStorage->file_put_contents($sourceDir . '/' . self::NFD_NAME, 'foo'); + $this->assertTrue($this->instance->file_exists($accessDir . '/' . self::NFC_NAME)); + $this->assertEquals('foo', $this->instance->file_get_contents($accessDir . '/' . self::NFC_NAME)); + + // try again to make it use cached path + $this->assertEquals('bar', $this->instance->file_get_contents($accessDir . '/test.txt')); + $this->assertTrue($this->instance->file_exists($accessDir . '/test.txt')); + $this->assertEquals('foo', $this->instance->file_get_contents($accessDir . '/' . self::NFC_NAME)); + $this->assertTrue($this->instance->file_exists($accessDir . '/' . self::NFC_NAME)); + } + + public function testCacheExtraSlash() { + $this->sourceStorage->file_put_contents(self::NFD_NAME, 'foo'); + $this->assertEquals(3, $this->instance->file_put_contents(self::NFC_NAME, 'bar')); + $this->assertEquals('bar', $this->instance->file_get_contents(self::NFC_NAME)); + clearstatcache(); + $this->assertEquals(5, $this->instance->file_put_contents('/' . self::NFC_NAME, 'baric')); + $this->assertEquals('baric', $this->instance->file_get_contents(self::NFC_NAME)); + clearstatcache(); + $this->assertEquals(8, $this->instance->file_put_contents('/' . self::NFC_NAME, 'barbaric')); + $this->assertEquals('barbaric', $this->instance->file_get_contents('//' . self::NFC_NAME)); + } +} |