From 85e0c78166d45b0ef9d00b1fd2164e4969ee9b83 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Thu, 23 May 2013 01:21:36 +0200 Subject: fix problems with german "Umlaut" in folder name --- lib/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index a98953b42aa..b73e2e83fcc 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -114,7 +114,7 @@ class Scanner { $size = 0; if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) { \OC_DB::beginTransaction(); - while ($file = readdir($dh)) { + while ($file = utf8_encode(readdir($dh))) { $child = ($path) ? $path . '/' . $file : $file; if (!$this->isIgnoredDir($file)) { $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); -- cgit v1.2.3 From d4b700ef4ebfa9aba9bcadda864ed5a2a92174ed Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Thu, 23 May 2013 20:29:46 +0200 Subject: revert previous fix and added normalizer to cache class --- lib/files/cache/cache.php | 24 +++++++++++++++ lib/files/cache/scanner.php | 2 +- tests/lib/files/cache/cache.php | 67 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 3341fe50525..a7e634c8e41 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -100,6 +100,9 @@ class Cache { */ public function get($file) { if (is_string($file) or $file == '') { + // normalize file + $file = $this->normalize($file); + $where = 'WHERE `storage` = ? AND `path_hash` = ?'; $params = array($this->getNumericStorageId(), md5($file)); } else { //file id @@ -177,6 +180,9 @@ class Cache { $this->update($id, $data); return $id; } else { + // normalize file + $file = $this->normalize($file); + if (isset($this->partial[$file])) { //add any saved partial data $data = array_merge($this->partial[$file], $data); unset($this->partial[$file]); @@ -265,6 +271,9 @@ class Cache { * @return int */ public function getId($file) { + // normalize file + $file = $this->normalize($file); + $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); @@ -549,4 +558,19 @@ class Cache { return null; } } + + /** + * normalize the given path + * @param $path + * @return string + */ + public function normalize($path) { + + //normalize unicode if possible + if (class_exists('Normalizer')) { + $path = \Normalizer::normalize($path); + } + + return $path; + } } diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index b73e2e83fcc..a98953b42aa 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -114,7 +114,7 @@ class Scanner { $size = 0; if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) { \OC_DB::beginTransaction(); - while ($file = utf8_encode(readdir($dh))) { + while ($file = readdir($dh)) { $child = ($path) ? $path . '/' . $file : $file; if (!$this->isIgnoredDir($file)) { $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); diff --git a/tests/lib/files/cache/cache.php b/tests/lib/files/cache/cache.php index 1612a673838..2b1e5a56212 100644 --- a/tests/lib/files/cache/cache.php +++ b/tests/lib/files/cache/cache.php @@ -8,6 +8,8 @@ namespace Test\Files\Cache; +use PHPUnit_Framework_MockObject_MockObject; + class LongId extends \OC\Files\Storage\Temporary { public function getId() { return 'long:' . str_repeat('foo', 50) . parent::getId(); @@ -237,6 +239,71 @@ class Cache extends \PHPUnit_Framework_TestCase { $this->assertEquals(array(md5($storageId), 'foo'), \OC\Files\Cache\Cache::getById($id)); } + /** + * @brief this test show the bug resulting if we have no normalizer installed + */ + public function testWithoutNormalizer() { + // create folder Schön with U+00F6 + $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; + + // create folder Schön with U+0308 + $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; + + /** + * @var \OC\Files\Cache\Cache | PHPUnit_Framework_MockObject_MockObject $cacheMock + */ + $cacheMock = $this->getMock('\OC\Files\Cache\Cache', array('normalize'), array($this->storage), '', true); + + $cacheMock->expects($this->any()) + ->method('normalize') + ->will($this->returnArgument(0)); + + $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->assertFalse($cacheMock->get('folder')); + $this->assertGreaterThan(0, $cacheMock->put('folder', $data)); + + $this->assertFalse($cacheMock->get('folder/' . $folderWith00F6)); + $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith00F6, $data)); + + $this->assertFalse($cacheMock->get('folder/' .$folderWith0308)); + $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith0308, $data)); + + // this is our bug, we have two different hashes with the same name (Schön) + $this->assertEquals(2, count($cacheMock->getFolderContents('folder'))); + } + + /** + * @brief this test shows that there is no bug if we use the normalizer + */ + public function testWithNormalizer() { + + if(!class_exists('Normalizer')) { + $this->markTestSkipped('The Normalizer extension is not available.'); + return; + } + + // folder name Schön with U+00F6 + $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; + + // folder name Schön with U+0308 + $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; + + $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->assertFalse($this->cache->get('folder')); + $this->assertGreaterThan(0, $this->cache->put('folder', $data)); + + $this->assertFalse($this->cache->get('folder/' . $folderWith00F6)); + $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith00F6, $data)); + + $this->assertTrue(is_array($this->cache->get('folder/' .$folderWith0308))); + $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith0308, $data)); + + // at this point we should have only one folder named "Schön" + $this->assertEquals(1, count($this->cache->getFolderContents('folder'))); + } + public function tearDown() { $this->cache->clear(); } -- cgit v1.2.3 From 661b5501b0e3f456d3a56d72e342074062ded0e8 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Fri, 24 May 2013 20:35:01 +0200 Subject: added normalizeUnicode() method to OC_Util --- lib/util.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/util.php b/lib/util.php index 01e2df7bfc4..224ed32061c 100755 --- a/lib/util.php +++ b/lib/util.php @@ -1,4 +1,7 @@ Date: Fri, 24 May 2013 20:36:20 +0200 Subject: changed builtin normalizer to \OC_Util::normalizeUnicode --- lib/files/cache/cache.php | 7 +------ lib/files/filesystem.php | 5 ++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index a7e634c8e41..865abd5286f 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -566,11 +566,6 @@ class Cache { */ public function normalize($path) { - //normalize unicode if possible - if (class_exists('Normalizer')) { - $path = \Normalizer::normalize($path); - } - - return $path; + return \OC_Util::normalizeUnicode($path); } } diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index d60d430d77c..5d7565f0d83 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -616,9 +616,8 @@ class Filesystem { $path = substr($path, 0, -1); } //normalize unicode if possible - if (class_exists('Normalizer')) { - $path = \Normalizer::normalize($path); - } + $path = \OC_Util::normalizeUnicode($path); + return $path; } -- cgit v1.2.3 From 5076c0d392f6eb17e368a9382cf5b0abe7408889 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Fri, 24 May 2013 20:37:11 +0200 Subject: changed tests for using new normalizer --- tests/lib/files/cache/cache.php | 4 ++-- tests/lib/files/filesystem.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/lib/files/cache/cache.php b/tests/lib/files/cache/cache.php index 2b1e5a56212..e693fb892cd 100644 --- a/tests/lib/files/cache/cache.php +++ b/tests/lib/files/cache/cache.php @@ -278,8 +278,8 @@ class Cache extends \PHPUnit_Framework_TestCase { */ public function testWithNormalizer() { - if(!class_exists('Normalizer')) { - $this->markTestSkipped('The Normalizer extension is not available.'); + if(!class_exists('Patchwork\PHP\Shim\Normalizer')) { + $this->markTestSkipped('The 3rdparty Normalizer extension is not available.'); return; } diff --git a/tests/lib/files/filesystem.php b/tests/lib/files/filesystem.php index 6ce45e6178a..bef70cc725b 100644 --- a/tests/lib/files/filesystem.php +++ b/tests/lib/files/filesystem.php @@ -72,7 +72,7 @@ class Filesystem extends \PHPUnit_Framework_TestCase { $this->assertEquals('/path', \OC\Files\Filesystem::normalizePath('\path')); $this->assertEquals('/foo/bar', \OC\Files\Filesystem::normalizePath('/foo//bar/')); $this->assertEquals('/foo/bar', \OC\Files\Filesystem::normalizePath('/foo////bar')); - if (class_exists('Normalizer')) { + if (class_exists('Patchwork\PHP\Shim\Normalizer')) { $this->assertEquals("/foo/bar\xC3\xBC", \OC\Files\Filesystem::normalizePath("/foo/baru\xCC\x88")); } } -- cgit v1.2.3 From d1939a1c38390265eaf7b17db05b649af331a673 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 24 May 2013 21:54:47 +0200 Subject: submodule 3rdparty updated --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index a13af72fbe8..e312294ef62 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit a13af72fbe8983686fc47489a750e60319f68ac2 +Subproject commit e312294ef62873df2b8c02e774f9dfe1b7fbc38d -- cgit v1.2.3 From f6bf9de6def749b26d44a6f8d8ef85d0561cfaf9 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 14:54:36 +0200 Subject: prevent for returning while false normalize string --- lib/util.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/util.php b/lib/util.php index 224ed32061c..1c5a044be8e 100755 --- a/lib/util.php +++ b/lib/util.php @@ -833,7 +833,12 @@ class OC_Util { */ public static function normalizeUnicode($value) { if(class_exists('Patchwork\PHP\Shim\Normalizer')) { - $value = \Patchwork\PHP\Shim\Normalizer::normalize($value); + $normalizedValue = \Patchwork\PHP\Shim\Normalizer::normalize($value); + if($normalizedValue === false) { + \OC_Log::write( 'core', 'normalizing failed for "' . $value . '"', \OC_Log::WARN); + } else { + $value = $normalizedValue; + } } return $value; -- cgit v1.2.3 From c245f5a99fcd0c273de8e86d5496e86d61151d6c Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 14:56:00 +0200 Subject: added more places where normalization is needed --- lib/files/cache/cache.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 2c34fb77925..dc5e3e20fc7 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -343,6 +343,10 @@ class Cache { * @param string $target */ public function move($source, $target) { + // normalize source and target + $source = $this->normalize($source); + $target = $this->normalize($target); + $sourceData = $this->get($source); $sourceId = $sourceData['fileid']; $newParentId = $this->getParentId($target); @@ -383,6 +387,9 @@ class Cache { * @return int, Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE */ public function getStatus($file) { + // normalize file + $file = $this->normalize($file); + $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); -- cgit v1.2.3 From 505a300776a958f4076f923b0966ab13eee3c4b5 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 20:35:12 +0200 Subject: we should also normalize on update and search because the database layer will not do this for us --- lib/files/cache/cache.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index dc5e3e20fc7..1e93cc59c6b 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -226,6 +226,17 @@ class Cache { * @param array $data */ public function update($id, array $data) { + + if(isset($data['path'])) { + // normalize path + $data['path'] = $this->normalize($data['path']); + } + + if(isset($data['name'])) { + // normalize path + $data['name'] = $this->normalize($data['name']); + } + list($queryParts, $params) = $this->buildParts($data); $params[] = $id; @@ -418,6 +429,10 @@ class Cache { * @return array of file data */ public function search($pattern) { + + // normalize pattern + $pattern = $this->normalize($pattern); + $query = \OC_DB::prepare(' SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?' -- cgit v1.2.3 From 4eddef1556ac7ee7fc0c7e82279672c52d9b6db9 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 20:36:51 +0200 Subject: improved tests to check if database layer normalize folder names --- tests/lib/files/cache/cache.php | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tests/lib/files/cache/cache.php b/tests/lib/files/cache/cache.php index e693fb892cd..7b0453edb0d 100644 --- a/tests/lib/files/cache/cache.php +++ b/tests/lib/files/cache/cache.php @@ -243,10 +243,10 @@ class Cache extends \PHPUnit_Framework_TestCase { * @brief this test show the bug resulting if we have no normalizer installed */ public function testWithoutNormalizer() { - // create folder Schön with U+00F6 + // folder name "Schön" with U+00F6 (normalized) $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; - // create folder Schön with U+0308 + // folder name "Schön" with U+0308 (un-normalized) $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; /** @@ -260,15 +260,24 @@ class Cache extends \PHPUnit_Framework_TestCase { $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + // put root folder $this->assertFalse($cacheMock->get('folder')); $this->assertGreaterThan(0, $cacheMock->put('folder', $data)); - $this->assertFalse($cacheMock->get('folder/' . $folderWith00F6)); - $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith00F6, $data)); - + // put un-normalized folder $this->assertFalse($cacheMock->get('folder/' .$folderWith0308)); $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith0308, $data)); + // get un-normalized folder by name + $unNormalizedFolderName = $cacheMock->get('folder/' .$folderWith0308); + + // check if database layer normalized the folder name (this should not happen) + $this->assertEquals($folderWith0308, $unNormalizedFolderName['name']); + + // put normalized folder + $this->assertFalse($cacheMock->get('folder/' . $folderWith00F6)); + $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith00F6, $data)); + // this is our bug, we have two different hashes with the same name (Schön) $this->assertEquals(2, count($cacheMock->getFolderContents('folder'))); } @@ -283,23 +292,32 @@ class Cache extends \PHPUnit_Framework_TestCase { return; } - // folder name Schön with U+00F6 + // folder name "Schön" with U+00F6 (normalized) $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; - // folder name Schön with U+0308 + // folder name "Schön" with U+0308 (un-normalized) $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + // put root folder $this->assertFalse($this->cache->get('folder')); $this->assertGreaterThan(0, $this->cache->put('folder', $data)); - $this->assertFalse($this->cache->get('folder/' . $folderWith00F6)); - $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith00F6, $data)); - - $this->assertTrue(is_array($this->cache->get('folder/' .$folderWith0308))); + // put un-normalized folder + $this->assertFalse($this->cache->get('folder/' .$folderWith0308)); $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith0308, $data)); + // get un-normalized folder by name + $unNormalizedFolderName = $this->cache->get('folder/' .$folderWith0308); + + // check if folder name was normalized + $this->assertEquals($folderWith00F6, $unNormalizedFolderName['name']); + + // put normalized folder + $this->assertTrue(is_array($this->cache->get('folder/' . $folderWith00F6))); + $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith00F6, $data)); + // at this point we should have only one folder named "Schön" $this->assertEquals(1, count($this->cache->getFolderContents('folder'))); } -- cgit v1.2.3