]> source.dussan.org Git - nextcloud-server.git/commitdiff
initial version of a local storage implementation which will use unique slugified...
authorThomas Mueller <thomas.mueller@tmit.eu>
Wed, 6 Feb 2013 22:36:38 +0000 (23:36 +0100)
committerThomas Mueller <thomas.mueller@tmit.eu>
Wed, 6 Feb 2013 22:41:52 +0000 (23:41 +0100)
This implementation will only be enabled on windows based system to solve the issues around UTF-8 file names with php on windows.

db_structure.xml
lib/files/mapper.php [new file with mode: 0644]
lib/files/storage/local.php
lib/files/storage/mappedlocal.php [new file with mode: 0644]
lib/files/storage/temporary.php
tests/lib/files/storage/storage.php

index f4111bfabd02bfccbea9860271a3e6a78e65b129..fc7f1082ffa28f7f9bc74a1c011faffc85f6df28 100644 (file)
 
        </table>
 
+       <table>
+
+               <name>*dbprefix*file_map</name>
+
+               <declaration>
+
+                       <field>
+                               <name>logic_path</name>
+                               <type>text</type>
+                               <default></default>
+                               <notnull>true</notnull>
+                               <length>512</length>
+                       </field>
+
+                       <field>
+                               <name>physic_path</name>
+                               <type>text</type>
+                               <default></default>
+                               <notnull>true</notnull>
+                               <length>512</length>
+                       </field>
+
+                       <index>
+                               <name>file_map_lp_index</name>
+                               <unique>true</unique>
+                               <field>
+                                       <name>logic_path</name>
+                                       <sorting>ascending</sorting>
+                               </field>
+                       </index>
+
+                       <index>
+                               <name>file_map_pp_index</name>
+                               <unique>true</unique>
+                               <field>
+                                       <name>physic_path</name>
+                                       <sorting>ascending</sorting>
+                               </field>
+                       </index>
+
+               </declaration>
+
+       </table>
+
        <table>
 
                <name>*dbprefix*mimetypes</name>
diff --git a/lib/files/mapper.php b/lib/files/mapper.php
new file mode 100644 (file)
index 0000000..90e4e1c
--- /dev/null
@@ -0,0 +1,216 @@
+<?php
+
+namespace OC\Files;
+
+/**
+ * class Mapper is responsible to translate logical paths to physical paths and reverse
+ */
+class Mapper
+{
+       /**
+        * @param string $logicPath
+        * @param bool $create indicates if the generated physical name shall be stored in the database or not
+        * @return string the physical path
+        */
+       public function logicToPhysical($logicPath, $create) {
+               $physicalPath = $this->resolveLogicPath($logicPath);
+               if ($physicalPath !== null) {
+                       return $physicalPath;
+               }
+
+               return $this->create($logicPath, $create);
+       }
+
+       /**
+        * @param string $physicalPath
+        * @return string|null
+        */
+       public function physicalToLogic($physicalPath) {
+               $logicPath = $this->resolvePhysicalPath($physicalPath);
+               if ($logicPath !== null) {
+                       return $logicPath;
+               }
+
+               $this->insert($physicalPath, $physicalPath);
+               return $physicalPath;
+       }
+
+       /**
+        * @param string $path
+        * @param bool $isLogicPath indicates if $path is logical or physical
+        * @param $recursive
+        */
+       public function removePath($path, $isLogicPath, $recursive) {
+               if ($recursive) {
+                       $path=$path.'%';
+               }
+
+               if ($isLogicPath) {
+                       $query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
+                       $query->execute(array($path));
+               } else {
+                       $query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `physic_path` LIKE ?');
+                       $query->execute(array($path));
+               }
+       }
+
+       /**
+        * @param $path1
+        * @param $path2
+        * @throws \Exception
+        */
+       public function copy($path1, $path2)
+       {
+               $path1 = $this->stripLast($path1);
+               $path2 = $this->stripLast($path2);
+               $physicPath1 = $this->logicToPhysical($path1, true);
+               $physicPath2 = $this->logicToPhysical($path2, true);
+
+               $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
+               $result = $query->execute(array($path1.'%'));
+               $updateQuery = \OC_DB::prepare('UPDATE `*PREFIX*file_map`'
+                       .' SET `logic_path` = ?'
+                       .' AND `physic_path` = ?'
+                       .' WHERE `logic_path` = ?');
+               while( $row = $result->fetchRow()) {
+                       $currentLogic = $row['logic_path'];
+                       $currentPhysic = $row['physic_path'];
+                       $newLogic = $path2.$this->stripRootFolder($currentLogic, $path1);
+                       $newPhysic = $physicPath2.$this->stripRootFolder($currentPhysic, $physicPath1);
+                       if ($path1 !== $currentLogic) {
+                               try {
+                                       $updateQuery->execute(array($newLogic, $newPhysic, $currentLogic));
+                               } catch (\Exception $e) {
+                                       error_log('Mapper::Copy failed '.$currentLogic.' -> '.$newLogic.'\n'.$e);
+                                       throw $e;
+                               }
+                       }
+               }
+       }
+
+       /**
+        * @param $path
+        * @param $root
+        * @return bool|string
+        */
+       public function stripRootFolder($path, $root) {
+               if (strpos($path, $root) !== 0) {
+                       // throw exception ???
+                       return false;
+               }
+               if (strlen($path) > strlen($root)) {
+                       return substr($path, strlen($root));
+               }
+
+               return '';
+       }
+
+       private function stripLast($path) {
+               if (substr($path, -1) == '/') {
+                       $path = substr_replace($path ,'',-1);
+               }
+               return $path;
+       }
+
+       private function resolveLogicPath($logicPath) {
+               $logicPath = $this->stripLast($logicPath);
+               $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` = ?');
+               $result = $query->execute(array($logicPath));
+               $result = $result->fetchRow();
+
+               return $result['physic_path'];
+       }
+
+       private function resolvePhysicalPath($physicalPath) {
+               $physicalPath = $this->stripLast($physicalPath);
+               $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `physic_path` = ?');
+               $result = $query->execute(array($physicalPath));
+               $result = $result->fetchRow();
+
+               return $result['logic_path'];
+       }
+
+       private function create($logicPath, $store) {
+               $logicPath = $this->stripLast($logicPath);
+               $index = 0;
+
+               // create the slugified path
+               $physicalPath = $this->slugifyPath($logicPath);
+
+               // detect duplicates
+               while ($this->resolvePhysicalPath($physicalPath) !== null) {
+                       $physicalPath = $this->slugifyPath($physicalPath, $index++);
+               }
+
+               // insert the new path mapping if requested
+               if ($store) {
+                       $this->insert($logicPath, $physicalPath);
+               }
+
+               return $physicalPath;
+       }
+
+       private function insert($logicPath, $physicalPath) {
+               $query = \OC_DB::prepare('INSERT INTO `*PREFIX*file_map`(`logic_path`,`physic_path`) VALUES(?,?)');
+               $query->execute(array($logicPath, $physicalPath));
+       }
+
+       private function slugifyPath($path, $index=null) {
+               $pathElements = explode('/', $path);
+               $sluggedElements = array();
+
+               // skip slugging the drive letter on windows - TODO: test if local path
+               if (strpos(strtolower(php_uname('s')), 'win') !== false) {
+                       $sluggedElements[]= $pathElements[0];
+                       array_shift($pathElements);
+               }
+               foreach ($pathElements as $pathElement) {
+                       // TODO: remove file ext before slugify on last element
+                       $sluggedElements[] = self::slugify($pathElement);
+               }
+
+               //
+               // TODO: add the index before the file extension
+               //
+               if ($index !== null) {
+                       $last= end($sluggedElements);
+                       array_pop($sluggedElements);
+                       array_push($sluggedElements, $last.'-'.$index);
+               }
+               return implode(DIRECTORY_SEPARATOR, $sluggedElements);
+       }
+
+       /**
+        * Modifies a string to remove all non ASCII characters and spaces.
+        *
+        * @param string $text
+        * @return string
+        */
+       private function slugify($text)
+       {
+               // replace non letter or digits by -
+               $text = preg_replace('~[^\\pL\d]+~u', '-', $text);
+
+               // trim
+               $text = trim($text, '-');
+
+               // transliterate
+               if (function_exists('iconv')) {
+                       $text = iconv('utf-8', 'us-ascii//TRANSLIT', $text);
+               }
+
+               // lowercase
+               $text = strtolower($text);
+
+               // remove unwanted characters
+               $text = preg_replace('~[^-\w]+~', '', $text);
+
+               if (empty($text))
+               {
+                       // TODO: we better generate a guid in this case
+                       return 'n-a';
+               }
+
+               return $text;
+       }
+}
index a5db4ba91947bb2961f491f1ed47c1164911873f..d387a89832015be212061ab06ce6b116c506a94a 100644 (file)
@@ -8,6 +8,10 @@
 
 namespace OC\Files\Storage;
 
+if (\OC_Util::runningOnWindows()) {
+       require_once 'mappedlocal.php';
+} else {
+
 /**
  * for local filestore, we only have to map the paths
  */
@@ -245,3 +249,4 @@ class Local extends \OC\Files\Storage\Common{
                return $this->filemtime($path)>$time;
        }
 }
+}
diff --git a/lib/files/storage/mappedlocal.php b/lib/files/storage/mappedlocal.php
new file mode 100644 (file)
index 0000000..80dd79b
--- /dev/null
@@ -0,0 +1,335 @@
+<?php
+/**
+ * Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+namespace OC\Files\Storage;
+
+/**
+ * for local filestore, we only have to map the paths
+ */
+class Local extends \OC\Files\Storage\Common{
+       protected $datadir;
+       private $mapper;
+
+       public function __construct($arguments) {
+               $this->datadir=$arguments['datadir'];
+               if(substr($this->datadir, -1)!=='/') {
+                       $this->datadir.='/';
+               }
+
+               $this->mapper= new \OC\Files\Mapper();
+       }
+       public function __destruct() {
+               if (defined('PHPUNIT_RUN')) {
+                       $this->mapper->removePath($this->datadir, true, true);
+               }
+       }
+       public function getId(){
+               return 'local::'.$this->datadir;
+       }
+       public function mkdir($path) {
+               return @mkdir($this->buildPath($path));
+       }
+       public function rmdir($path) {
+               if ($result = @rmdir($this->buildPath($path))) {
+                       $this->cleanMapper($path);
+               }
+               return $result;
+       }
+       public function opendir($path) {
+               $files = array('.', '..');
+               $physicalPath= $this->buildPath($path);
+
+               $logicalPath = $this->mapper->physicalToLogic($physicalPath);
+               $dh = opendir($physicalPath);
+               while ($file = readdir($dh)) {
+                       if ($file === '.' or $file === '..') {
+                               continue;
+                       }
+
+                       $logicalFilePath = $this->mapper->physicalToLogic($physicalPath.DIRECTORY_SEPARATOR.$file);
+
+                       $file= $this->mapper->stripRootFolder($logicalFilePath, $logicalPath);
+                       $file = $this->stripLeading($file);
+                       $files[]= $file;
+               }
+
+               \OC\Files\Stream\Dir::register('local-win32'.$path, $files);
+               return opendir('fakedir://local-win32'.$path);
+       }
+       public function is_dir($path) {
+               if(substr($path,-1)=='/') {
+                       $path=substr($path, 0, -1);
+               }
+               return is_dir($this->buildPath($path));
+       }
+       public function is_file($path) {
+               return is_file($this->buildPath($path));
+       }
+       public function stat($path) {
+               $fullPath = $this->buildPath($path);
+               $statResult = stat($fullPath);
+
+               if ($statResult['size'] < 0) {
+                       $size = self::getFileSizeFromOS($fullPath);
+                       $statResult['size'] = $size;
+                       $statResult[7] = $size;
+               }
+               return $statResult;
+       }
+       public function filetype($path) {
+               $filetype=filetype($this->buildPath($path));
+               if($filetype=='link') {
+                       $filetype=filetype(realpath($this->buildPath($path)));
+               }
+               return $filetype;
+       }
+       public function filesize($path) {
+               if($this->is_dir($path)) {
+                       return 0;
+               }else{
+                       $fullPath = $this->buildPath($path);
+                       $fileSize = filesize($fullPath);
+                       if ($fileSize < 0) {
+                               return self::getFileSizeFromOS($fullPath);
+                       }
+
+                       return $fileSize;
+               }
+       }
+       public function isReadable($path) {
+               return is_readable($this->buildPath($path));
+       }
+       public function isUpdatable($path) {
+               return is_writable($this->buildPath($path));
+       }
+       public function file_exists($path) {
+               return file_exists($this->buildPath($path));
+       }
+       public function filemtime($path) {
+               return filemtime($this->buildPath($path));
+       }
+       public function touch($path, $mtime=null) {
+               // sets the modification time of the file to the given value.
+               // If mtime is nil the current time is set.
+               // note that the access time of the file always changes to the current time.
+               if(!is_null($mtime)) {
+                       $result=touch( $this->buildPath($path), $mtime );
+               }else{
+                       $result=touch( $this->buildPath($path));
+               }
+               if( $result ) {
+                       clearstatcache( true, $this->buildPath($path) );
+               }
+
+               return $result;
+       }
+       public function file_get_contents($path) {
+               return file_get_contents($this->buildPath($path));
+       }
+       public function file_put_contents($path, $data) {//trigger_error("$path = ".var_export($path, 1));
+               return file_put_contents($this->buildPath($path), $data);
+       }
+       public function unlink($path) {
+               return $this->delTree($path);
+       }
+       public function rename($path1, $path2) {
+               if (!$this->isUpdatable($path1)) {
+                       \OC_Log::write('core','unable to rename, file is not writable : '.$path1,\OC_Log::ERROR);
+                       return false;
+               }
+               if(! $this->file_exists($path1)) {
+                       \OC_Log::write('core','unable to rename, file does not exists : '.$path1,\OC_Log::ERROR);
+                       return false;
+               }
+
+               $physicPath1 = $this->buildPath($path1);
+               $physicPath2 = $this->buildPath($path2);
+               if($return=rename($physicPath1, $physicPath2)) {
+                       // mapper needs to create copies or all children
+                       $this->copyMapping($path1, $path2);
+                       $this->cleanMapper($physicPath1, false, true);
+               }
+               return $return;
+       }
+       public function copy($path1, $path2) {
+               if($this->is_dir($path2)) {
+                       if(!$this->file_exists($path2)) {
+                               $this->mkdir($path2);
+                       }
+                       $source=substr($path1, strrpos($path1, '/')+1);
+                       $path2.=$source;
+               }
+               if($return=copy($this->buildPath($path1), $this->buildPath($path2))) {
+                       // mapper needs to create copies or all children
+                       $this->copyMapping($path1, $path2);
+               }
+               return $return;
+       }
+       public function fopen($path, $mode) {
+               if($return=fopen($this->buildPath($path), $mode)) {
+                       switch($mode) {
+                               case 'r':
+                                       break;
+                               case 'r+':
+                               case 'w+':
+                               case 'x+':
+                               case 'a+':
+                                       break;
+                               case 'w':
+                               case 'x':
+                               case 'a':
+                                       break;
+                       }
+               }
+               return $return;
+       }
+
+       public function getMimeType($path) {
+               if($this->isReadable($path)) {
+                       return \OC_Helper::getMimeType($this->buildPath($path));
+               }else{
+                       return false;
+               }
+       }
+
+       private function delTree($dir, $isLogicPath=true) {
+               $dirRelative=$dir;
+               if ($isLogicPath) {
+                       $dir=$this->buildPath($dir);
+               }
+               if (!file_exists($dir)) {
+                       return true;
+               }
+               if (!is_dir($dir) || is_link($dir)) {
+                       if($return=unlink($dir)) {
+                               $this->cleanMapper($dir, false);
+                               return $return;
+                       }
+               }
+               foreach (scandir($dir) as $item) {
+                       if ($item == '.' || $item == '..') {
+                               continue;
+                       }
+                       if(is_file($dir.'/'.$item)) {
+                               if(unlink($dir.'/'.$item)) {
+                                       $this->cleanMapper($dir.'/'.$item, false);
+                               }
+                       }elseif(is_dir($dir.'/'.$item)) {
+                               if (!$this->delTree($dir. "/" . $item, false)) {
+                                       return false;
+                               };
+                       }
+               }
+               if($return=rmdir($dir)) {
+                       $this->cleanMapper($dir, false);
+               }
+               return $return;
+       }
+
+       private static function getFileSizeFromOS($fullPath) {
+               $name = strtolower(php_uname('s'));
+               // Windows OS: we use COM to access the filesystem
+               if (strpos($name, 'win') !== false) {
+                       if (class_exists('COM')) {
+                               $fsobj = new \COM("Scripting.FileSystemObject");
+                               $f = $fsobj->GetFile($fullPath);
+                               return $f->Size;
+                       }
+               } else if (strpos($name, 'bsd') !== false) {
+                       if (\OC_Helper::is_function_enabled('exec')) {
+                               return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
+                       }
+               } else if (strpos($name, 'linux') !== false) {
+                       if (\OC_Helper::is_function_enabled('exec')) {
+                               return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
+                       }
+               } else {
+                       \OC_Log::write('core', 'Unable to determine file size of "'.$fullPath.'". Unknown OS: '.$name, \OC_Log::ERROR);
+               }
+
+               return 0;
+       }
+
+       public function hash($path, $type, $raw=false) {
+               return hash_file($type, $this->buildPath($path), $raw);
+       }
+
+       public function free_space($path) {
+               return @disk_free_space($this->buildPath($path));
+       }
+
+       public function search($query) {
+               return $this->searchInDir($query);
+       }
+       public function getLocalFile($path) {
+               return $this->buildPath($path);
+       }
+       public function getLocalFolder($path) {
+               return $this->buildPath($path);
+       }
+
+       protected function searchInDir($query, $dir='', $isLogicPath=true) {
+               $files=array();
+               $physicalDir = $this->buildPath($dir);
+               foreach (scandir($physicalDir) as $item) {
+                       if ($item == '.' || $item == '..')
+                               continue;
+                       $physicalItem = $this->mapper->physicalToLogic($physicalDir.DIRECTORY_SEPARATOR.$item);
+                       $item = substr($physicalItem, strlen($physicalDir)+1);
+
+                       if(strstr(strtolower($item), strtolower($query)) !== false) {
+                               $files[]=$dir.'/'.$item;
+                       }
+                       if(is_dir($physicalItem)) {
+                               $files=array_merge($files, $this->searchInDir($query, $physicalItem, false));
+                       }
+               }
+               return $files;
+       }
+
+       /**
+        * check if a file or folder has been updated since $time
+        * @param string $path
+        * @param int $time
+        * @return bool
+        */
+       public function hasUpdated($path, $time) {
+               return $this->filemtime($path)>$time;
+       }
+
+       private function buildPath($path, $create=true) {
+               $path = $this->stripLeading($path);
+               $fullPath = $this->datadir.$path;
+               return $this->mapper->logicToPhysical($fullPath, $create);
+       }
+
+       private function cleanMapper($path, $isLogicPath=true, $recursive=true) {
+               $fullPath = $path;
+               if ($isLogicPath) {
+                       $fullPath = $this->datadir.$path;
+               }
+               $this->mapper->removePath($fullPath, $isLogicPath, $recursive);
+       }
+
+       private function copyMapping($path1, $path2) {
+               $path1 = $this->stripLeading($path1);
+               $path2 = $this->stripLeading($path2);
+
+               $fullPath1 = $this->datadir.$path1;
+               $fullPath2 = $this->datadir.$path2;
+
+               $this->mapper->copy($fullPath1, $fullPath2);
+       }
+
+       private function stripLeading($path) {
+               if(strpos($path, '/') === 0) {
+                       $path = substr($path, 1);
+               }
+
+               return $path;
+       }
+}
index 542d2cd9f48ad466da6abe4a5346ac1edfa1afbd..d84dbda2e39dd844e68ddf0c9bad7cfb2206fe3c 100644 (file)
@@ -21,6 +21,7 @@ class Temporary extends Local{
        }
 
        public function __destruct() {
+               parent::__destruct();
                $this->cleanUp();
        }
 }
index 781c0f92c9262111c7db57ddf310b42e23f0051f..c74a16f509f5ae73fe85a7a77aa41b299c8ec47e 100644 (file)
@@ -146,10 +146,19 @@ abstract class Storage extends \PHPUnit_Framework_TestCase {
                $localFolder = $this->instance->getLocalFolder('/folder');
 
                $this->assertTrue(is_dir($localFolder));
-               $this->assertTrue(file_exists($localFolder . '/lorem.txt'));
-               $this->assertEquals(file_get_contents($localFolder . '/lorem.txt'), file_get_contents($textFile));
-               $this->assertEquals(file_get_contents($localFolder . '/bar.txt'), 'asd');
-               $this->assertEquals(file_get_contents($localFolder . '/recursive/file.txt'), 'foo');
+
+               // test below require to use instance->getLocalFile because the physical storage might be different
+               $localFile = $this->instance->getLocalFile('/folder/lorem.txt');
+               $this->assertTrue(file_exists($localFile));
+               $this->assertEquals(file_get_contents($localFile), file_get_contents($textFile));
+
+               $localFile = $this->instance->getLocalFile('/folder/bar.txt');
+               $this->assertTrue(file_exists($localFile));
+               $this->assertEquals(file_get_contents($localFile), 'asd');
+
+               $localFile = $this->instance->getLocalFile('/folder/recursive/file.txt');
+               $this->assertTrue(file_exists($localFile));
+               $this->assertEquals(file_get_contents($localFile), 'foo');
        }
 
        public function testStat() {