From 3e70d563a6774567ec77e9d9adf6b9ccb1e9619d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sun, 10 Feb 2013 12:27:35 +0100 Subject: Cache: bookkeeping of storage_mtime --- lib/files/cache/cache.php | 17 ++++++++++++++--- lib/files/cache/scanner.php | 1 + lib/files/cache/watcher.php | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index dcb6e8fd39a..d696f003c57 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -114,7 +114,7 @@ class Cache { $params = array($file); } $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `storage_mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` ' . $where); $result = $query->execute($params); $data = $result->fetchRow(); @@ -133,6 +133,9 @@ class Cache { $data['storage'] = $this->storageId; $data['mimetype'] = $this->getMimetype($data['mimetype']); $data['mimepart'] = $this->getMimetype($data['mimepart']); + if ($data['storage_mtime'] == 0) { + $data['storage_mtime'] = $data['mtime']; + } } return $data; @@ -148,13 +151,16 @@ class Cache { $fileId = $this->getId($folder); if ($fileId > -1) { $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `storage_mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC'); $result = $query->execute(array($fileId)); $files = $result->fetchAll(); foreach ($files as &$file) { $file['mimetype'] = $this->getMimetype($file['mimetype']); $file['mimepart'] = $this->getMimetype($file['mimepart']); + if ($file['storage_mtime'] == 0) { + $file['storage_mtime'] = $file['mtime']; + } } return $files; } else { @@ -226,7 +232,7 @@ class Cache { * @return array */ function buildParts(array $data) { - $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'etag'); + $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', 'etag'); $params = array(); $queryParts = array(); foreach ($data as $name => $value) { @@ -238,6 +244,11 @@ class Cache { $params[] = $this->getMimetypeId(substr($value, 0, strpos($value, '/'))); $queryParts[] = '`mimepart`'; $value = $this->getMimetypeId($value); + } elseif ($name === 'storage_mtime') { + if (!isset($data['mtime'])) { + $params[] = $value; + $queryParts[] = '`mtime`'; + } } $params[] = $value; $queryParts[] = '`' . $name . '`'; diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 9a5546dce3f..2623a167e9f 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -51,6 +51,7 @@ class Scanner { $data['size'] = $this->storage->filesize($path); } $data['etag'] = $this->storage->getETag($path); + $data['storage_mtime'] = $data['mtime']; return $data; } diff --git a/lib/files/cache/watcher.php b/lib/files/cache/watcher.php index 31059ec7f56..8bfd4602f3a 100644 --- a/lib/files/cache/watcher.php +++ b/lib/files/cache/watcher.php @@ -43,7 +43,7 @@ class Watcher { */ public function checkUpdate($path) { $cachedEntry = $this->cache->get($path); - if ($this->storage->hasUpdated($path, $cachedEntry['mtime'])) { + if ($this->storage->hasUpdated($path, $cachedEntry['storage_mtime'])) { if ($this->storage->is_dir($path)) { $this->scanner->scan($path, Scanner::SCAN_SHALLOW); } else { -- cgit v1.2.3 From 2fa34d6772ae578911dca17bbb0dc7a755024d05 Mon Sep 17 00:00:00 2001 From: Arthur Schiwon Date: Fri, 5 Apr 2013 11:27:08 +0200 Subject: Make FileCache upgrade more robust, fixes #2650 --- lib/files/cache/legacy.php | 22 +++++++++++++++++----- lib/files/cache/upgrade.php | 4 ++++ 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/legacy.php b/lib/files/cache/legacy.php index 9556a2639a3..b8e2548639b 100644 --- a/lib/files/cache/legacy.php +++ b/lib/files/cache/legacy.php @@ -80,7 +80,7 @@ class Legacy { } $result = $query->execute(array($path)); $data = $result->fetchRow(); - $data['etag'] = $this->getEtag($data['path']); + $data['etag'] = $this->getEtag($data['path'], $data['user']); return $data; } @@ -90,12 +90,24 @@ class Legacy { * @param type $path * @return string */ - function getEtag($path) { + function getEtag($path, $user = null) { static $query = null; - list(, $user, , $relativePath) = explode('/', $path, 4); - if (is_null($relativePath)) { + + $pathDetails = explode('/', $path, 4); + if((!$user) && !isset($pathDetails[1])) { + //no user!? Too odd, return empty string. + return ''; + } else if(!$user) { + //guess user from path, if no user passed. + $user = $pathDetails[1]; + } + + if(!isset($pathDetails[3]) || is_null($pathDetails[3])) { $relativePath = ''; + } else { + $relativePath = $pathDetails[3]; } + if(is_null($query)){ $query = \OC_DB::prepare('SELECT `propertyvalue` FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ? AND `propertyname` = \'{DAV:}getetag\''); } @@ -118,7 +130,7 @@ class Legacy { $result = $query->execute(array($id)); $data = $result->fetchAll(); foreach ($data as $i => $item) { - $data[$i]['etag'] = $this->getEtag($item['path']); + $data[$i]['etag'] = $this->getEtag($item['path'], $item['user']); } return $data; } diff --git a/lib/files/cache/upgrade.php b/lib/files/cache/upgrade.php index 797f4e6ba8c..ca044ba81de 100644 --- a/lib/files/cache/upgrade.php +++ b/lib/files/cache/upgrade.php @@ -127,6 +127,10 @@ class Upgrade { * @return array */ function getNewData($data) { + //Make sure there is a path, otherwise we can do nothing. + if(!isset($data['path'])) { + return false; + } $newData = $data; /** * @var \OC\Files\Storage\Storage $storage -- cgit v1.2.3 From 10be42f5b7b17ca55c242960885a9b50e217af3f Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 19 Apr 2013 15:03:59 +0200 Subject: Cache: only look for child entires when doing a move operation when moving a folder --- lib/files/cache/cache.php | 27 +++++++++++++++------------ tests/lib/files/cache/cache.php | 7 ++++--- 2 files changed, 19 insertions(+), 15 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 71b70abe3fe..d30c5cd16ed 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -205,7 +205,7 @@ class Cache { . ' VALUES(' . implode(', ', $valuesPlaceholder) . ')'); $result = $query->execute($params); if (\OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'Insert to cache failed: '.$result, \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'Insert to cache failed: ' . $result, \OCP\Util::ERROR); } return (int)\OC_DB::insertid('*PREFIX*filecache'); @@ -328,19 +328,22 @@ class Cache { * @param string $target */ public function move($source, $target) { - $sourceId = $this->getId($source); + $sourceData = $this->get($source); + $sourceId = $sourceData['fileid']; $newParentId = $this->getParentId($target); - //find all child entries - $query = \OC_DB::prepare('SELECT `path`, `fileid` FROM `*PREFIX*filecache` WHERE `path` LIKE ?'); - $result = $query->execute(array($source . '/%')); - $childEntries = $result->fetchAll(); - $sourceLength = strlen($source); - $query = \OC_DB::prepare('UPDATE `*PREFIX*filecache` SET `path` = ?, `path_hash` = ? WHERE `fileid` = ?'); - - foreach ($childEntries as $child) { - $targetPath = $target . substr($child['path'], $sourceLength); - $query->execute(array($targetPath, md5($targetPath), $child['fileid'])); + if ($sourceData['mimetype'] === 'httpd/unix-directory') { + //find all child entries + $query = \OC_DB::prepare('SELECT `path`, `fileid` FROM `*PREFIX*filecache` WHERE `path` LIKE ?'); + $result = $query->execute(array($source . '/%')); + $childEntries = $result->fetchAll(); + $sourceLength = strlen($source); + $query = \OC_DB::prepare('UPDATE `*PREFIX*filecache` SET `path` = ?, `path_hash` = ? WHERE `fileid` = ?'); + + foreach ($childEntries as $child) { + $targetPath = $target . substr($child['path'], $sourceLength); + $query->execute(array($targetPath, md5($targetPath), $child['fileid'])); + } } $query = \OC_DB::prepare('UPDATE `*PREFIX*filecache` SET `path` = ?, `path_hash` = ?, `name` = ?, `parent` =?' diff --git a/tests/lib/files/cache/cache.php b/tests/lib/files/cache/cache.php index 250842805e5..4051a6e234b 100644 --- a/tests/lib/files/cache/cache.php +++ b/tests/lib/files/cache/cache.php @@ -162,10 +162,11 @@ class Cache extends \PHPUnit_Framework_TestCase { $file4 = 'folder/foo/1'; $file5 = 'folder/foo/2'; $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'foo/bar'); + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); - $this->cache->put($file1, $data); - $this->cache->put($file2, $data); - $this->cache->put($file3, $data); + $this->cache->put($file1, $folderData); + $this->cache->put($file2, $folderData); + $this->cache->put($file3, $folderData); $this->cache->put($file4, $data); $this->cache->put($file5, $data); -- cgit v1.2.3 From 15dae6198fc4855519a0ed2347c29485a061f6aa Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sat, 20 Apr 2013 16:38:03 +0200 Subject: Cache: add a backgroundjob to check for external changes to the filesystem --- apps/files/appinfo/app.php | 4 +- lib/files/cache/backgroundwatcher.php | 73 +++++++++++++++++++++++++++++++++++ lib/files/cache/cache.php | 1 + lib/files/cache/permissions.php | 15 +++++++ tests/lib/files/cache/permissions.php | 6 ++- 5 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 lib/files/cache/backgroundwatcher.php (limited to 'lib/files/cache') diff --git a/apps/files/appinfo/app.php b/apps/files/appinfo/app.php index 703b1c7cb6c..05ab1722b3e 100644 --- a/apps/files/appinfo/app.php +++ b/apps/files/appinfo/app.php @@ -18,4 +18,6 @@ OC_Search::registerProvider('OC_Search_Provider_File'); \OC_Hook::connect('OC_Filesystem', 'post_write', '\OC\Files\Cache\Updater', 'writeHook'); \OC_Hook::connect('OC_Filesystem', 'post_touch', '\OC\Files\Cache\Updater', 'touchHook'); \OC_Hook::connect('OC_Filesystem', 'post_delete', '\OC\Files\Cache\Updater', 'deleteHook'); -\OC_Hook::connect('OC_Filesystem', 'post_rename', '\OC\Files\Cache\Updater', 'renameHook'); \ No newline at end of file +\OC_Hook::connect('OC_Filesystem', 'post_rename', '\OC\Files\Cache\Updater', 'renameHook'); + +\OC_BackgroundJob_RegularTask::register('\OC\Files\Cache\BackgroundWatcher', 'checkNext'); diff --git a/lib/files/cache/backgroundwatcher.php b/lib/files/cache/backgroundwatcher.php new file mode 100644 index 00000000000..3bcd447f53a --- /dev/null +++ b/lib/files/cache/backgroundwatcher.php @@ -0,0 +1,73 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Files\Cache; + +use \OC\Files\Mount; +use \OC\Files\Filesystem; + +class BackgroundWatcher { + static private function checkUpdate($id) { + $cacheItem = Cache::getById($id); + if (is_null($cacheItem)) { + return; + } + list($storageId, $internalPath) = $cacheItem; + $mounts = Mount::findByStorageId($storageId); + + if (count($mounts) === 0) { + //if the storage we need isn't mounted on default, try to find a user that has access to the storage + $permissionsCache = new Permissions($storageId); + $users = $permissionsCache->getUsers($id); + if (count($users) === 0) { + return; + } + Filesystem::initMountPoints($users[0]); + $mounts = Mount::findByStorageId($storageId); + if (count($mounts) === 0) { + return; + } + } + $storage = $mounts[0]->getStorage(); + $watcher = new Watcher($storage); + $watcher->checkUpdate($internalPath); + } + + /** + * get the next fileid in the cache + * + * @param int $previous + * @return int + */ + static private function getNextFileId($previous) { + $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `fileid` > ? ORDER BY `fileid` ASC', 1); + $result = $query->execute(array($previous)); + if ($row = $result->fetchRow()) { + return $row['fileid']; + } else { + return 0; + } + } + + static public function checkNext() { + $previous = \OC_Appconfig::getValue('files', 'backgroundwatcher_previous', 0); + $next = self::getNextFileId($previous); + error_log($next); + \OC_Appconfig::setValue('files', 'backgroundwatcher_previous', $next); + self::checkUpdate($next); + } + + static public function checkAll() { + $previous = 0; + $next = 1; + while ($next != 0) { + $next = self::getNextFileId($previous); + self::checkUpdate($next); + } + } +} diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 71b70abe3fe..3ed3490f29b 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -517,6 +517,7 @@ class Cache { /** * get the storage id of the storage for a file and the internal path of the file * + * @param int $id * @return array, first element holding the storage id, second the path */ static public function getById($id) { diff --git a/lib/files/cache/permissions.php b/lib/files/cache/permissions.php index a5c9c144054..faa5ff5eacc 100644 --- a/lib/files/cache/permissions.php +++ b/lib/files/cache/permissions.php @@ -107,4 +107,19 @@ class Permissions { $query->execute(array($fileId, $user)); } } + + /** + * get the list of users which have permissions stored for a file + * + * @param int $fileId + */ + public function getUsers($fileId) { + $query = \OC_DB::prepare('SELECT `user` FROM `*PREFIX*permissions` WHERE `fileid` = ?'); + $result = $query->execute(array($fileId)); + $users = array(); + while ($row = $result->fetchRow()) { + $users[] = $row['user']; + } + return $users; + } } diff --git a/tests/lib/files/cache/permissions.php b/tests/lib/files/cache/permissions.php index 56dbbc4518e..7e6e11e2eb2 100644 --- a/tests/lib/files/cache/permissions.php +++ b/tests/lib/files/cache/permissions.php @@ -14,8 +14,8 @@ class Permissions extends \PHPUnit_Framework_TestCase { */ private $permissionsCache; - function setUp(){ - $this->permissionsCache=new \OC\Files\Cache\Permissions('dummy'); + function setUp() { + $this->permissionsCache = new \OC\Files\Cache\Permissions('dummy'); } function testSimple() { @@ -23,8 +23,10 @@ class Permissions extends \PHPUnit_Framework_TestCase { $user = uniqid(); $this->assertEquals(-1, $this->permissionsCache->get(1, $user)); + $this->assertNotContains($user, $this->permissionsCache->getUsers(1)); $this->permissionsCache->set(1, $user, 1); $this->assertEquals(1, $this->permissionsCache->get(1, $user)); + $this->assertContains($user, $this->permissionsCache->getUsers(1)); $this->assertEquals(-1, $this->permissionsCache->get(2, $user)); $this->assertEquals(-1, $this->permissionsCache->get(1, $user . '2')); -- cgit v1.2.3 From eed5e9f804c0aac0467e1f502d86266ea232f350 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sat, 20 Apr 2013 16:56:47 +0200 Subject: Cache: check one folder and one file each time the backgroundwatcher runs Because there are usually way less folders than files it walks trought the list of all folder quicker, this causes new files to be detected quicker --- lib/files/cache/backgroundwatcher.php | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/backgroundwatcher.php b/lib/files/cache/backgroundwatcher.php index 3bcd447f53a..7549745e7d7 100644 --- a/lib/files/cache/backgroundwatcher.php +++ b/lib/files/cache/backgroundwatcher.php @@ -12,6 +12,18 @@ use \OC\Files\Mount; use \OC\Files\Filesystem; class BackgroundWatcher { + static $folderMimetype = null; + + static private function getFolderMimetype() { + if (!is_null(self::$folderMimetype)) { + return self::$folderMimetype; + } + $query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*mimetypes` WHERE `mimetype` = ?'); + $result = $query->execute(array('httpd/unix-directory')); + $row = $result->fetchRow(); + return $row['id']; + } + static private function checkUpdate($id) { $cacheItem = Cache::getById($id); if (is_null($cacheItem)) { @@ -42,10 +54,15 @@ class BackgroundWatcher { * get the next fileid in the cache * * @param int $previous + * @param bool $folder * @return int */ - static private function getNextFileId($previous) { - $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `fileid` > ? ORDER BY `fileid` ASC', 1); + static private function getNextFileId($previous, $folder) { + if ($folder) { + $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `fileid` > ? AND mimetype = ' . self::getFolderMimetype() . ' ORDER BY `fileid` ASC', 1); + } else { + $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `fileid` > ? AND mimetype != ' . self::getFolderMimetype() . ' ORDER BY `fileid` ASC', 1); + } $result = $query->execute(array($previous)); if ($row = $result->fetchRow()) { return $row['fileid']; @@ -55,18 +72,32 @@ class BackgroundWatcher { } static public function checkNext() { - $previous = \OC_Appconfig::getValue('files', 'backgroundwatcher_previous', 0); - $next = self::getNextFileId($previous); - error_log($next); - \OC_Appconfig::setValue('files', 'backgroundwatcher_previous', $next); - self::checkUpdate($next); + // check both 1 file and 1 folder, this way new files are detected quicker because there are less folders than files usually + $previousFile = \OC_Appconfig::getValue('files', 'backgroundwatcher_previous_file', 0); + $previousFolder = \OC_Appconfig::getValue('files', 'backgroundwatcher_previous_folder', 0); + $nextFile = self::getNextFileId($previousFile, false); + $nextFolder = self::getNextFileId($previousFolder, true); + \OC_Appconfig::setValue('files', 'backgroundwatcher_previous_file', $nextFile); + \OC_Appconfig::setValue('files', 'backgroundwatcher_previous_folder', $nextFolder); + if ($nextFile > 0) { + self::checkUpdate($nextFile); + } + if ($nextFolder > 0) { + self::checkUpdate($nextFolder); + } } static public function checkAll() { $previous = 0; $next = 1; while ($next != 0) { - $next = self::getNextFileId($previous); + $next = self::getNextFileId($previous, true); + self::checkUpdate($next); + } + $previous = 0; + $next = 1; + while ($next != 0) { + $next = self::getNextFileId($previous, false); self::checkUpdate($next); } } -- cgit v1.2.3 From fac288a4ad8f6db907c0a2ddfeb0b772fe616db5 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Thu, 25 Apr 2013 15:20:06 +0200 Subject: added unencrypted file size to file cache improved file size calculation and speeds --- apps/files_encryption/lib/proxy.php | 113 +++++++++++++---------------------- apps/files_encryption/lib/stream.php | 14 +++-- db_structure.xml | 8 +++ lib/files/cache/cache.php | 11 ++-- 4 files changed, 62 insertions(+), 84 deletions(-) (limited to 'lib/files/cache') diff --git a/apps/files_encryption/lib/proxy.php b/apps/files_encryption/lib/proxy.php index b805ec648d4..c07b9a8a7aa 100644 --- a/apps/files_encryption/lib/proxy.php +++ b/apps/files_encryption/lib/proxy.php @@ -108,7 +108,8 @@ class Proxy extends \OC_FileProxy { $size = strlen( $data ); // Disable encryption proxy to prevent recursive calls - \OC_FileProxy::$enabled = false; + $proxyStatus = \OC_FileProxy::$enabled; + \OC_FileProxy::$enabled = false; // Check if there is an existing key we can reuse if ( $encKeyfile = Keymanager::getFileKey( $rootView, $userId, $filePath ) ) { @@ -135,10 +136,8 @@ class Proxy extends \OC_FileProxy { // Fetch public keys for all users who will share the file $publicKeys = Keymanager::getPublicKeys( $rootView, $uniqueUserIds ); - - \OC_FileProxy::$enabled = false; - - // Encrypt plain keyfile to multiple sharefiles + + // Encrypt plain keyfile to multiple sharefiles $multiEncrypted = Crypt::multiKeyEncrypt( $plainKey, $publicKeys ); // Save sharekeys to user folders @@ -157,7 +156,7 @@ class Proxy extends \OC_FileProxy { \OC\Files\Filesystem::putFileInfo( $path, array( 'encrypted'=>true, 'size' => $size ), '' ); // Re-enable proxy - our work is done - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; } } @@ -182,9 +181,10 @@ class Proxy extends \OC_FileProxy { // TODO check for existing key file and reuse it if possible to avoid problems with versioning etc. // Disable encryption proxy to prevent recursive calls - \OC_FileProxy::$enabled = false; - - // If data is a catfile + $proxyStatus = \OC_FileProxy::$enabled; + \OC_FileProxy::$enabled = false; + + // If data is a catfile if ( Crypt::mode() == 'server' && Crypt::isCatfileContent( $data ) // TODO: Do we really need this check? Can't we assume it is properly encrypted? @@ -215,7 +215,7 @@ class Proxy extends \OC_FileProxy { } - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; if ( ! isset( $plainData ) ) { @@ -240,7 +240,8 @@ class Proxy extends \OC_FileProxy { $path = Keymanager::fixPartialFilePath( $path ); // Disable encryption proxy to prevent recursive calls - \OC_FileProxy::$enabled = false; + $proxyStatus = \OC_FileProxy::$enabled; + \OC_FileProxy::$enabled = false; $view = new \OC_FilesystemView( '/' ); @@ -265,7 +266,7 @@ class Proxy extends \OC_FileProxy { } - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; // If we don't return true then file delete will fail; better // to leave orphaned keyfiles than to disallow file deletion @@ -282,6 +283,7 @@ class Proxy extends \OC_FileProxy { { // Disable encryption proxy to prevent recursive calls + $proxyStatus = \OC_FileProxy::$enabled; \OC_FileProxy::$enabled = false; $view = new \OC_FilesystemView('/'); @@ -318,7 +320,7 @@ class Proxy extends \OC_FileProxy { // Rename keyfile so it isn't orphaned $result = $view->rename($oldKeyfilePath, $newKeyfilePath); - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; return $result; @@ -337,9 +339,10 @@ class Proxy extends \OC_FileProxy { $path_f = implode( '/', array_slice( $path_split, 3 ) ); // Disable encryption proxy to prevent recursive calls - \OC_FileProxy::$enabled = false; - - $meta = stream_get_meta_data( $result ); + $proxyStatus = \OC_FileProxy::$enabled; + \OC_FileProxy::$enabled = false; + + $meta = stream_get_meta_data( $result ); $view = new \OC_FilesystemView( '' ); @@ -369,13 +372,13 @@ class Proxy extends \OC_FileProxy { // NOTE: this is the case for new files saved via WebDAV - if ( - $view->file_exists( $path ) - and $view->filesize( $path ) > 0 - ) { - $x = $view->file_get_contents( $path ); - - $tmp = tmpfile(); +// if ( +// $view->file_exists( $path ) +// and $view->filesize( $path ) > 0 +// ) { +// $x = $view->file_get_contents( $path ); +// +// $tmp = tmpfile(); // // Make a temporary copy of the original file // \OCP\Files::streamCopy( $result, $tmp ); @@ -387,14 +390,14 @@ class Proxy extends \OC_FileProxy { // // fclose( $tmp ); - } +// } $result = fopen( 'crypt://'.$path_f, $meta['mode'] ); } // Re-enable the proxy - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; return $result; @@ -417,15 +420,15 @@ class Proxy extends \OC_FileProxy { // if path is a folder do nothing if(is_array($data) && array_key_exists('size', $data)) { // Disable encryption proxy to prevent recursive calls + $proxyStatus = \OC_FileProxy::$enabled; \OC_FileProxy::$enabled = false; + // get file size $data['size'] = self::postFileSize($path, $data['size']); // Re-enable the proxy - \OC_FileProxy::$enabled = true; - - trigger_error('postGetFileInfo '.$path.' size: '.$data['size']); + \OC_FileProxy::$enabled = $proxyStatus; } return $data; @@ -437,7 +440,7 @@ class Proxy extends \OC_FileProxy { $cached = \OC\Files\Filesystem::getFileInfo( $path, '' ); - $data['size'] = $cached['size']; + $data['size'] = $cached['unencrypted_size']; } @@ -453,56 +456,20 @@ class Proxy extends \OC_FileProxy { return $size; } + $path = Keymanager::fixPartialFilePath( $path ); + // Reformat path for use with OC_FSV $path_split = explode('/', $path); $path_f = implode('/', array_slice($path_split, 3)); - $userId = \OCP\User::getUser(); - $util = new Util( $view, $userId ); - - - // FIXME: is there a better solution to check if file belongs to files path? - // only get file size if file is in 'files' path - if (count($path_split) >= 2 && $path_split[2] == 'files' && $util->isEncryptedPath($path)) { - - // Disable encryption proxy to prevent recursive calls - \OC_FileProxy::$enabled = false; - - // open stream - $result = fopen('crypt://' . $path_f, "r"); - - if(is_resource($result)) { - // don't trust the given size, allways get the size from filesystem - $size = $view->filesize($path); - - // calculate last chunk nr - $lastChunckNr = floor($size / 8192); - - // calculate last chunk position - $lastChunckPos = ($lastChunckNr * 8192); - - // seek to end - fseek($result, $lastChunckPos); - - // get the content of the last chunck - $lastChunkContent = fgets($result); - - // calc the real file size with the size of the last chunk - $realSize = (($lastChunckNr * 6126) + strlen($lastChunkContent)); - - // set the size - $size = $realSize; - } - - // enable proxy - \OC_FileProxy::$enabled = true; - - return $size; + // get file info from database/cache + $fileInfo = \OC\Files\Filesystem::getFileInfo($path_f); + // if file is encrypted return real file size + if(is_array($fileInfo) && $fileInfo['encrypted'] == 1) { + return $fileInfo['unencrypted_size']; } else { - - return $size; - + return $fileInfo['size']; } } } diff --git a/apps/files_encryption/lib/stream.php b/apps/files_encryption/lib/stream.php index 9a37c3b08ee..7e42627f8ca 100644 --- a/apps/files_encryption/lib/stream.php +++ b/apps/files_encryption/lib/stream.php @@ -64,6 +64,7 @@ class Stream { private $count; private $writeCache; public $size; + public $unencryptedSize; private $publicKey; private $keyfile; private $encKeyfile; @@ -105,6 +106,7 @@ class Stream { } else { // Disable fileproxies so we can get the file size and open the source file without recursive encryption + $proxyStatus = \OC_FileProxy::$enabled; \OC_FileProxy::$enabled = false; if ( @@ -116,6 +118,7 @@ class Stream { // We're writing a new file so start write counter with 0 bytes $this->size = 0; + $this->unencryptedSize = 0; } else { @@ -129,7 +132,7 @@ class Stream { $this->handle = $this->rootView->fopen( $this->rawPath, $mode ); - \OC_FileProxy::$enabled = true; + \OC_FileProxy::$enabled = $proxyStatus; if ( ! is_resource( $this->handle ) ) { @@ -301,7 +304,7 @@ class Stream { // automatically attempted when the file is written to disk - // we are handling that separately here and we don't want to // get into an infinite loop - \OC_FileProxy::$enabled = false; + //\OC_FileProxy::$enabled = false; // Get the length of the unencrypted data that we are handling $length = strlen( $data ); @@ -438,7 +441,8 @@ class Stream { } $this->size = max( $this->size, $pointer + $length ); - + $this->unencryptedSize += $length; + return $length; } @@ -501,9 +505,7 @@ class Stream { $this->meta['mode']!='r' and $this->meta['mode']!='rb' ) { - - \OC\Files\Filesystem::putFileInfo( $this->relPath, array( 'encrypted' => true, 'size' => $this->size ), '' ); - + \OC\Files\Filesystem::putFileInfo( $this->relPath, array( 'encrypted' => 1, 'size' => $this->size, 'unencrypted_size' => $this->unencryptedSize ), '' ); } return fclose( $this->handle ); diff --git a/db_structure.xml b/db_structure.xml index dce90697b1c..366f51a82d0 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -288,6 +288,14 @@ 4 + + unencrypted_size + integer + + true + 8 + + etag text diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 71b70abe3fe..4e32ff2ba8a 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -117,7 +117,7 @@ class Cache { $params = array($file); } $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` ' . $where); $result = $query->execute($params); $data = $result->fetchRow(); @@ -133,6 +133,7 @@ class Cache { $data['size'] = (int)$data['size']; $data['mtime'] = (int)$data['mtime']; $data['encrypted'] = (bool)$data['encrypted']; + $data['unencrypted_size'] = (int)$data['unencrypted_size']; $data['storage'] = $this->storageId; $data['mimetype'] = $this->getMimetype($data['mimetype']); $data['mimepart'] = $this->getMimetype($data['mimepart']); @@ -151,7 +152,7 @@ class Cache { $fileId = $this->getId($folder); if ($fileId > -1) { $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC'); $result = $query->execute(array($fileId)); $files = $result->fetchAll(); @@ -234,7 +235,7 @@ class Cache { * @return array */ function buildParts(array $data) { - $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'etag'); + $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'unencrypted_size', 'etag'); $params = array(); $queryParts = array(); foreach ($data as $name => $value) { @@ -391,7 +392,7 @@ class Cache { */ public function search($pattern) { $query = \OC_DB::prepare(' - SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?' ); $result = $query->execute(array($pattern, $this->numericId)); @@ -417,7 +418,7 @@ class Cache { $where = '`mimepart` = ?'; } $query = \OC_DB::prepare(' - SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` + SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?' ); $mimetype = $this->getMimetypeId($mimetype); -- cgit v1.2.3 From bcd9a6903317a786f5f947374a8680130e4a8297 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 26 Apr 2013 00:00:18 +0200 Subject: Cache: seperate handing of numeric storage id's to it's own class --- lib/files/cache/cache.php | 47 +++++++++++++--------------------- lib/files/cache/storage.php | 59 +++++++++++++++++++++++++++++++++++++++++++ lib/files/storage/common.php | 10 +++++++- lib/files/storage/storage.php | 5 ++++ 4 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 lib/files/cache/storage.php (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 71b70abe3fe..857fe980be6 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -30,11 +30,9 @@ class Cache { private $storageId; /** - * numeric storage id - * - * @var int $numericId + * @var Storage $storageCache */ - private $numericId; + private $storageCache; private $mimetypeIds = array(); private $mimetypes = array(); @@ -52,19 +50,11 @@ class Cache { $this->storageId = md5($this->storageId); } - $query = \OC_DB::prepare('SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'); - $result = $query->execute(array($this->storageId)); - if ($row = $result->fetchRow()) { - $this->numericId = $row['numeric_id']; - } else { - $query = \OC_DB::prepare('INSERT INTO `*PREFIX*storages`(`id`) VALUES(?)'); - $query->execute(array($this->storageId)); - $this->numericId = \OC_DB::insertid('*PREFIX*storages'); - } + $this->storageCache = new Storage($storage); } public function getNumericStorageId() { - return $this->numericId; + return $this->storageCache->getNumericId(); } /** @@ -111,7 +101,7 @@ class Cache { public function get($file) { if (is_string($file) or $file == '') { $where = 'WHERE `storage` = ? AND `path_hash` = ?'; - $params = array($this->numericId, md5($file)); + $params = array($this->getNumericStorageId(), md5($file)); } else { //file id $where = 'WHERE `fileid` = ?'; $params = array($file); @@ -198,14 +188,14 @@ class Cache { list($queryParts, $params) = $this->buildParts($data); $queryParts[] = '`storage`'; - $params[] = $this->numericId; + $params[] = $this->getNumericStorageId(); $valuesPlaceholder = array_fill(0, count($queryParts), '?'); $query = \OC_DB::prepare('INSERT INTO `*PREFIX*filecache`(' . implode(', ', $queryParts) . ')' . ' VALUES(' . implode(', ', $valuesPlaceholder) . ')'); $result = $query->execute($params); if (\OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'Insert to cache failed: '.$result, \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'Insert to cache failed: ' . $result, \OCP\Util::ERROR); } return (int)\OC_DB::insertid('*PREFIX*filecache'); @@ -264,7 +254,7 @@ class Cache { $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); - $result = $query->execute(array($this->numericId, $pathHash)); + $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); if ($row = $result->fetchRow()) { return $row['fileid']; @@ -353,7 +343,7 @@ class Cache { */ public function clear() { $query = \OC_DB::prepare('DELETE FROM `*PREFIX*filecache` WHERE storage = ?'); - $query->execute(array($this->numericId)); + $query->execute(array($this->getNumericStorageId())); $query = \OC_DB::prepare('DELETE FROM `*PREFIX*storages` WHERE id = ?'); $query->execute(array($this->storageId)); @@ -367,7 +357,7 @@ class Cache { public function getStatus($file) { $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); - $result = $query->execute(array($this->numericId, $pathHash)); + $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); if ($row = $result->fetchRow()) { if ((int)$row['size'] === -1) { return self::SHALLOW; @@ -394,7 +384,7 @@ class Cache { SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?' ); - $result = $query->execute(array($pattern, $this->numericId)); + $result = $query->execute(array($pattern, $this->getNumericStorageId())); $files = array(); while ($row = $result->fetchRow()) { $row['mimetype'] = $this->getMimetype($row['mimetype']); @@ -421,7 +411,7 @@ class Cache { FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?' ); $mimetype = $this->getMimetypeId($mimetype); - $result = $query->execute(array($mimetype, $this->numericId)); + $result = $query->execute(array($mimetype, $this->getNumericStorageId())); $files = array(); while ($row = $result->fetchRow()) { $row['mimetype'] = $this->getMimetype($row['mimetype']); @@ -459,7 +449,7 @@ class Cache { return 0; } $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `parent` = ? AND `storage` = ?'); - $result = $query->execute(array($id, $this->numericId)); + $result = $query->execute(array($id, $this->getNumericStorageId())); $totalSize = 0; $hasChilds = 0; while ($row = $result->fetchRow()) { @@ -486,7 +476,7 @@ class Cache { */ public function getAll() { $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ?'); - $result = $query->execute(array($this->numericId)); + $result = $query->execute(array($this->getNumericStorageId())); $ids = array(); while ($row = $result->fetchRow()) { $ids[] = $row['fileid']; @@ -506,7 +496,7 @@ class Cache { public function getIncomplete() { $query = \OC_DB::prepare('SELECT `path` FROM `*PREFIX*filecache`' . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC LIMIT 1'); - $result = $query->execute(array($this->numericId)); + $result = $query->execute(array($this->getNumericStorageId())); if ($row = $result->fetchRow()) { return $row['path']; } else { @@ -517,6 +507,7 @@ class Cache { /** * get the storage id of the storage for a file and the internal path of the file * + * @param int $id * @return array, first element holding the storage id, second the path */ static public function getById($id) { @@ -529,10 +520,8 @@ class Cache { return null; } - $query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*storages` WHERE `numeric_id` = ?'); - $result = $query->execute(array($numericId)); - if ($row = $result->fetchRow()) { - return array($row['id'], $path); + if ($id = Storage::getStorageId($numericId)) { + return array($id, $path); } else { return null; } diff --git a/lib/files/cache/storage.php b/lib/files/cache/storage.php new file mode 100644 index 00000000000..72de376798c --- /dev/null +++ b/lib/files/cache/storage.php @@ -0,0 +1,59 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Files\Cache; + +/** + * Class Storage + * + * cache storage specific data + * + * @package OC\Files\Cache + */ +class Storage { + private $storageId; + private $numericId; + + /** + * @param \OC\Files\Storage\Storage|string $storage + */ + public function __construct($storage) { + if ($storage instanceof \OC\Files\Storage\Storage) { + $this->storageId = $storage->getId(); + } else { + $this->storageId = $storage; + } + if (strlen($this->storageId) > 64) { + $this->storageId = md5($this->storageId); + } + + $query = \OC_DB::prepare('SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'); + $result = $query->execute(array($this->storageId)); + if ($row = $result->fetchRow()) { + $this->numericId = $row['numeric_id']; + } else { + $query = \OC_DB::prepare('INSERT INTO `*PREFIX*storages`(`id`) VALUES(?)'); + $query->execute(array($this->storageId)); + $this->numericId = \OC_DB::insertid('*PREFIX*storages'); + } + } + + public function getNumericId() { + return $this->numericId; + } + + public static function getStorageId($numericId) { + $query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*storages` WHERE `numeric_id` = ?'); + $result = $query->execute(array($numericId)); + if ($row = $result->fetchRow()) { + return $row['id']; + } else { + return null; + } + } +} diff --git a/lib/files/storage/common.php b/lib/files/storage/common.php index 38fe5e546f6..e87fe3b5239 100644 --- a/lib/files/storage/common.php +++ b/lib/files/storage/common.php @@ -25,6 +25,7 @@ abstract class Common implements \OC\Files\Storage\Storage { private $scanner; private $permissioncache; private $watcher; + private $storageCache; public function __construct($parameters) { } @@ -300,6 +301,13 @@ abstract class Common implements \OC\Files\Storage\Storage { return $this->watcher; } + public function getStorageCache(){ + if (!isset($this->storageCache)) { + $this->storageCache = new \OC\Files\Cache\Storage($this); + } + return $this->storageCache; + } + /** * get the owner of a path * @@ -361,7 +369,7 @@ abstract class Common implements \OC\Files\Storage\Storage { * get the free space in the storage * * @param $path - * return int + * @return int */ public function free_space($path) { return \OC\Files\FREE_SPACE_UNKNOWN; diff --git a/lib/files/storage/storage.php b/lib/files/storage/storage.php index 1da82da2163..c96caebf4af 100644 --- a/lib/files/storage/storage.php +++ b/lib/files/storage/storage.php @@ -328,6 +328,11 @@ interface Storage { */ public function getWatcher($path = ''); + /** + * @return \OC\Files\Cache\Storage + */ + public function getStorageCache(); + /** * get the ETag for a file or folder * -- cgit v1.2.3 From f55aaad858396484d35f87ca09e5f53e9848ddf6 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Mon, 29 Apr 2013 15:43:48 +0200 Subject: fix for infinite loop causing on files_encryption branch when testing "apps/files_encryption/test/crypt.php" on Method testSymmetricStreamEncryptShortFileContent --- lib/files/cache/cache.php | 2 +- lib/files/cache/scanner.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 857fe980be6..47f3c272b13 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -430,7 +430,7 @@ class Cache { $this->calculateFolderSize($path); if ($path !== '') { $parent = dirname($path); - if ($parent === '.') { + if ($parent === '.' or $parent === '/') { $parent = ''; } $this->correctFolderSize($parent); diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index f019d4fc608..5241acec1ee 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -68,7 +68,7 @@ class Scanner { if ($data) { if ($file) { $parent = dirname($file); - if ($parent === '.') { + if ($parent === '.' or $parent === '/') { $parent = ''; } if (!$this->cache->inCache($parent)) { -- cgit v1.2.3 From 0e30e68b22bf3db2872dcff50fe1c0186aa1c57e Mon Sep 17 00:00:00 2001 From: Björn Schießle Date: Wed, 8 May 2013 15:49:45 +0200 Subject: update etag for for the touched file --- lib/files/cache/updater.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/updater.php b/lib/files/cache/updater.php index 92a16d9d9b6..e054b9dafcf 100644 --- a/lib/files/cache/updater.php +++ b/lib/files/cache/updater.php @@ -132,7 +132,15 @@ class Updater { * @param array $params */ static public function touchHook($params) { - self::writeUpdate($params['path']); + $path = $params['path']; + list($storage, $internalPath) = self::resolvePath($path); + $cache = $storage->getCache(); + $id = $cache->getId($internalPath); + if ($id !== -1) { + $cache->update($id, array('etag' => $storage->getETag($internalPath))); + self::correctFolder($parent, $time); + } + self::writeUpdate($path); } /** -- cgit v1.2.3 From 2e81efc37eeb95b7e5d392349ad393bb94d6928a Mon Sep 17 00:00:00 2001 From: Björn Schießle Date: Wed, 8 May 2013 22:57:08 +0200 Subject: don't call correctFolder() in touchHook, it will be called later in the writeUpdate() --- lib/files/cache/updater.php | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/updater.php b/lib/files/cache/updater.php index e054b9dafcf..417a47f3830 100644 --- a/lib/files/cache/updater.php +++ b/lib/files/cache/updater.php @@ -138,7 +138,6 @@ class Updater { $id = $cache->getId($internalPath); if ($id !== -1) { $cache->update($id, array('etag' => $storage->getETag($internalPath))); - self::correctFolder($parent, $time); } self::writeUpdate($path); } -- cgit v1.2.3 From c3b0d3d38cb2cfe1501a28081dde34dbec2d2621 Mon Sep 17 00:00:00 2001 From: Jörn Friedrich Dreyer Date: Fri, 10 May 2013 12:00:13 +0200 Subject: rename isIgnoredFile to isPartialFile, remove check of blacklisted files in isPartialFile, correct usage of isPartialFile and isFileBlacklisted --- lib/files/cache/scanner.php | 10 +++++----- lib/files/view.php | 16 +++++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 5241acec1ee..661bc486330 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -62,7 +62,9 @@ class Scanner { * @return array with metadata of the scanned file */ public function scanFile($file, $checkExisting = false) { - if (!self::isIgnoredFile($file)) { + if ( ! self::isPartialFile($file) + and ! \OC\Files\Filesystem::isFileBlacklisted($file) + ) { \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); $data = $this->getData($file); if ($data) { @@ -166,10 +168,8 @@ class Scanner { * @param String $file * @return boolean */ - public static function isIgnoredFile($file) { - if (pathinfo($file, PATHINFO_EXTENSION) === 'part' - || \OC\Files\Filesystem::isFileBlacklisted($file) - ) { + public static function isPartialFile($file) { + if (pathinfo($file, PATHINFO_EXTENSION) === 'part') { return true; } return false; diff --git a/lib/files/view.php b/lib/files/view.php index f89b7f66ffd..35053e0729c 100644 --- a/lib/files/view.php +++ b/lib/files/view.php @@ -263,12 +263,13 @@ class View { if (is_resource($data)) { //not having to deal with streams in file_put_contents makes life easier $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); if (\OC_FileProxy::runPreProxies('file_put_contents', $absolutePath, $data) - && Filesystem::isValidPath($path) + and Filesystem::isValidPath($path) + and ! Filesystem::isFileBlacklisted($path) ) { $path = $this->getRelativePath($absolutePath); $exists = $this->file_exists($path); $run = true; - if ($this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isIgnoredFile($path)) { + if ($this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isPartialFile($path)) { if (!$exists) { \OC_Hook::emit( Filesystem::CLASSNAME, @@ -296,7 +297,7 @@ class View { list ($count, $result) = \OC_Helper::streamCopy($data, $target); fclose($target); fclose($data); - if ($this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isIgnoredFile($path)) { + if ($this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isPartialFile($path)) { if (!$exists) { \OC_Hook::emit( Filesystem::CLASSNAME, @@ -340,6 +341,7 @@ class View { \OC_FileProxy::runPreProxies('rename', $absolutePath1, $absolutePath2) and Filesystem::isValidPath($path2) and Filesystem::isValidPath($path1) + and ! Filesystem::isFileBlacklisted($path2) ) { $path1 = $this->getRelativePath($absolutePath1); $path2 = $this->getRelativePath($absolutePath2); @@ -404,6 +406,7 @@ class View { \OC_FileProxy::runPreProxies('copy', $absolutePath1, $absolutePath2) and Filesystem::isValidPath($path2) and Filesystem::isValidPath($path1) + and ! Filesystem::isFileBlacklisted($path2) ) { $path1 = $this->getRelativePath($absolutePath1); $path2 = $this->getRelativePath($absolutePath2); @@ -606,7 +609,10 @@ class View { private function basicOperation($operation, $path, $hooks = array(), $extraParam = null) { $postFix = (substr($path, -1, 1) === '/') ? '/' : ''; $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); - if (\OC_FileProxy::runPreProxies($operation, $absolutePath, $extraParam) and Filesystem::isValidPath($path)) { + if (\OC_FileProxy::runPreProxies($operation, $absolutePath, $extraParam) + and Filesystem::isValidPath($path) + and ! Filesystem::isFileBlacklisted($path) + ) { $path = $this->getRelativePath($absolutePath); if ($path == null) { return false; @@ -635,7 +641,7 @@ class View { private function runHooks($hooks, $path, $post = false) { $prefix = ($post) ? 'post_' : ''; $run = true; - if (Filesystem::$loaded and $this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isIgnoredFile($path)) { + if (Filesystem::$loaded and $this->fakeRoot == Filesystem::getRoot()) { foreach ($hooks as $hook) { if ($hook != 'read') { \OC_Hook::emit( -- cgit v1.2.3 From 71eed76dbef92363d5f6eeb423496c6b8dac0579 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Mon, 13 May 2013 11:17:08 -0400 Subject: Prevent backgroundScan() from looping if opendir() is failing for the same path --- lib/files/cache/scanner.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 661bc486330..b99dea23cbc 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -179,9 +179,11 @@ class Scanner { * walk over any folders that are not fully scanned yet and scan them */ public function backgroundScan() { - while (($path = $this->cache->getIncomplete()) !== false) { + $lastPath = null; + while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { $this->scan($path); $this->cache->correctFolderSize($path); + $lastPath = $path; } } } -- cgit v1.2.3 From d91161186b87965eab165ea536e026dde47df88a Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Wed, 15 May 2013 09:33:38 -0400 Subject: Fix finding mount in background watcher, fixes #3353 --- lib/files/cache/backgroundwatcher.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/backgroundwatcher.php b/lib/files/cache/backgroundwatcher.php index 7549745e7d7..8933101577d 100644 --- a/lib/files/cache/backgroundwatcher.php +++ b/lib/files/cache/backgroundwatcher.php @@ -30,7 +30,7 @@ class BackgroundWatcher { return; } list($storageId, $internalPath) = $cacheItem; - $mounts = Mount::findByStorageId($storageId); + $mounts = Filesystem::getMountByStorageId($storageId); if (count($mounts) === 0) { //if the storage we need isn't mounted on default, try to find a user that has access to the storage @@ -40,7 +40,7 @@ class BackgroundWatcher { return; } Filesystem::initMountPoints($users[0]); - $mounts = Mount::findByStorageId($storageId); + $mounts = Filesystem::getMountByStorageId($storageId); if (count($mounts) === 0) { return; } -- cgit v1.2.3 From 41e2d64c86fffc3e507a1ad0788bcb498db2c640 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 19 May 2013 14:15:49 -0400 Subject: Add support for copying/moving folders between storages, move isIgnoredDir() to Filesystem --- lib/files/cache/scanner.php | 14 +------------- lib/files/filesystem.php | 13 +++++++++++++ lib/files/view.php | 32 ++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 21 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 661bc486330..0b1947f17ca 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -115,7 +115,7 @@ class Scanner { \OC_DB::beginTransaction(); while ($file = readdir($dh)) { $child = ($path) ? $path . '/' . $file : $file; - if (!$this->isIgnoredDir($file)) { + if (!\OC\Files\Filesystem::isIgnoredDir($file)) { $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); if ($data) { if ($data['size'] === -1) { @@ -149,18 +149,6 @@ class Scanner { return $size; } - /** - * @brief check if the directory should be ignored when scanning - * NOTE: the special directories . and .. would cause never ending recursion - * @param String $dir - * @return boolean - */ - private function isIgnoredDir($dir) { - if ($dir === '.' || $dir === '..') { - return true; - } - return false; - } /** * @brief check if the file should be ignored when scanning * NOTE: files with a '.part' extension are ignored as well! diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index d60d430d77c..99d87011df2 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -453,6 +453,19 @@ class Filesystem { return (in_array($filename, $blacklist)); } + /** + * @brief check if the directory should be ignored when scanning + * NOTE: the special directories . and .. would cause never ending recursion + * @param String $dir + * @return boolean + */ + static public function isIgnoredDir($dir) { + if ($dir === '.' || $dir === '..') { + return true; + } + return false; + } + /** * following functions are equivalent to their php builtin equivalents for arguments/return values. */ diff --git a/lib/files/view.php b/lib/files/view.php index f35e1e3dc16..875a6c1a1f8 100644 --- a/lib/files/view.php +++ b/lib/files/view.php @@ -372,11 +372,18 @@ class View { $result = false; } } else { - $source = $this->fopen($path1 . $postFix1, 'r'); - $target = $this->fopen($path2 . $postFix2, 'w'); - list($count, $result) = \OC_Helper::streamCopy($source, $target); - list($storage1, $internalPath1) = Filesystem::resolvePath($absolutePath1 . $postFix1); - $storage1->unlink($internalPath1); + if ($this->is_dir($path1)) { + $result = $this->copy($path1, $path2); + if ($result === true) { + $result = $this->deleteAll($path1); + } + } else { + $source = $this->fopen($path1 . $postFix1, 'r'); + $target = $this->fopen($path2 . $postFix2, 'w'); + list($count, $result) = \OC_Helper::streamCopy($source, $target); + list($storage1, $internalPath1) = Filesystem::resolvePath($absolutePath1 . $postFix1); + $storage1->unlink($internalPath1); + } } if ($this->fakeRoot == Filesystem::getRoot() && !Cache\Scanner::isPartialFile($path1)) { \OC_Hook::emit( @@ -459,9 +466,18 @@ class View { $result = false; } } else { - $source = $this->fopen($path1 . $postFix1, 'r'); - $target = $this->fopen($path2 . $postFix2, 'w'); - list($count, $result) = \OC_Helper::streamCopy($source, $target); + if ($this->is_dir($path1) && ($dh = $this->opendir($path1))) { + $this->mkdir($path2); + while ($file = readdir($dh)) { + if (!Filesystem::isIgnoredDir($file)) { + $result = $this->copy($path1 . '/' . $file, $path2 . '/' . $file); + } + } + } else { + $source = $this->fopen($path1 . $postFix1, 'r'); + $target = $this->fopen($path2 . $postFix2, 'w'); + list($count, $result) = \OC_Helper::streamCopy($source, $target); + } } if ($this->fakeRoot == Filesystem::getRoot()) { \OC_Hook::emit( -- cgit v1.2.3 From 079f918d5ca0d242e77717aaeac82bcf011dc745 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sun, 19 May 2013 22:30:03 +0200 Subject: fix for webdav and wrong reference for findByStorageId --- lib/files/cache/backgroundwatcher.php | 2 +- lib/files/filesystem.php | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/backgroundwatcher.php b/lib/files/cache/backgroundwatcher.php index 7549745e7d7..b5770d0582b 100644 --- a/lib/files/cache/backgroundwatcher.php +++ b/lib/files/cache/backgroundwatcher.php @@ -30,7 +30,7 @@ class BackgroundWatcher { return; } list($storageId, $internalPath) = $cacheItem; - $mounts = Mount::findByStorageId($storageId); + $mounts = Mount\Manager::findByStorageId($storageId); if (count($mounts) === 0) { //if the storage we need isn't mounted on default, try to find a user that has access to the storage diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index d60d430d77c..d0cac9dc1d3 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -236,7 +236,9 @@ class Filesystem { } static public function initMounts(){ - self::$mounts = new Mount\Manager(); + if(!self::$mounts) { + self::$mounts = new Mount\Manager(); + } } /** -- cgit v1.2.3 From b3b6738d599480fc9bf40a53313598c4766571fb Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Mon, 20 May 2013 01:47:35 +0200 Subject: Revert "fix for webdav and wrong reference for findByStorageId" This reverts commit 079f918d5ca0d242e77717aaeac82bcf011dc745. --- lib/files/cache/backgroundwatcher.php | 2 +- lib/files/filesystem.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/backgroundwatcher.php b/lib/files/cache/backgroundwatcher.php index b5770d0582b..7549745e7d7 100644 --- a/lib/files/cache/backgroundwatcher.php +++ b/lib/files/cache/backgroundwatcher.php @@ -30,7 +30,7 @@ class BackgroundWatcher { return; } list($storageId, $internalPath) = $cacheItem; - $mounts = Mount\Manager::findByStorageId($storageId); + $mounts = Mount::findByStorageId($storageId); if (count($mounts) === 0) { //if the storage we need isn't mounted on default, try to find a user that has access to the storage diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index d0cac9dc1d3..d60d430d77c 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -236,9 +236,7 @@ class Filesystem { } static public function initMounts(){ - if(!self::$mounts) { - self::$mounts = new Mount\Manager(); - } + self::$mounts = new Mount\Manager(); } /** -- cgit v1.2.3 From 2ea2abf11e31904db187dfe9975cc34247fde1d7 Mon Sep 17 00:00:00 2001 From: Roland Hager Date: Thu, 16 May 2013 17:18:07 +0200 Subject: Fixing UPDATE error in filecache table when renaming files by calling move(). Add storage id to the where clause to avoid updating entries of other users. --- lib/files/cache/cache.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 0617471079b..0e7a96aaca8 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -335,8 +335,8 @@ class Cache { if ($sourceData['mimetype'] === 'httpd/unix-directory') { //find all child entries - $query = \OC_DB::prepare('SELECT `path`, `fileid` FROM `*PREFIX*filecache` WHERE `path` LIKE ?'); - $result = $query->execute(array($source . '/%')); + $query = \OC_DB::prepare('SELECT `path`, `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path` LIKE ?'); + $result = $query->execute(array($this->getNumericStorageId(), $source . '/%')); $childEntries = $result->fetchAll(); $sourceLength = strlen($source); $query = \OC_DB::prepare('UPDATE `*PREFIX*filecache` SET `path` = ?, `path_hash` = ? WHERE `fileid` = ?'); -- cgit v1.2.3 From eceb3c8ed5f3068808bc8fc5ceec5fda1ac50db9 Mon Sep 17 00:00:00 2001 From: Jörn Friedrich Dreyer Date: Mon, 29 Apr 2013 12:25:27 +0200 Subject: add debug output --- lib/app.php | 5 ++++- lib/db.php | 9 ++------- lib/files/cache/cache.php | 15 ++++++++++++--- lib/setup.php | 12 ++++++++++-- lib/user.php | 2 +- lib/user/database.php | 2 +- 6 files changed, 30 insertions(+), 15 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/app.php b/lib/app.php index 55b4543ec9f..0a7069ca60c 100644 --- a/lib/app.php +++ b/lib/app.php @@ -173,8 +173,11 @@ class OC_App{ } $apps=array('files'); $query = OC_DB::prepare( 'SELECT `appid` FROM `*PREFIX*appconfig`' - .' WHERE `configkey` = \'enabled\' AND `configvalue`=\'yes\'' ); + .' WHERE `configkey` = \'enabled\' AND to_char(`configvalue`)=\'yes\'' ); $result=$query->execute(); + if( \OC_DB::isError($result)) { + throw new DatabaseException($result->getMessage(), $query); + } while($row=$result->fetchRow()) { if(array_search($row['appid'], $apps)===false) { $apps[]=$row['appid']; diff --git a/lib/db.php b/lib/db.php index 8f6f50bda6e..5b45f81f998 100644 --- a/lib/db.php +++ b/lib/db.php @@ -276,15 +276,10 @@ class OC_DB { 'phptype' => 'oci8', 'username' => $user, 'password' => $pass, + 'service' => $name, + 'hostspec' => $host, 'charset' => 'AL32UTF8', ); - if ($host != '') { - $dsn['hostspec'] = $host; - $dsn['database'] = $name; - } else { // use dbname for hostspec - $dsn['hostspec'] = $name; - $dsn['database'] = $user; - } break; case 'mssql': $dsn = array( diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 0617471079b..adffe766ddd 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -145,8 +145,11 @@ class Cache { if ($fileId > -1) { $query = \OC_DB::prepare( 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `storage_mtime`, `encrypted`, `etag` - FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC'); + FROM `*PREFIX*filecache` WHERE `parent` = ? ORDER BY `name` ASC'); $result = $query->execute(array($fileId)); + if (\OC_DB::isError($result)) { + \OCP\Util::writeLog('cache', 'getFolderContents failed: '.$result->getMessage(), \OCP\Util::ERROR); + } $files = $result->fetchAll(); foreach ($files as &$file) { $file['mimetype'] = $this->getMimetype($file['mimetype']); @@ -201,7 +204,7 @@ class Cache { . ' VALUES(' . implode(', ', $valuesPlaceholder) . ')'); $result = $query->execute($params); if (\OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'Insert to cache failed: ' . $result, \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'Insert to cache failed: ' . $result->getMessage(), \OCP\Util::ERROR); } return (int)\OC_DB::insertid('*PREFIX*filecache'); @@ -372,6 +375,9 @@ class Cache { $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); + if( \OC_DB::isError($result)) { + \OCP\Util::writeLog('cache', 'get status failed: '.$result->getMessage(), \OCP\Util::ERROR); + } if ($row = $result->fetchRow()) { if ((int)$row['size'] === -1) { return self::SHALLOW; @@ -509,8 +515,11 @@ class Cache { */ public function getIncomplete() { $query = \OC_DB::prepare('SELECT `path` FROM `*PREFIX*filecache`' - . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC LIMIT 1'); + . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC',1); $result = $query->execute(array($this->getNumericStorageId())); + if (\OC_DB::isError($result)) { + \OCP\Util::writeLog('cache', 'getIncomplete failed: '.$result->getMessage(), \OCP\Util::ERROR); + } if ($row = $result->fetchRow()) { return $row['path']; } else { diff --git a/lib/setup.php b/lib/setup.php index f1ac6b8b2b8..6f608ec5fb1 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -152,8 +152,12 @@ class OC_Setup { self::setupOCIDatabase($dbhost, $dbuser, $dbpass, $dbname, $dbtableprefix, $dbtablespace, $username); } catch (Exception $e) { $error[] = array( - 'error' => $l->t('Oracle username and/or password not valid'), - 'hint' => $l->t('You need to enter either an existing account or the administrator.') + 'error' => $l->t('Oracle connection could not be established'), + 'hint' => $e->getMessage().' Check environment: ORACLE_HOME='.getenv('ORACLE_HOME') + .' ORACLE_SID='.getenv('ORACLE_SID') + .' LD_LIBRARY_PATH='.getenv('LD_LIBRARY_PATH') + .' NLS_LANG='.getenv('NLS_LANG') + .' tnsnames.ora is '.(is_readable(getenv('ORACLE_HOME').'/network/admin/tnsnames.ora')?'':'not ').'readable' ); return $error; } @@ -452,9 +456,13 @@ class OC_Setup { } else { $easy_connect_string = '//'.$e_host.'/'.$e_dbname; } + \OC_Log::write('setup oracle', 'connect string: '.$easy_connect_string, \OC_Log::DEBUG); $connection = @oci_connect($dbuser, $dbpass, $easy_connect_string); if(!$connection) { $e = oci_error(); + if (is_array ($e) && isset ($e['message'])) { + throw new Exception($e['message']); + } throw new Exception($l->t('Oracle username and/or password not valid')); } //check for roles creation rights in oracle diff --git a/lib/user.php b/lib/user.php index b607874afaf..78f5edfb5f6 100644 --- a/lib/user.php +++ b/lib/user.php @@ -609,7 +609,7 @@ class OC_User { */ public static function isEnabled($userid) { $sql = 'SELECT `userid` FROM `*PREFIX*preferences`' - .' WHERE `userid` = ? AND `appid` = ? AND `configkey` = ? AND `configvalue` = ?'; + .' WHERE `userid` = ? AND `appid` = ? AND `configkey` = ? AND to_char(`configvalue`) = ?'; $stmt = OC_DB::prepare($sql); if ( ! OC_DB::isError($stmt) ) { $result = $stmt->execute(array($userid, 'core', 'enabled', 'false')); diff --git a/lib/user/database.php b/lib/user/database.php index 63c64ed43d3..d70b620f2ab 100644 --- a/lib/user/database.php +++ b/lib/user/database.php @@ -136,7 +136,7 @@ class OC_User_Database extends OC_User_Backend { */ public function getDisplayName($uid) { if( $this->userExists($uid) ) { - $query = OC_DB::prepare( 'SELECT displayname FROM `*PREFIX*users` WHERE `uid` = ?' ); + $query = OC_DB::prepare( 'SELECT `displayname` FROM `*PREFIX*users` WHERE `uid` = ?' ); $result = $query->execute( array( $uid ))->fetchAll(); $displayName = trim($result[0]['displayname'], ' '); if ( !empty($displayName) ) { -- cgit v1.2.3 From 1f1abe595d413883f060f5f2983bac654e0ec7ef Mon Sep 17 00:00:00 2001 From: Jörn Friedrich Dreyer Date: Wed, 8 May 2013 16:29:15 +0200 Subject: cleanup codestyle --- lib/files/cache/cache.php | 6 +++--- lib/setup.php | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index adffe766ddd..3341fe50525 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -148,7 +148,7 @@ class Cache { FROM `*PREFIX*filecache` WHERE `parent` = ? ORDER BY `name` ASC'); $result = $query->execute(array($fileId)); if (\OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'getFolderContents failed: '.$result->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'getFolderContents failed: ' . $result->getMessage(), \OCP\Util::ERROR); } $files = $result->fetchAll(); foreach ($files as &$file) { @@ -376,7 +376,7 @@ class Cache { $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); if( \OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'get status failed: '.$result->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'get status failed: ' . $result->getMessage(), \OCP\Util::ERROR); } if ($row = $result->fetchRow()) { if ((int)$row['size'] === -1) { @@ -518,7 +518,7 @@ class Cache { . ' WHERE `storage` = ? AND `size` = -1 ORDER BY `fileid` DESC',1); $result = $query->execute(array($this->getNumericStorageId())); if (\OC_DB::isError($result)) { - \OCP\Util::writeLog('cache', 'getIncomplete failed: '.$result->getMessage(), \OCP\Util::ERROR); + \OCP\Util::writeLog('cache', 'getIncomplete failed: ' . $result->getMessage(), \OCP\Util::ERROR); } if ($row = $result->fetchRow()) { return $row['path']; diff --git a/lib/setup.php b/lib/setup.php index 6f608ec5fb1..a63cc664dbc 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -456,7 +456,7 @@ class OC_Setup { } else { $easy_connect_string = '//'.$e_host.'/'.$e_dbname; } - \OC_Log::write('setup oracle', 'connect string: '.$easy_connect_string, \OC_Log::DEBUG); + \OC_Log::write('setup oracle', 'connect string: ' . $easy_connect_string, \OC_Log::DEBUG); $connection = @oci_connect($dbuser, $dbpass, $easy_connect_string); if(!$connection) { $e = oci_error(); -- cgit v1.2.3 From 85e0c78166d45b0ef9d00b1fd2164e4969ee9b83 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Thu, 23 May 2013 01:21:36 +0200 Subject: fix problems with german "Umlaut" in folder name --- lib/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index a98953b42aa..b73e2e83fcc 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -114,7 +114,7 @@ class Scanner { $size = 0; if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) { \OC_DB::beginTransaction(); - while ($file = readdir($dh)) { + while ($file = utf8_encode(readdir($dh))) { $child = ($path) ? $path . '/' . $file : $file; if (!$this->isIgnoredDir($file)) { $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); -- cgit v1.2.3 From d4b700ef4ebfa9aba9bcadda864ed5a2a92174ed Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Thu, 23 May 2013 20:29:46 +0200 Subject: revert previous fix and added normalizer to cache class --- lib/files/cache/cache.php | 24 +++++++++++++++ lib/files/cache/scanner.php | 2 +- tests/lib/files/cache/cache.php | 67 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 3341fe50525..a7e634c8e41 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -100,6 +100,9 @@ class Cache { */ public function get($file) { if (is_string($file) or $file == '') { + // normalize file + $file = $this->normalize($file); + $where = 'WHERE `storage` = ? AND `path_hash` = ?'; $params = array($this->getNumericStorageId(), md5($file)); } else { //file id @@ -177,6 +180,9 @@ class Cache { $this->update($id, $data); return $id; } else { + // normalize file + $file = $this->normalize($file); + if (isset($this->partial[$file])) { //add any saved partial data $data = array_merge($this->partial[$file], $data); unset($this->partial[$file]); @@ -265,6 +271,9 @@ class Cache { * @return int */ public function getId($file) { + // normalize file + $file = $this->normalize($file); + $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); @@ -549,4 +558,19 @@ class Cache { return null; } } + + /** + * normalize the given path + * @param $path + * @return string + */ + public function normalize($path) { + + //normalize unicode if possible + if (class_exists('Normalizer')) { + $path = \Normalizer::normalize($path); + } + + return $path; + } } diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index b73e2e83fcc..a98953b42aa 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -114,7 +114,7 @@ class Scanner { $size = 0; if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) { \OC_DB::beginTransaction(); - while ($file = utf8_encode(readdir($dh))) { + while ($file = readdir($dh)) { $child = ($path) ? $path . '/' . $file : $file; if (!$this->isIgnoredDir($file)) { $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); diff --git a/tests/lib/files/cache/cache.php b/tests/lib/files/cache/cache.php index 1612a673838..2b1e5a56212 100644 --- a/tests/lib/files/cache/cache.php +++ b/tests/lib/files/cache/cache.php @@ -8,6 +8,8 @@ namespace Test\Files\Cache; +use PHPUnit_Framework_MockObject_MockObject; + class LongId extends \OC\Files\Storage\Temporary { public function getId() { return 'long:' . str_repeat('foo', 50) . parent::getId(); @@ -237,6 +239,71 @@ class Cache extends \PHPUnit_Framework_TestCase { $this->assertEquals(array(md5($storageId), 'foo'), \OC\Files\Cache\Cache::getById($id)); } + /** + * @brief this test show the bug resulting if we have no normalizer installed + */ + public function testWithoutNormalizer() { + // create folder Schön with U+00F6 + $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; + + // create folder Schön with U+0308 + $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; + + /** + * @var \OC\Files\Cache\Cache | PHPUnit_Framework_MockObject_MockObject $cacheMock + */ + $cacheMock = $this->getMock('\OC\Files\Cache\Cache', array('normalize'), array($this->storage), '', true); + + $cacheMock->expects($this->any()) + ->method('normalize') + ->will($this->returnArgument(0)); + + $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->assertFalse($cacheMock->get('folder')); + $this->assertGreaterThan(0, $cacheMock->put('folder', $data)); + + $this->assertFalse($cacheMock->get('folder/' . $folderWith00F6)); + $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith00F6, $data)); + + $this->assertFalse($cacheMock->get('folder/' .$folderWith0308)); + $this->assertGreaterThan(0, $cacheMock->put('folder/' .$folderWith0308, $data)); + + // this is our bug, we have two different hashes with the same name (Schön) + $this->assertEquals(2, count($cacheMock->getFolderContents('folder'))); + } + + /** + * @brief this test shows that there is no bug if we use the normalizer + */ + public function testWithNormalizer() { + + if(!class_exists('Normalizer')) { + $this->markTestSkipped('The Normalizer extension is not available.'); + return; + } + + // folder name Schön with U+00F6 + $folderWith00F6 = "\x53\x63\x68\xc3\xb6\x6e"; + + // folder name Schön with U+0308 + $folderWith0308 = "\x53\x63\x68\x6f\xcc\x88\x6e"; + + $data = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->assertFalse($this->cache->get('folder')); + $this->assertGreaterThan(0, $this->cache->put('folder', $data)); + + $this->assertFalse($this->cache->get('folder/' . $folderWith00F6)); + $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith00F6, $data)); + + $this->assertTrue(is_array($this->cache->get('folder/' .$folderWith0308))); + $this->assertGreaterThan(0, $this->cache->put('folder/' .$folderWith0308, $data)); + + // at this point we should have only one folder named "Schön" + $this->assertEquals(1, count($this->cache->getFolderContents('folder'))); + } + public function tearDown() { $this->cache->clear(); } -- cgit v1.2.3 From 736f10039dd1b494966f1214916dc62119478176 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Fri, 24 May 2013 20:36:20 +0200 Subject: changed builtin normalizer to \OC_Util::normalizeUnicode --- lib/files/cache/cache.php | 7 +------ lib/files/filesystem.php | 5 ++--- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index a7e634c8e41..865abd5286f 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -566,11 +566,6 @@ class Cache { */ public function normalize($path) { - //normalize unicode if possible - if (class_exists('Normalizer')) { - $path = \Normalizer::normalize($path); - } - - return $path; + return \OC_Util::normalizeUnicode($path); } } diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index d60d430d77c..5d7565f0d83 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -616,9 +616,8 @@ class Filesystem { $path = substr($path, 0, -1); } //normalize unicode if possible - if (class_exists('Normalizer')) { - $path = \Normalizer::normalize($path); - } + $path = \OC_Util::normalizeUnicode($path); + return $path; } -- cgit v1.2.3 From c245f5a99fcd0c273de8e86d5496e86d61151d6c Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 14:56:00 +0200 Subject: added more places where normalization is needed --- lib/files/cache/cache.php | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 2c34fb77925..dc5e3e20fc7 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -343,6 +343,10 @@ class Cache { * @param string $target */ public function move($source, $target) { + // normalize source and target + $source = $this->normalize($source); + $target = $this->normalize($target); + $sourceData = $this->get($source); $sourceId = $sourceData['fileid']; $newParentId = $this->getParentId($target); @@ -383,6 +387,9 @@ class Cache { * @return int, Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE */ public function getStatus($file) { + // normalize file + $file = $this->normalize($file); + $pathHash = md5($file); $query = \OC_DB::prepare('SELECT `size` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'); $result = $query->execute(array($this->getNumericStorageId(), $pathHash)); -- cgit v1.2.3 From 505a300776a958f4076f923b0966ab13eee3c4b5 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 25 May 2013 20:35:12 +0200 Subject: we should also normalize on update and search because the database layer will not do this for us --- lib/files/cache/cache.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'lib/files/cache') diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index dc5e3e20fc7..1e93cc59c6b 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -226,6 +226,17 @@ class Cache { * @param array $data */ public function update($id, array $data) { + + if(isset($data['path'])) { + // normalize path + $data['path'] = $this->normalize($data['path']); + } + + if(isset($data['name'])) { + // normalize path + $data['name'] = $this->normalize($data['name']); + } + list($queryParts, $params) = $this->buildParts($data); $params[] = $id; @@ -418,6 +429,10 @@ class Cache { * @return array of file data */ public function search($pattern) { + + // normalize pattern + $pattern = $this->normalize($pattern); + $query = \OC_DB::prepare(' SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?' -- cgit v1.2.3 From 44d217c42529c3f4a0df0a56255887450d37e00b Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 29 May 2013 15:25:42 +0200 Subject: Cache: provide a function to get the permissions of all files in a folder with one query --- lib/files/cache/permissions.php | 19 +++++++++++++++++++ lib/files/view.php | 22 +++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/permissions.php b/lib/files/cache/permissions.php index faa5ff5eacc..73473142c74 100644 --- a/lib/files/cache/permissions.php +++ b/lib/files/cache/permissions.php @@ -85,6 +85,25 @@ class Permissions { return $filePermissions; } + /** + * get the permissions for all files in a folder + * + * @param int $parentId + * @return int[] + */ + public function getDirectoryPermissions($parentId) { + $query = \OC_DB::prepare('SELECT `*PREFIX*permissions`.`fileid`, `permissions` + FROM `*PREFIX*permissions` INNER JOIN `*PREFIX*filecache` ON `*PREFIX*permissions`.fileid = `*PREFIX*filecache`.fileid + WHERE `*PREFIX*filecache`.parent = ?'); + + $result = $query->execute(array($parentId)); + $filePermissions = array(); + while ($row = $result->fetchRow()) { + $filePermissions[$row['fileid']] = $row['permissions']; + } + return $filePermissions; + } + /** * remove the permissions for a file * diff --git a/lib/files/view.php b/lib/files/view.php index 8e7727d335c..ffe0e155c6c 100644 --- a/lib/files/view.php +++ b/lib/files/view.php @@ -267,7 +267,7 @@ class View { $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); if (\OC_FileProxy::runPreProxies('file_put_contents', $absolutePath, $data) and Filesystem::isValidPath($path) - and ! Filesystem::isFileBlacklisted($path) + and !Filesystem::isFileBlacklisted($path) ) { $path = $this->getRelativePath($absolutePath); $exists = $this->file_exists($path); @@ -344,7 +344,7 @@ class View { \OC_FileProxy::runPreProxies('rename', $absolutePath1, $absolutePath2) and Filesystem::isValidPath($path2) and Filesystem::isValidPath($path1) - and ! Filesystem::isFileBlacklisted($path2) + and !Filesystem::isFileBlacklisted($path2) ) { $path1 = $this->getRelativePath($absolutePath1); $path2 = $this->getRelativePath($absolutePath2); @@ -371,7 +371,7 @@ class View { list(, $internalPath2) = Filesystem::resolvePath($absolutePath2 . $postFix2); if ($storage) { $result = $storage->rename($internalPath1, $internalPath2); - \OC_FileProxy::runPostProxies('rename', $absolutePath1, $absolutePath2); + \OC_FileProxy::runPostProxies('rename', $absolutePath1, $absolutePath2); } else { $result = false; } @@ -418,7 +418,7 @@ class View { \OC_FileProxy::runPreProxies('copy', $absolutePath1, $absolutePath2) and Filesystem::isValidPath($path2) and Filesystem::isValidPath($path1) - and ! Filesystem::isFileBlacklisted($path2) + and !Filesystem::isFileBlacklisted($path2) ) { $path1 = $this->getRelativePath($absolutePath1); $path2 = $this->getRelativePath($absolutePath2); @@ -632,7 +632,7 @@ class View { $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); if (\OC_FileProxy::runPreProxies($operation, $absolutePath, $extraParam) and Filesystem::isValidPath($path) - and ! Filesystem::isFileBlacklisted($path) + and !Filesystem::isFileBlacklisted($path) ) { $path = $this->getRelativePath($absolutePath); if ($path == null) { @@ -760,7 +760,7 @@ class View { } } - $data = \OC_FileProxy::runPostProxies('getFileInfo', $path, $data); + $data = \OC_FileProxy::runPostProxies('getFileInfo', $path, $data); return $data; } @@ -797,18 +797,18 @@ class View { } $files = $cache->getFolderContents($internalPath); //TODO: mimetype_filter + $permissions = $permissionsCache->getDirectoryPermissions($cache->getId($internalPath)); $ids = array(); foreach ($files as $i => $file) { $files[$i]['type'] = $file['mimetype'] === 'httpd/unix-directory' ? 'dir' : 'file'; $ids[] = $file['fileid']; - $permissions = $permissionsCache->get($file['fileid'], $user); - if ($permissions === -1) { - $permissions = $storage->getPermissions($file['path']); - $permissionsCache->set($file['fileid'], $user, $permissions); + if (!isset($permissions[$file['fileid']])) { + $permissions[$file['fileid']] = $storage->getPermissions($file['path']); + $permissionsCache->set($file['fileid'], $user, $permissions[$file['fileid']]); } - $files[$i]['permissions'] = $permissions; + $files[$i]['permissions'] = $permissions[$file['fileid']]; } //add a folder for any mountpoint in this directory and add the sizes of other mountpoints to the folders -- cgit v1.2.3 From 04805d0be9add28f0b374df01ca4211a997d21d0 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Wed, 29 May 2013 10:19:03 -0400 Subject: Only update metadata that has changed --- lib/files/cache/scanner.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 46122221dc2..cd022d25132 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -78,7 +78,7 @@ class Scanner { $this->scanFile($parent); } } - if($cacheData = $this->cache->get($file)) { + if ($cacheData = $this->cache->get($file)) { if ($data['mtime'] === $cacheData['mtime'] && $data['size'] === $cacheData['size']) { $data['etag'] = $cacheData['etag']; @@ -89,7 +89,14 @@ class Scanner { $data['size'] = $cacheData['size']; } } - $this->cache->put($file, $data); + $newData = $data; + if ($cacheData) { + // Only update metadata that has changed + $newData = array_diff($data, $cacheData); + } + if (!empty($newData)) { + $this->cache->put($file, $newData); + } } return $data; } -- cgit v1.2.3 From ea553676789a4b62c1850dc156552ba8276fde5f Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Wed, 29 May 2013 10:40:05 -0400 Subject: Organize conditionals in a better order --- lib/files/cache/scanner.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index cd022d25132..8f9a7921956 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -78,19 +78,15 @@ class Scanner { $this->scanFile($parent); } } + $newData = $data; if ($cacheData = $this->cache->get($file)) { + if ($checkExisting && $data['size'] === -1) { + $data['size'] = $cacheData['size']; + } if ($data['mtime'] === $cacheData['mtime'] && $data['size'] === $cacheData['size']) { $data['etag'] = $cacheData['etag']; } - } - if ($checkExisting and $cacheData) { - if ($data['size'] === -1) { - $data['size'] = $cacheData['size']; - } - } - $newData = $data; - if ($cacheData) { // Only update metadata that has changed $newData = array_diff($data, $cacheData); } -- cgit v1.2.3 From dbbf195727d698a5ba46d7cfee06eb9ea7916d96 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Wed, 29 May 2013 18:08:18 -0400 Subject: Only retrieve the current user's permissions --- lib/files/cache/permissions.php | 9 +++++---- lib/files/view.php | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'lib/files/cache') diff --git a/lib/files/cache/permissions.php b/lib/files/cache/permissions.php index 73473142c74..29c30b0f36c 100644 --- a/lib/files/cache/permissions.php +++ b/lib/files/cache/permissions.php @@ -89,14 +89,15 @@ class Permissions { * get the permissions for all files in a folder * * @param int $parentId + * @param string $user * @return int[] */ - public function getDirectoryPermissions($parentId) { + public function getDirectoryPermissions($parentId, $user) { $query = \OC_DB::prepare('SELECT `*PREFIX*permissions`.`fileid`, `permissions` - FROM `*PREFIX*permissions` INNER JOIN `*PREFIX*filecache` ON `*PREFIX*permissions`.fileid = `*PREFIX*filecache`.fileid - WHERE `*PREFIX*filecache`.parent = ?'); + FROM `*PREFIX*permissions` INNER JOIN `*PREFIX*filecache` ON `*PREFIX*permissions`.`fileid` = `*PREFIX*filecache`.`fileid` + WHERE `*PREFIX*filecache`.`parent` = ? AND `*PREFIX*permissions`.`user` = ?'); - $result = $query->execute(array($parentId)); + $result = $query->execute(array($parentId, $user)); $filePermissions = array(); while ($row = $result->fetchRow()) { $filePermissions[$row['fileid']] = $row['permissions']; diff --git a/lib/files/view.php b/lib/files/view.php index ffe0e155c6c..ecb0f30400a 100644 --- a/lib/files/view.php +++ b/lib/files/view.php @@ -797,7 +797,7 @@ class View { } $files = $cache->getFolderContents($internalPath); //TODO: mimetype_filter - $permissions = $permissionsCache->getDirectoryPermissions($cache->getId($internalPath)); + $permissions = $permissionsCache->getDirectoryPermissions($cache->getId($internalPath), $user); $ids = array(); foreach ($files as $i => $file) { -- cgit v1.2.3