From 6195f13bda086b242cbcdbff86260a81bf6163c9 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 16 Oct 2013 15:37:58 +0200 Subject: Use CURL to get filesize on 32bit systems. --- lib/private/files/storage/local.php | 75 +++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 11 deletions(-) (limited to 'lib/private/files/storage') diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 943c4163088..fc2a590f6c8 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -90,10 +90,10 @@ if (\OC_Util::runningOnWindows()) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - if ($statResult['size'] < 0) { - $size = self::getFileSizeFromOS($fullPath); - $statResult['size'] = $size; - $statResult[7] = $size; + $filesize = self::getFileSizeWithTricks($fullPath); + if (!is_null($filesize)) { + $statResult['size'] = $filesize; + $statResult[7] = $filesize; } return $statResult; } @@ -111,12 +111,13 @@ if (\OC_Util::runningOnWindows()) { return 0; } else { $fullPath = $this->datadir . $path; - $fileSize = filesize($fullPath); - if ($fileSize < 0) { - return self::getFileSizeFromOS($fullPath); + + $filesize = self::getFileSizeWithTricks($fullPath); + if (!is_null($filesize)) { + return $filesize; } - return $fileSize; + return filesize($fullPath); } } @@ -221,8 +222,60 @@ if (\OC_Util::runningOnWindows()) { } /** - * @param string $fullPath - */ + * @brief Tries to get the filesize via various workarounds if necessary. + * @param string $fullPath + * @return mixed Number of bytes as float on success and workaround necessary, null otherwise. + */ + private static function getFileSizeWithTricks($fullPath) { + if (PHP_INT_SIZE === 4) { + // filesize() and stat() are unreliable on 32bit systems + // for big files. + // In some cases they cause an E_WARNING and return false, + // in some other cases they silently wrap around at 2^32, + // i.e. e.g. report 674347008 bytes instead of 4969314304. + $filesize = self::getFileSizeFromCurl($fullPath); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = self::getFileSizeFromOS($fullPath); + if (!is_null($filesize)) { + return $filesize; + } + } + + return null; + } + + /** + * @brief Tries to get the filesize via a CURL HEAD request. + * @param string $fullPath + * @return mixed Number of bytes as float on success, null otherwise. + */ + private static function getFileSizeFromCurl($fullPath) { + if (function_exists('curl_init')) { + $ch = curl_init("file://$fullPath"); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HEADER, true); + $data = curl_exec($ch); + curl_close($ch); + if ($data !== false) { + $matches = array(); + preg_match('/Content-Length: (\d+)/', $data, $matches); + if (isset($matches[1])) { + return (float) $matches[1]; + } + } + } + + return null; + } + + /** + * @brief Tries to get the filesize via COM and exec(). + * @param string $fullPath + * @return mixed Number of bytes as float on success, null otherwise. + */ private static function getFileSizeFromOS($fullPath) { $name = strtolower(php_uname('s')); // Windows OS: we use COM to access the filesystem @@ -246,7 +299,7 @@ if (\OC_Util::runningOnWindows()) { \OC_Log::ERROR); } - return 0; + return null; } public function hash($type, $path, $raw = false) { -- cgit v1.2.3 From 3f8f8027d25ab39283d0ab13afcf7252dde8f240 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 7 Jan 2014 13:50:57 +0100 Subject: Cast to numeric instead of float, i.e. use an integer if possible. --- lib/private/files/storage/local.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/private/files/storage') diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index fc2a590f6c8..7fa748a7fd7 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -224,7 +224,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via various workarounds if necessary. * @param string $fullPath - * @return mixed Number of bytes as float on success and workaround necessary, null otherwise. + * @return mixed Number of bytes on success and workaround necessary, null otherwise. */ private static function getFileSizeWithTricks($fullPath) { if (PHP_INT_SIZE === 4) { @@ -249,7 +249,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via a CURL HEAD request. * @param string $fullPath - * @return mixed Number of bytes as float on success, null otherwise. + * @return mixed Number of bytes on success, null otherwise. */ private static function getFileSizeFromCurl($fullPath) { if (function_exists('curl_init')) { @@ -263,7 +263,7 @@ if (\OC_Util::runningOnWindows()) { $matches = array(); preg_match('/Content-Length: (\d+)/', $data, $matches); if (isset($matches[1])) { - return (float) $matches[1]; + return 0 + $matches[1]; } } } @@ -274,7 +274,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via COM and exec(). * @param string $fullPath - * @return mixed Number of bytes as float on success, null otherwise. + * @return mixed Number of bytes on success, null otherwise. */ private static function getFileSizeFromOS($fullPath) { $name = strtolower(php_uname('s')); @@ -287,11 +287,11 @@ if (\OC_Util::runningOnWindows()) { } } else if (strpos($name, 'bsd') !== false) { if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -f %z ' . escapeshellarg($fullPath)); + return 0 + exec('stat -f %z ' . escapeshellarg($fullPath)); } } else if (strpos($name, 'linux') !== false) { if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -c %s ' . escapeshellarg($fullPath)); + return 0 + exec('stat -c %s ' . escapeshellarg($fullPath)); } } else { \OC_Log::write('core', -- cgit v1.2.3 From c8fa1fd68784c48c1df537310f16d6753f79b029 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 9 Feb 2014 01:25:33 +0100 Subject: Refactor Large File handling code. --- lib/private/files/storage/local.php | 98 +++------------------------- lib/private/files/storage/mappedlocal.php | 53 ++++----------- lib/private/largefilehelper.php | 103 ++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 131 deletions(-) create mode 100644 lib/private/largefilehelper.php (limited to 'lib/private/files/storage') diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 7fa748a7fd7..883a69d7681 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -89,9 +89,8 @@ if (\OC_Util::runningOnWindows()) { public function stat($path) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - - $filesize = self::getFileSizeWithTricks($fullPath); - if (!is_null($filesize)) { + if (PHP_INT_SIZE === 4) { + $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; } @@ -109,16 +108,16 @@ if (\OC_Util::runningOnWindows()) { public function filesize($path) { if ($this->is_dir($path)) { return 0; - } else { - $fullPath = $this->datadir . $path; - - $filesize = self::getFileSizeWithTricks($fullPath); + } + $fullPath = $this->datadir . $path; + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper; + $filesize = $helper->getFilesize($fullPath); if (!is_null($filesize)) { return $filesize; } - - return filesize($fullPath); } + return filesize($fullPath); } public function isReadable($path) { @@ -221,87 +220,6 @@ if (\OC_Util::runningOnWindows()) { return $return; } - /** - * @brief Tries to get the filesize via various workarounds if necessary. - * @param string $fullPath - * @return mixed Number of bytes on success and workaround necessary, null otherwise. - */ - private static function getFileSizeWithTricks($fullPath) { - if (PHP_INT_SIZE === 4) { - // filesize() and stat() are unreliable on 32bit systems - // for big files. - // In some cases they cause an E_WARNING and return false, - // in some other cases they silently wrap around at 2^32, - // i.e. e.g. report 674347008 bytes instead of 4969314304. - $filesize = self::getFileSizeFromCurl($fullPath); - if (!is_null($filesize)) { - return $filesize; - } - $filesize = self::getFileSizeFromOS($fullPath); - if (!is_null($filesize)) { - return $filesize; - } - } - - return null; - } - - /** - * @brief Tries to get the filesize via a CURL HEAD request. - * @param string $fullPath - * @return mixed Number of bytes on success, null otherwise. - */ - private static function getFileSizeFromCurl($fullPath) { - if (function_exists('curl_init')) { - $ch = curl_init("file://$fullPath"); - curl_setopt($ch, CURLOPT_NOBODY, true); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HEADER, true); - $data = curl_exec($ch); - curl_close($ch); - if ($data !== false) { - $matches = array(); - preg_match('/Content-Length: (\d+)/', $data, $matches); - if (isset($matches[1])) { - return 0 + $matches[1]; - } - } - } - - return null; - } - - /** - * @brief Tries to get the filesize via COM and exec(). - * @param string $fullPath - * @return mixed Number of bytes on success, null otherwise. - */ - private static function getFileSizeFromOS($fullPath) { - $name = strtolower(php_uname('s')); - // Windows OS: we use COM to access the filesystem - if (strpos($name, 'win') !== false) { - if (class_exists('COM')) { - $fsobj = new \COM("Scripting.FileSystemObject"); - $f = $fsobj->GetFile($fullPath); - return $f->Size; - } - } else if (strpos($name, 'bsd') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return 0 + exec('stat -f %z ' . escapeshellarg($fullPath)); - } - } else if (strpos($name, 'linux') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return 0 + exec('stat -c %s ' . escapeshellarg($fullPath)); - } - } else { - \OC_Log::write('core', - 'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name, - \OC_Log::ERROR); - } - - return null; - } - public function hash($type, $path, $raw = false) { return hash_file($type, $this->datadir . $path, $raw); } diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index 3ebdcf9538f..78d2616ba9d 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -111,11 +111,10 @@ class MappedLocal extends \OC\Files\Storage\Common { public function stat($path) { $fullPath = $this->buildPath($path); $statResult = stat($fullPath); - - if ($statResult['size'] < 0) { - $size = self::getFileSizeFromOS($fullPath); - $statResult['size'] = $size; - $statResult[7] = $size; + if (PHP_INT_SIZE === 4) { + $filesize = $this->filesize($path); + $statResult['size'] = $filesize; + $statResult[7] = $filesize; } return $statResult; } @@ -131,15 +130,16 @@ class MappedLocal extends \OC\Files\Storage\Common { public function filesize($path) { if ($this->is_dir($path)) { return 0; - } else { - $fullPath = $this->buildPath($path); - $fileSize = filesize($fullPath); - if ($fileSize < 0) { - return self::getFileSizeFromOS($fullPath); + } + $fullPath = $this->buildPath($path); + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper; + $filesize = $helper->getFilesize($fullPath); + if (!is_null($filesize)) { + return $filesize; } - - return $fileSize; } + return filesize($fullPath); } public function isReadable($path) { @@ -294,35 +294,6 @@ class MappedLocal extends \OC\Files\Storage\Common { return $return; } - /** - * @param string $fullPath - */ - private static function getFileSizeFromOS($fullPath) { - $name = strtolower(php_uname('s')); - // Windows OS: we use COM to access the filesystem - if (strpos($name, 'win') !== false) { - if (class_exists('COM')) { - $fsobj = new \COM("Scripting.FileSystemObject"); - $f = $fsobj->GetFile($fullPath); - return $f->Size; - } - } else if (strpos($name, 'bsd') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -f %z ' . escapeshellarg($fullPath)); - } - } else if (strpos($name, 'linux') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -c %s ' . escapeshellarg($fullPath)); - } - } else { - \OC_Log::write('core', - 'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name, - \OC_Log::ERROR); - } - - return 0; - } - public function hash($type, $path, $raw = false) { return hash_file($type, $this->buildPath($path), $raw); } diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php new file mode 100644 index 00000000000..ca8f7522177 --- /dev/null +++ b/lib/private/largefilehelper.php @@ -0,0 +1,103 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC; + +/** + * Helper class for large files on 32-bit platforms. + */ +class LargeFileHelper { + /** + * @brief Tries to get the filesize of a file via various workarounds that + * even work for large files on 32-bit platforms. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesize($filename) { + $filesize = $this->getFilesizeViaCurl($filename); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = $this->getFilesizeViaCOM($filename); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = $this->getFilesizeViaExec($filename); + if (!is_null($filesize)) { + return $filesize; + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via a CURL HEAD request. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaCurl($filename) { + if (function_exists('curl_init')) { + $ch = curl_init("file://$filename"); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HEADER, true); + $data = curl_exec($ch); + curl_close($ch); + if ($data !== false) { + $matches = array(); + preg_match('/Content-Length: (\d+)/', $data, $matches); + if (isset($matches[1])) { + return 0 + $matches[1]; + } + } + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via the Windows DOM extension. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaCOM($filename) { + if (class_exists('COM')) { + $fsobj = new \COM("Scripting.FileSystemObject"); + $file = $fsobj->GetFile($filename); + return 0 + $file->Size; + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via an exec() call. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaExec($filename) { + if (\OC_Helper::is_function_enabled('exec')) { + $os = strtolower(php_uname('s')); + if (strpos($os, 'linux') !== false) { + return 0 + exec('stat -c %s ' . escapeshellarg($filename)); + } else if (strpos($os, 'bsd') !== false) { + return 0 + exec('stat -f %z ' . escapeshellarg($filename)); + } + } + return null; + } +} -- cgit v1.2.3 From fb7ec2bb22055886d55a779b9c19f48daee438c5 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 12 Feb 2014 13:58:07 +0100 Subject: Only call $this->filesize() for files. --- lib/private/files/storage/local.php | 2 +- lib/private/files/storage/mappedlocal.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/private/files/storage') diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 883a69d7681..f4bc5085364 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -89,7 +89,7 @@ if (\OC_Util::runningOnWindows()) { public function stat($path) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - if (PHP_INT_SIZE === 4) { + if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) { $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index 78d2616ba9d..f38b48db5de 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -111,7 +111,7 @@ class MappedLocal extends \OC\Files\Storage\Common { public function stat($path) { $fullPath = $this->buildPath($path); $statResult = stat($fullPath); - if (PHP_INT_SIZE === 4) { + if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) { $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; -- cgit v1.2.3 From 0417e52134e87c379b3b4c22a53a0c06a711baef Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 18 Feb 2014 12:57:44 +0100 Subject: Increase file size limit from 2 GiB to 4 GiB when workarounds are unavailable. --- lib/private/files/storage/local.php | 5 +---- lib/private/files/storage/mappedlocal.php | 5 +---- lib/private/largefilehelper.php | 23 ++++++++++++++++++++++- tests/lib/largefilehelpergetfilesize.php | 7 +++++++ 4 files changed, 31 insertions(+), 9 deletions(-) (limited to 'lib/private/files/storage') diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index f4bc5085364..e33747bbd52 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -112,10 +112,7 @@ if (\OC_Util::runningOnWindows()) { $fullPath = $this->datadir . $path; if (PHP_INT_SIZE === 4) { $helper = new \OC\LargeFileHelper; - $filesize = $helper->getFilesize($fullPath); - if (!is_null($filesize)) { - return $filesize; - } + return $helper->getFilesize($fullPath); } return filesize($fullPath); } diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index f38b48db5de..ea4deaa66e8 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -134,10 +134,7 @@ class MappedLocal extends \OC\Files\Storage\Common { $fullPath = $this->buildPath($path); if (PHP_INT_SIZE === 4) { $helper = new \OC\LargeFileHelper; - $filesize = $helper->getFilesize($fullPath); - if (!is_null($filesize)) { - return $filesize; - } + return $helper->getFilesize($fullPath); } return filesize($fullPath); } diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 08869d7c82a..3d15e786042 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -83,7 +83,7 @@ class LargeFileHelper { if (!is_null($filesize)) { return $filesize; } - return null; + return $this->getFilesizeNative($filename); } /** @@ -159,6 +159,27 @@ class LargeFileHelper { return null; } + /** + * @brief Gets the filesize via a filesize() call and converts negative + * signed int to positive float. As the result of filesize() will + * wrap around after a filesize of 2^32 bytes = 4 GiB, this should + * only be used as a last resort. + * + * @param string $filename Path to the file. + * + * @return int|float Number of bytes as number (float or int). + */ + public function getFilesizeNative($filename) { + $result = filesize($filename); + if ($result < 0) { + // For filesizes between 2 GiB and 4 GiB, filesize() will return a + // negative int, as the PHP data type int is signed. Interpret the + // returned int as an unsigned integer and put it into a float. + return (float) sprintf('%u', $result); + } + return $result; + } + protected function exec($cmd) { $result = trim(exec($cmd)); return ctype_digit($result) ? 0 + $result : null; diff --git a/tests/lib/largefilehelpergetfilesize.php b/tests/lib/largefilehelpergetfilesize.php index 001f636a52a..699dd6891ad 100644 --- a/tests/lib/largefilehelpergetfilesize.php +++ b/tests/lib/largefilehelpergetfilesize.php @@ -59,4 +59,11 @@ class LargeFileHelperGetFilesize extends \PHPUnit_Framework_TestCase { $this->helper->getFilesizeViaExec($this->filename) ); } + + public function testGetFilesizeNative() { + $this->assertSame( + $this->filesize, + $this->helper->getFilesizeNative($this->filename) + ); + } } -- cgit v1.2.3