]> source.dussan.org Git - nextcloud-server.git/commitdiff
extend storage api to allow directly writing a stream to storage
authorRobin Appelman <robin@icewind.nl>
Fri, 26 Oct 2018 17:15:23 +0000 (19:15 +0200)
committerRobin Appelman <robin@icewind.nl>
Wed, 31 Oct 2018 20:10:44 +0000 (21:10 +0100)
this removes the need for temporary storages with some external storage backends.
The new method is added to a separate interface to maintain compatibility with
storage backends implementing the storage interface directly (without inheriting common)

Currently the interface is implemented for objectstorage based storages and local storage
and used by webdav uploads

Signed-off-by: Robin Appelman <robin@icewind.nl>
apps/dav/lib/Connector/Sabre/File.php
lib/private/Files/ObjectStore/ObjectStoreStorage.php
lib/private/Files/Storage/Common.php
lib/private/Files/Storage/Local.php
lib/private/Files/Stream/CountReadStream.php [new file with mode: 0644]
lib/public/Files/Storage/IWriteStreamStorage.php [new file with mode: 0644]
tests/lib/Files/Stream/CountReadStreamTest.php [new file with mode: 0644]

index 9e927ff85e5bf5182e41bdcdd2c095ae0cf36cbf..a8e6d8b907c429b5943d7a856d4c37f27f62e1bd 100644 (file)
@@ -164,14 +164,19 @@ class File extends Node implements IFile {
                                $this->changeLock(ILockingProvider::LOCK_EXCLUSIVE);
                        }
 
-                       $target = $partStorage->fopen($internalPartPath, 'wb');
-                       if ($target === false) {
-                               \OC::$server->getLogger()->error('\OC\Files\Filesystem::fopen() failed', ['app' => 'webdav']);
-                               // because we have no clue about the cause we can only throw back a 500/Internal Server Error
-                               throw new Exception('Could not write file contents');
+                       if ($partStorage->instanceOfStorage(Storage\IWriteStreamStorage::class)) {
+                               $count = $partStorage->writeStream($internalPartPath, $data);
+                               $result = $count > 0;
+                       } else {
+                               $target = $partStorage->fopen($internalPartPath, 'wb');
+                               if ($target === false) {
+                                       \OC::$server->getLogger()->error('\OC\Files\Filesystem::fopen() failed', ['app' => 'webdav']);
+                                       // because we have no clue about the cause we can only throw back a 500/Internal Server Error
+                                       throw new Exception('Could not write file contents');
+                               }
+                               list($count, $result) = \OC_Helper::streamCopy($data, $target);
+                               fclose($target);
                        }
-                       list($count, $result) = \OC_Helper::streamCopy($data, $target);
-                       fclose($target);
 
                        if ($result === false) {
                                $expected = -1;
@@ -185,7 +190,7 @@ class File extends Node implements IFile {
                        // double check if the file was fully received
                        // compare expected and actual size
                        if (isset($_SERVER['CONTENT_LENGTH']) && $_SERVER['REQUEST_METHOD'] === 'PUT') {
-                               $expected = (int) $_SERVER['CONTENT_LENGTH'];
+                               $expected = (int)$_SERVER['CONTENT_LENGTH'];
                                if ($count !== $expected) {
                                        throw new BadRequest('expected filesize ' . $expected . ' got ' . $count);
                                }
@@ -219,7 +224,7 @@ class File extends Node implements IFile {
                                        $renameOkay = $storage->moveFromStorage($partStorage, $internalPartPath, $internalPath);
                                        $fileExists = $storage->file_exists($internalPath);
                                        if ($renameOkay === false || $fileExists === false) {
-                                               \OC::$server->getLogger()->error('renaming part file to final file failed ($run: ' . ( $run ? 'true' : 'false' ) . ', $renameOkay: '  . ( $renameOkay ? 'true' : 'false' ) . ', $fileExists: ' . ( $fileExists ? 'true' : 'false' ) . ')', ['app' => 'webdav']);
+                                               \OC::$server->getLogger()->error('renaming part file to final file failed ($run: ' . ($run ? 'true' : 'false') . ', $renameOkay: ' . ($renameOkay ? 'true' : 'false') . ', $fileExists: ' . ($fileExists ? 'true' : 'false') . ')', ['app' => 'webdav']);
                                                throw new Exception('Could not rename part file to final file');
                                        }
                                } catch (ForbiddenException $ex) {
@@ -246,7 +251,7 @@ class File extends Node implements IFile {
                                        $this->header('X-OC-MTime: accepted');
                                }
                        }
-                                       
+
                        if ($view) {
                                $this->emitPostHooks($exists);
                        }
@@ -443,7 +448,7 @@ class File extends Node implements IFile {
                //detect aborted upload
                if (isset ($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'PUT') {
                        if (isset($_SERVER['CONTENT_LENGTH'])) {
-                               $expected = (int) $_SERVER['CONTENT_LENGTH'];
+                               $expected = (int)$_SERVER['CONTENT_LENGTH'];
                                if ($bytesWritten !== $expected) {
                                        $chunk_handler->remove($info['index']);
                                        throw new BadRequest(
index 3ce919a4cbe709de05f96d7c80496437abd1c8fc..71acd27783c82192df2a3d2608fa68b56bc79701 100644 (file)
@@ -28,6 +28,7 @@ namespace OC\Files\ObjectStore;
 use Icewind\Streams\CallbackWrapper;
 use Icewind\Streams\IteratorDirectory;
 use OC\Files\Cache\CacheEntry;
+use OC\Files\Stream\CountReadStream;
 use OCP\Files\ObjectStore\IObjectStore;
 
 class ObjectStoreStorage extends \OC\Files\Storage\Common {
@@ -382,25 +383,48 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
        }
 
        public function writeBack($tmpFile, $path) {
+               $size = filesize($tmpFile);
+               $this->writeStream($path, fopen($tmpFile, 'r'), $size);
+       }
+
+       /**
+        * external changes are not supported, exclusive access to the object storage is assumed
+        *
+        * @param string $path
+        * @param int $time
+        * @return false
+        */
+       public function hasUpdated($path, $time) {
+               return false;
+       }
+
+       public function needsPartFile() {
+               return false;
+       }
+
+       public function file_put_contents($path, $data) {
+               $stream = fopen('php://temp', 'r+');
+               fwrite($stream, $data);
+               rewind($stream);
+               return $this->writeStream($path, $stream, strlen($data)) > 0;
+       }
+
+       public function writeStream(string $path, $stream, int $size = null): int {
                $stat = $this->stat($path);
                if (empty($stat)) {
                        // create new file
-                       $stat = array(
+                       $stat = [
                                'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
-                       );
+                       ];
                }
                // update stat with new data
                $mTime = time();
-               $stat['size'] = filesize($tmpFile);
+               $stat['size'] = (int)$size;
                $stat['mtime'] = $mTime;
                $stat['storage_mtime'] = $mTime;
 
-               // run path based detection first, to use file extension because $tmpFile is only a random string
                $mimetypeDetector = \OC::$server->getMimeTypeDetector();
                $mimetype = $mimetypeDetector->detectPath($path);
-               if ($mimetype === 'application/octet-stream') {
-                       $mimetype = $mimetypeDetector->detect($tmpFile);
-               }
 
                $stat['mimetype'] = $mimetype;
                $stat['etag'] = $this->getETag($path);
@@ -408,7 +432,20 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
                $fileId = $this->getCache()->put($path, $stat);
                try {
                        //upload to object storage
-                       $this->objectStore->writeObject($this->getURN($fileId), fopen($tmpFile, 'r'));
+                       if ($size === null) {
+                               $countStream = CountReadStream::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
+                                       $this->getCache()->update($fileId, [
+                                               'size' => $writtenSize
+                                       ]);
+                                       $size = $writtenSize;
+                               });
+                               $this->objectStore->writeObject($this->getURN($fileId), $countStream);
+                               if (is_resource($countStream)) {
+                                       fclose($countStream);
+                               }
+                       } else {
+                               $this->objectStore->writeObject($this->getURN($fileId), $stream);
+                       }
                } catch (\Exception $ex) {
                        $this->getCache()->remove($path);
                        $this->logger->logException($ex, [
@@ -417,20 +454,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
                        ]);
                        throw $ex; // make this bubble up
                }
-       }
 
-       /**
-        * external changes are not supported, exclusive access to the object storage is assumed
-        *
-        * @param string $path
-        * @param int $time
-        * @return false
-        */
-       public function hasUpdated($path, $time) {
-               return false;
-       }
-
-       public function needsPartFile() {
-               return false;
+               return $size;
        }
 }
index b6c82f3a1dfb9da72cd6ef13aa1e06a083d0fb72..6324050b472d543423bb2c3ff4733e93eda82fa3 100644 (file)
@@ -54,6 +54,7 @@ use OCP\Files\InvalidPathException;
 use OCP\Files\ReservedWordException;
 use OCP\Files\Storage\ILockingStorage;
 use OCP\Files\Storage\IStorage;
+use OCP\Files\Storage\IWriteStreamStorage;
 use OCP\ILogger;
 use OCP\Lock\ILockingProvider;
 use OCP\Lock\LockedException;
@@ -69,7 +70,7 @@ use OCP\Lock\LockedException;
  * Some \OC\Files\Storage\Common methods call functions which are first defined
  * in classes which extend it, e.g. $this->stat() .
  */
-abstract class Common implements Storage, ILockingStorage {
+abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 
        use LocalTempFileTrait;
 
@@ -809,4 +810,20 @@ abstract class Common implements Storage, ILockingStorage {
        public function needsPartFile() {
                return true;
        }
+
+       /**
+        * fallback implementation
+        *
+        * @param string $path
+        * @param resource $stream
+        * @param int $size
+        * @return int
+        */
+       public function writeStream(string $path, $stream, int $size = null): int {
+               $target = $this->fopen($path, 'w');
+               list($count, $result) = \OC_Helper::streamCopy($stream, $target);
+               fclose($stream);
+               fclose($target);
+               return $count;
+       }
 }
index 46b53dcf95c57bd5ce0efe2f63363d0565e94b5d..5f7232e64b3a99ee8ed0e6f7e38da22cd29a158d 100644 (file)
@@ -462,4 +462,8 @@ class Local extends \OC\Files\Storage\Common {
                        return parent::moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
                }
        }
+
+       public function writeStream(string $path, $stream, int $size = null): int {
+               return (int)file_put_contents($this->getSourcePath($path), $stream);
+       }
 }
diff --git a/lib/private/Files/Stream/CountReadStream.php b/lib/private/Files/Stream/CountReadStream.php
new file mode 100644 (file)
index 0000000..93cadf8
--- /dev/null
@@ -0,0 +1,65 @@
+<?php declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Stream;
+
+use Icewind\Streams\Wrapper;
+
+class CountReadStream extends Wrapper {
+       /** @var int */
+       private $count;
+
+       /** @var callback */
+       private $callback;
+
+       public static function wrap($source, $callback) {
+               $context = stream_context_create(array(
+                       'count' => array(
+                               'source' => $source,
+                               'callback' => $callback,
+                       )
+               ));
+               return Wrapper::wrapSource($source, $context, 'count', self::class);
+       }
+
+       public function dir_opendir($path, $options) {
+               return false;
+       }
+
+       public function stream_open($path, $mode, $options, &$opened_path) {
+               $context = $this->loadContext('count');
+
+               $this->callback = $context['callback'];
+               return true;
+       }
+
+       public function stream_read($count) {
+               $result = parent::stream_read($count);
+               $this->count += strlen($result);
+               return $result;
+       }
+
+       public function stream_close() {
+               $result = parent::stream_close();
+               call_user_func($this->callback, $this->count);
+               return $result;
+       }
+}
diff --git a/lib/public/Files/Storage/IWriteStreamStorage.php b/lib/public/Files/Storage/IWriteStreamStorage.php
new file mode 100644 (file)
index 0000000..39a28dd
--- /dev/null
@@ -0,0 +1,40 @@
+<?php declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCP\Files\Storage;
+
+/**
+ * Interface that adds the ability to write a stream directly to file
+ *
+ * @since 15.0.0
+ */
+interface IWriteStreamStorage extends IStorage {
+       /**
+        * Write the data from a stream to a file
+        *
+        * @param string $path
+        * @param resource $stream
+        * @param int|null $size the size of the stream if known in advance
+        * @return int the number of bytes written
+        * @since 15.0.0
+        */
+       public function writeStream(string $path, $stream, int $size = null): int;
+}
diff --git a/tests/lib/Files/Stream/CountReadStreamTest.php b/tests/lib/Files/Stream/CountReadStreamTest.php
new file mode 100644 (file)
index 0000000..99291d1
--- /dev/null
@@ -0,0 +1,49 @@
+<?php declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Test\Files\Stream;
+
+use OC\Files\Stream\CountReadStream;
+use Test\TestCase;
+
+class CountReadStreamTest extends TestCase {
+       private function getStream($data) {
+               $handle = fopen('php://temp', 'w+');
+               fwrite($handle, $data);
+               rewind($handle);
+               return $handle;
+       }
+
+       public function testBasicCount() {
+               $source = $this->getStream('foo');
+               $stream = CountReadStream::wrap($source, function ($size) {
+                       $this->assertEquals(3, $size);
+               });
+               stream_get_contents($stream);
+       }
+
+       public function testLarger() {
+               $stream = CountReadStream::wrap(fopen(__DIR__ . '/../../../data/testimage.mp4', 'r'), function ($size) {
+                       $this->assertEquals(383631, $size);
+               });
+               stream_get_contents($stream);
+       }
+}