Initial implementation of the new chunked upload - as specified in https://dragotin.wordpress.com/2015/06/22/owncloud-chunking-ng/

10 files changed, 712 insertions, 1 deletions

diff --git a/apps/dav/bin/chunkperf.php b/apps/dav/bin/chunkperf.php
new file mode 100644
index 00000000000..a193f001a41
--- /dev/null
+++ b/apps/dav/bin/chunkperf.php
@@ -0,0 +1,76 @@
+<?php
+/**
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ *
+ * @copyright Copyright (c) 2016, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+require '../../../../3rdparty/autoload.php';
+
+if ($argc !== 6) {
+	echo "Invalid number of arguments" . PHP_EOL;
+	exit;
+}
+
+/**
+ * @param \Sabre\DAV\Client $client
+ * @param $uploadUrl
+ * @return mixed
+ */
+function request($client, $method, $uploadUrl, $data = null, $headers = []) {
+	echo "$method $uploadUrl ... ";
+	$t0 = microtime(true);
+	$result = $client->request($method, $uploadUrl, $data, $headers);
+	$t1 = microtime(true);
+	echo $result['statusCode'] . " - " . ($t1 - $t0) . ' seconds' . PHP_EOL;
+	if (!in_array($result['statusCode'],  [200, 201])) {
+		echo $result['body'] . PHP_EOL;
+	}
+	return $result;
+}
+
+$baseUri = $argv[1];
+$userName = $argv[2];
+$password = $argv[3];
+$file = $argv[4];
+$chunkSize = $argv[5] * 1024 * 1024;
+
+$client = new \Sabre\DAV\Client([
+	'baseUri' => $baseUri,
+	'userName' => $userName,
+	'password' => $password
+]);
+
+$transfer = uniqid('transfer', true);
+$uploadUrl = "$baseUri/uploads/$userName/$transfer";
+
+request($client, 'MKCOL', $uploadUrl);
+
+$size = filesize($file);
+$stream = fopen($file, 'r');
+
+$index = 0;
+while(!feof($stream)) {
+	request($client, 'PUT', "$uploadUrl/$index", fread($stream, $chunkSize));
+	$index++;
+}
+
+$destination = pathinfo($file, PATHINFO_BASENAME);
+//echo "Moving $uploadUrl/.file to it's final destination $baseUri/files/$userName/$destination" . PHP_EOL;
+request($client, 'MOVE', "$uploadUrl/.file", null, [
+	'Destination' => "$baseUri/files/$userName/$destination"
+]);
diff --git a/apps/dav/lib/connector/sabre/file.php b/apps/dav/lib/connector/sabre/file.php
index 6b698c6e5a9..943e9150e74 100644
--- a/apps/dav/lib/connector/sabre/file.php
+++ b/apps/dav/lib/connector/sabre/file.php
@@ -143,7 +143,7 @@ class File extends Node implements IFile {
 			// if content length is sent by client:
 			// double check if the file was fully received
 			// compare expected and actual size
-			if (isset($_SERVER['CONTENT_LENGTH']) && $_SERVER['REQUEST_METHOD'] !== 'LOCK') {
+			if (isset($_SERVER['CONTENT_LENGTH']) && $_SERVER['REQUEST_METHOD'] === 'PUT') {
 				$expected = $_SERVER['CONTENT_LENGTH'];
 				if ($count != $expected) {
 					throw new BadRequest('expected filesize ' . $expected . ' got ' . $count);
diff --git a/apps/dav/lib/rootcollection.php b/apps/dav/lib/rootcollection.php
index ea796c09175..b6e1747e990 100644
--- a/apps/dav/lib/rootcollection.php
+++ b/apps/dav/lib/rootcollection.php
@@ -89,6 +89,9 @@ class RootCollection extends SimpleCollection {
 		$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system');
 		$systemAddressBookRoot->disableListing = $disableListing;
 
+		$uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users');
+		$uploadCollection->disableListing = $disableListing;
+
 		$children = [
 				new SimpleCollection('principals', [
 						$userPrincipals,
@@ -102,6 +105,7 @@ class RootCollection extends SimpleCollection {
 				$systemTagCollection,
 				$systemTagRelationsCollection,
 				$commentsCollection,
+				$uploadCollection,
 		];
 
 		parent::__construct('root', $children);
diff --git a/apps/dav/lib/upload/assemblystream.php b/apps/dav/lib/upload/assemblystream.php
new file mode 100644
index 00000000000..4b80a591ce4
--- /dev/null
+++ b/apps/dav/lib/upload/assemblystream.php
@@ -0,0 +1,234 @@
+<?php
+
+namespace OCA\DAV\Upload;
+
+use Sabre\DAV\IFile;
+
+/**
+ * Class AssemblyStream
+ *
+ * The assembly stream is a virtual stream that wraps multiple chunks.
+ * Reading from the stream transparently accessed the underlying chunks and
+ * give a representation as if they were already merged together.
+ *
+ * @package OCA\DAV\Upload
+ */
+class AssemblyStream implements \Icewind\Streams\File {
+
+	/** @var resource */
+	private $context;
+
+	/** @var IFile[] */
+	private $nodes;
+
+	/** @var int */
+	private $pos = 0;
+
+	/** @var array */
+	private $sortedNodes;
+
+	/** @var int */
+	private $size;
+
+	/**
+	 * @param string $path
+	 * @param string $mode
+	 * @param int $options
+	 * @param string &$opened_path
+	 * @return bool
+	 */
+	public function stream_open($path, $mode, $options, &$opened_path) {
+		$this->loadContext('assembly');
+
+		// sort the nodes
+		$nodes = $this->nodes;
+		// http://stackoverflow.com/a/10985500
+		@usort($nodes, function(IFile $a, IFile $b) {
+			return strcmp($a->getName(), $b->getName());
+		});
+		$this->nodes = $nodes;
+
+		// build additional information
+		$this->sortedNodes = [];
+		$start = 0;
+		foreach($this->nodes as $node) {
+			$size = $node->getSize();
+			$name = $node->getName();
+			$this->sortedNodes[$name] = ['node' => $node, 'start' => $start, 'end' => $start + $size];
+			$start += $size;
+			$this->size = $start;
+		}
+		return true;
+	}
+
+	/**
+	 * @param string $offset
+	 * @param int $whence
+	 * @return bool
+	 */
+	public function stream_seek($offset, $whence = SEEK_SET) {
+		return false;
+	}
+
+	/**
+	 * @return int
+	 */
+	public function stream_tell() {
+		return $this->pos;
+	}
+
+	/**
+	 * @param int $count
+	 * @return string
+	 */
+	public function stream_read($count) {
+
+		list($node, $posInNode) = $this->getNodeForPosition($this->pos);
+		if (is_null($node)) {
+			return null;
+		}
+		$stream = $this->getStream($node);
+
+		fseek($stream, $posInNode);
+		$data = fread($stream, $count);
+		$read = strlen($data);
+
+		// update position
+		$this->pos += $read;
+		return $data;
+	}
+
+	/**
+	 * @param string $data
+	 * @return int
+	 */
+	public function stream_write($data) {
+		return false;
+	}
+
+	/**
+	 * @param int $option
+	 * @param int $arg1
+	 * @param int $arg2
+	 * @return bool
+	 */
+	public function stream_set_option($option, $arg1, $arg2) {
+		return false;
+	}
+
+	/**
+	 * @param int $size
+	 * @return bool
+	 */
+	public function stream_truncate($size) {
+		return false;
+	}
+
+	/**
+	 * @return array
+	 */
+	public function stream_stat() {
+		return [];
+	}
+
+	/**
+	 * @param int $operation
+	 * @return bool
+	 */
+	public function stream_lock($operation) {
+		return false;
+	}
+
+	/**
+	 * @return bool
+	 */
+	public function stream_flush() {
+		return false;
+	}
+
+	/**
+	 * @return bool
+	 */
+	public function stream_eof() {
+		return $this->pos >= $this->size;
+	}
+
+	/**
+	 * @return bool
+	 */
+	public function stream_close() {
+		return true;
+	}
+
+
+	/**
+	 * Load the source from the stream context and return the context options
+	 *
+	 * @param string $name
+	 * @return array
+	 * @throws \Exception
+	 */
+	protected function loadContext($name) {
+		$context = stream_context_get_options($this->context);
+		if (isset($context[$name])) {
+			$context = $context[$name];
+		} else {
+			throw new \BadMethodCallException('Invalid context, "' . $name . '" options not set');
+		}
+		if (isset($context['nodes']) and is_array($context['nodes'])) {
+			$this->nodes = $context['nodes'];
+		} else {
+			throw new \BadMethodCallException('Invalid context, nodes not set');
+		}
+		return $context;
+	}
+
+	/**
+	 * @param IFile[] $nodes
+	 * @return resource
+	 *
+	 * @throws \BadMethodCallException
+	 */
+	public static function wrap(array $nodes) {
+		$context = stream_context_create([
+			'assembly' => [
+				'nodes' => $nodes]
+		]);
+		stream_wrapper_register('assembly', '\OCA\DAV\Upload\AssemblyStream');
+		try {
+			$wrapped = fopen('assembly://', 'r', null, $context);
+		} catch (\BadMethodCallException $e) {
+			stream_wrapper_unregister('assembly');
+			throw $e;
+		}
+		stream_wrapper_unregister('assembly');
+		return $wrapped;
+	}
+
+	/**
+	 * @param $pos
+	 * @return IFile | null
+	 */
+	private function getNodeForPosition($pos) {
+		foreach($this->sortedNodes as $node) {
+			if ($pos >= $node['start'] && $pos < $node['end']) {
+				return [$node['node'], $pos - $node['start']];
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * @param IFile $node
+	 * @return resource
+	 */
+	private function getStream(IFile $node) {
+		$data = $node->get();
+		if (is_resource($data)) {
+			return $data;
+		}
+
+		return fopen('data://text/plain,' . $data,'r');
+	}
+
+}
diff --git a/apps/dav/lib/upload/futurefile.php b/apps/dav/lib/upload/futurefile.php
new file mode 100644
index 00000000000..aca81afc055
--- /dev/null
+++ b/apps/dav/lib/upload/futurefile.php
@@ -0,0 +1,103 @@
+<?php
+
+namespace OCA\DAV\Upload;
+
+use OCA\DAV\Connector\Sabre\Directory;
+use OCA\DAV\Upload\AssemblyStream;
+use Sabre\DAV\Exception\Forbidden;
+use Sabre\DAV\IFile;
+
+/**
+ * Class FutureFile
+ *
+ * The FutureFile is a SabreDav IFile which connects the chunked upload directory
+ * with the AssemblyStream, who does the final assembly job
+ *
+ * @package OCA\DAV\Upload
+ */
+class FutureFile implements \Sabre\DAV\IFile {
+
+	/** @var Directory */
+	private $root;
+	/** @var string */
+	private $name;
+
+	/**
+	 * @param Directory $root
+	 * @param string $name
+	 */
+	function __construct(Directory $root, $name) {
+		$this->root = $root;
+		$this->name = $name;
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function put($data) {
+		throw new Forbidden('Permission denied to put into this file');
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function get() {
+		$nodes = $this->root->getChildren();
+		return AssemblyStream::wrap($nodes);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getContentType() {
+		return 'application/octet-stream';
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getETag() {
+		return $this->root->getETag();
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getSize() {
+		$children = $this->root->getChildren();
+		$sizes = array_map(function($node) {
+			/** @var IFile $node */
+			return $node->getSize();
+		}, $children);
+
+		return array_sum($sizes);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function delete() {
+		$this->root->delete();
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getName() {
+		return $this->name;
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function setName($name) {
+		throw new Forbidden('Permission denied to rename this file');
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getLastModified() {
+		return $this->root->getLastModified();
+	}
+}
diff --git a/apps/dav/lib/upload/rootcollection.php b/apps/dav/lib/upload/rootcollection.php
new file mode 100644
index 00000000000..673a3734318
--- /dev/null
+++ b/apps/dav/lib/upload/rootcollection.php
@@ -0,0 +1,23 @@
+<?php
+
+namespace OCA\DAV\Upload;
+
+use Sabre\DAVACL\AbstractPrincipalCollection;
+
+class RootCollection extends AbstractPrincipalCollection {
+
+	/**
+	 * @inheritdoc
+	 */
+	function getChildForPrincipal(array $principalInfo) {
+		return new UploadHome($principalInfo);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	function getName() {
+		return 'uploads';
+	}
+
+}
diff --git a/apps/dav/lib/upload/uploadfolder.php b/apps/dav/lib/upload/uploadfolder.php
new file mode 100644
index 00000000000..01fbf1f8dc9
--- /dev/null
+++ b/apps/dav/lib/upload/uploadfolder.php
@@ -0,0 +1,61 @@
+<?php
+
+namespace OCA\DAV\Upload;
+
+use OCA\DAV\Connector\Sabre\Directory;
+use Sabre\DAV\Exception\Forbidden;
+use Sabre\DAV\ICollection;
+
+class UploadFolder implements ICollection {
+
+	private $node;
+
+	function __construct(Directory $node) {
+		$this->node = $node;
+	}
+
+	function createFile($name, $data = null) {
+		// TODO: verify name - should be a simple number
+		$this->node->createFile($name, $data);
+	}
+
+	function createDirectory($name) {
+		throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
+	}
+
+	function getChild($name) {
+		if ($name === '.file') {
+			return new FutureFile($this->node, '.file');
+		}
+		return $this->node->getChild($name);
+	}
+
+	function getChildren() {
+		$children = $this->node->getChildren();
+		$children[] = new FutureFile($this->node, '.file');
+		return $children;
+	}
+
+	function childExists($name) {
+		if ($name === '.file') {
+			return true;
+		}
+		return $this->node->childExists($name);
+	}
+
+	function delete() {
+		$this->node->delete();
+	}
+
+	function getName() {
+		return $this->node->getName();
+	}
+
+	function setName($name) {
+		throw new Forbidden('Permission denied to rename this folder');
+	}
+
+	function getLastModified() {
+		return $this->node->getLastModified();
+	}
+}
diff --git a/apps/dav/lib/upload/uploadhome.php b/apps/dav/lib/upload/uploadhome.php
new file mode 100644
index 00000000000..ae4dcfa4931
--- /dev/null
+++ b/apps/dav/lib/upload/uploadhome.php
@@ -0,0 +1,74 @@
+<?php
+
+namespace OCA\DAV\Upload;
+
+use OC\Files\Filesystem;
+use OC\Files\View;
+use OCA\DAV\Connector\Sabre\Directory;
+use Sabre\DAV\Exception\Forbidden;
+use Sabre\DAV\ICollection;
+
+class UploadHome implements ICollection {
+	/**
+	 * FilesHome constructor.
+	 *
+	 * @param array $principalInfo
+	 */
+	public function __construct($principalInfo) {
+		$this->principalInfo = $principalInfo;
+	}
+
+	function createFile($name, $data = null) {
+		throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
+	}
+
+	function createDirectory($name) {
+		$this->impl()->createDirectory($name);
+	}
+
+	function getChild($name) {
+		return new UploadFolder($this->impl()->getChild($name));
+	}
+
+	function getChildren() {
+		return array_map(function($node) {
+			return new UploadFolder($node);
+		}, $this->impl()->getChildren());
+	}
+
+	function childExists($name) {
+		return !is_null($this->getChild($name));
+	}
+
+	function delete() {
+		$this->impl()->delete();
+	}
+
+	function getName() {
+		return 'uploads';
+	}
+
+	function setName($name) {
+		throw new Forbidden('Permission denied to rename this folder');
+	}
+
+	function getLastModified() {
+		return $this->impl()->getLastModified();
+	}
+
+	/**
+	 * @return Directory
+	 */
+	private function impl() {
+		$rootView = new View();
+		$user = \OC::$server->getUserSession()->getUser();
+		Filesystem::initMountPoints($user->getUID());
+		if (!$rootView->file_exists('/' . $user->getUID() . '/uploads')) {
+			$rootView->mkdir('/' . $user->getUID() . '/uploads');
+		}
+		$view = new View('/' . $user->getUID() . '/uploads');
+		$rootInfo = $view->getFileInfo('');
+		$impl = new Directory($view, $rootInfo);
+		return $impl;
+	}
+}
diff --git a/apps/dav/tests/unit/upload/assemblystreamtest.php b/apps/dav/tests/unit/upload/assemblystreamtest.php
new file mode 100644
index 00000000000..373d525a9dd
--- /dev/null
+++ b/apps/dav/tests/unit/upload/assemblystreamtest.php
@@ -0,0 +1,47 @@
+<?php
+
+class AssemblyStreamTest extends \PHPUnit_Framework_TestCase {
+
+	/**
+	 * @dataProvider providesNodes()
+	 */
+	public function testGetContents($expected, $nodes) {
+		$stream = \OCA\DAV\Upload\AssemblyStream::wrap($nodes);
+		$content = stream_get_contents($stream);
+
+		$this->assertEquals($expected, $content);
+	}
+
+	function providesNodes() {
+		return[
+			'one node only' => ['1234567890', [
+				$this->buildNode('0', '1234567890')
+			]],
+			'two nodes' => ['1234567890', [
+				$this->buildNode('1', '67890'),
+				$this->buildNode('0', '12345')
+			]]
+		];
+	}
+
+	private function buildNode($name, $data) {
+		$node = $this->getMockBuilder('\Sabre\DAV\File')
+			->setMethods(['getName', 'get', 'getSize'])
+			->getMockForAbstractClass();
+
+		$node->expects($this->any())
+			->method('getName')
+			->willReturn($name);
+
+		$node->expects($this->any())
+			->method('get')
+			->willReturn($data);
+
+		$node->expects($this->any())
+			->method('getSize')
+			->willReturn(strlen($data));
+
+		return $node;
+	}
+}
+
diff --git a/apps/dav/tests/unit/upload/futurefiletest.php b/apps/dav/tests/unit/upload/futurefiletest.php
new file mode 100644
index 00000000000..c0c14bf04d7
--- /dev/null
+++ b/apps/dav/tests/unit/upload/futurefiletest.php
@@ -0,0 +1,89 @@
+<?php
+
+class FutureFileTest extends \PHPUnit_Framework_TestCase {
+
+	public function testGetContentType() {
+		$f = $this->mockFutureFile();
+		$this->assertEquals('application/octet-stream', $f->getContentType());
+	}
+
+	public function testGetETag() {
+		$f = $this->mockFutureFile();
+		$this->assertEquals('1234567890', $f->getETag());
+	}
+
+	public function testGetName() {
+		$f = $this->mockFutureFile();
+		$this->assertEquals('foo.txt', $f->getName());
+	}
+
+	public function testGetLastModified() {
+		$f = $this->mockFutureFile();
+		$this->assertEquals(12121212, $f->getLastModified());
+	}
+
+	public function testGetSize() {
+		$f = $this->mockFutureFile();
+		$this->assertEquals(0, $f->getSize());
+	}
+
+	public function testGet() {
+		$f = $this->mockFutureFile();
+		$stream = $f->get();
+		$this->assertTrue(is_resource($stream));
+	}
+
+	public function testDelete() {
+		$d = $this->getMockBuilder('OCA\DAV\Connector\Sabre\Directory')
+			->disableOriginalConstructor()
+			->setMethods(['delete'])
+			->getMock();
+
+		$d->expects($this->once())
+			->method('delete');
+
+		$f = new \OCA\DAV\Upload\FutureFile($d, 'foo.txt');
+		$f->delete();
+	}
+
+	/**
+	 * @expectedException Sabre\DAV\Exception\Forbidden
+	 */
+	public function testPut() {
+		$f = $this->mockFutureFile();
+		$f->put('');
+	}
+
+	/**
+	 * @expectedException Sabre\DAV\Exception\Forbidden
+	 */
+	public function testSetName() {
+		$f = $this->mockFutureFile();
+		$f->setName('');
+	}
+
+	/**
+	 * @return \OCA\DAV\Upload\FutureFile
+	 */
+	private function mockFutureFile() {
+		$d = $this->getMockBuilder('OCA\DAV\Connector\Sabre\Directory')
+			->disableOriginalConstructor()
+			->setMethods(['getETag', 'getLastModified', 'getChildren'])
+			->getMock();
+
+		$d->expects($this->any())
+			->method('getETag')
+			->willReturn('1234567890');
+
+		$d->expects($this->any())
+			->method('getLastModified')
+			->willReturn(12121212);
+
+		$d->expects($this->any())
+			->method('getChildren')
+			->willReturn([]);
+
+		return new \OCA\DAV\Upload\FutureFile($d, 'foo.txt');
+	}
+}
+