diff options
| -rw-r--r-- | apps/dav/composer/composer/autoload_classmap.php | 2 | ||||
| -rw-r--r-- | apps/dav/composer/composer/autoload_static.php | 2 | ||||
| -rw-r--r-- | apps/dav/lib/BulkUpload/BulkUploadPlugin.php | 100 | ||||
| -rw-r--r-- | apps/dav/lib/BulkUpload/MultipartRequestParser.php | 239 | ||||
| -rw-r--r-- | apps/dav/lib/Capabilities.php | 2 | ||||
| -rw-r--r-- | apps/dav/lib/Server.php | 6 | ||||
| -rwxr-xr-x | apps/dav/tests/benchmarks/benchmark.sh | 52 | ||||
| -rwxr-xr-x | apps/dav/tests/benchmarks/bulk_upload.sh | 78 | ||||
| -rwxr-xr-x | apps/dav/tests/benchmarks/single_upload.sh | 61 | ||||
| -rw-r--r-- | apps/dav/tests/unit/CapabilitiesTest.php | 1 | ||||
| -rw-r--r-- | apps/dav/tests/unit/Files/MultipartRequestParserTest.php | 281 | ||||
| -rw-r--r-- | build/integration/features/bootstrap/WebDav.php | 51 | ||||
| -rw-r--r-- | build/integration/features/webdav-related.feature | 11 |
13 files changed, 886 insertions, 0 deletions
diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 62c41a0828d..1a536c98272 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -22,6 +22,8 @@ return array( 'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => $baseDir . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php', 'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php', + 'OCA\\DAV\\BulkUpload\\BulkUploadPlugin' => $baseDir . '/../lib/BulkUpload/BulkUploadPlugin.php', + 'OCA\\DAV\\BulkUpload\\MultipartRequestParser' => $baseDir . '/../lib/BulkUpload/MultipartRequestParser.php', 'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index 5d5f57eb51b..b65d4477800 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -37,6 +37,8 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php', 'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php', + 'OCA\\DAV\\BulkUpload\\BulkUploadPlugin' => __DIR__ . '/..' . '/../lib/BulkUpload/BulkUploadPlugin.php', + 'OCA\\DAV\\BulkUpload\\MultipartRequestParser' => __DIR__ . '/..' . '/../lib/BulkUpload/MultipartRequestParser.php', 'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php', diff --git a/apps/dav/lib/BulkUpload/BulkUploadPlugin.php b/apps/dav/lib/BulkUpload/BulkUploadPlugin.php new file mode 100644 index 00000000000..0766ae37a17 --- /dev/null +++ b/apps/dav/lib/BulkUpload/BulkUploadPlugin.php @@ -0,0 +1,100 @@ +<?php +/** + * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me> + * + * @author Louis Chemineau <louis@chmn.me> + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OCA\DAV\BulkUpload; + +use Psr\Log\LoggerInterface; +use Sabre\DAV\Server; +use Sabre\DAV\ServerPlugin; +use Sabre\HTTP\RequestInterface; +use Sabre\HTTP\ResponseInterface; +use OCP\Files\Folder; +use OCP\AppFramework\Http; + +class BulkUploadPlugin extends ServerPlugin { + + /** @var Folder */ + private $userFolder; + + /** @var LoggerInterface */ + private $logger; + + public function __construct(Folder $userFolder, LoggerInterface $logger) { + $this->userFolder = $userFolder; + $this->logger = $logger; + } + + /** + * Register listener on POST requests with the httpPost method. + */ + public function initialize(Server $server): void { + $server->on('method:POST', [$this, 'httpPost'], 10); + } + + /** + * Handle POST requests on /dav/bulk + * - parsing is done with a MultipartContentsParser object + * - writing is done with the userFolder service + * + * Will respond with an object containing an ETag for every written files. + */ + public function httpPost(RequestInterface $request, ResponseInterface $response): bool { + // Limit bulk upload to the /dav/bulk endpoint + if ($request->getPath() !== "bulk") { + return true; + } + + $multiPartParser = new MultipartRequestParser($request); + $writtenFiles = []; + + while (!$multiPartParser->isAtLastBoundary()) { + try { + [$headers, $content] = $multiPartParser->parseNextPart(); + } catch (\Exception $e) { + // Return early if an error occurs during parsing. + $this->logger->error($e->getMessage()); + $response->setStatus(Http::STATUS_BAD_REQUEST); + $response->setBody(json_encode($writtenFiles)); + return false; + } + + try { + $node = $this->userFolder->newFile($headers['x-file-path'], $content); + $writtenFiles[$headers['x-file-path']] = [ + "error" => false, + "etag" => $node->getETag(), + ]; + } catch (\Exception $e) { + $this->logger->error($e->getMessage(), ['path' => $headers['x-file-path']]); + $writtenFiles[$headers['x-file-path']] = [ + "error" => true, + "message" => $e->getMessage(), + ]; + } + } + + $response->setStatus(Http::STATUS_OK); + $response->setBody(json_encode($writtenFiles)); + + return false; + } +} diff --git a/apps/dav/lib/BulkUpload/MultipartRequestParser.php b/apps/dav/lib/BulkUpload/MultipartRequestParser.php new file mode 100644 index 00000000000..7554447fc93 --- /dev/null +++ b/apps/dav/lib/BulkUpload/MultipartRequestParser.php @@ -0,0 +1,239 @@ +<?php +/** + * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me> + * + * @author Louis Chemineau <louis@chmn.me> + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OCA\DAV\BulkUpload; + +use Sabre\HTTP\RequestInterface; +use Sabre\DAV\Exception; +use Sabre\DAV\Exception\BadRequest; +use Sabre\DAV\Exception\LengthRequired; +use OCP\AppFramework\Http; + +class MultipartRequestParser { + + /** @var resource */ + private $stream; + + /** @var string */ + private $boundary = ""; + + /** @var string */ + private $lastBoundary = ""; + + /** + * @throws BadRequest + */ + public function __construct(RequestInterface $request) { + $stream = $request->getBody(); + $contentType = $request->getHeader('Content-Type'); + + if (!is_resource($stream)) { + throw new BadRequest('Body should be of type resource'); + } + + if ($contentType === null) { + throw new BadRequest("Content-Type can not be null"); + } + + $this->stream = $stream; + + $boundary = $this->parseBoundaryFromHeaders($contentType); + $this->boundary = '--'.$boundary."\r\n"; + $this->lastBoundary = '--'.$boundary."--\r\n"; + } + + /** + * Parse the boundary from the Content-Type header. + * Example: Content-Type: "multipart/related; boundary=boundary_bf38b9b4b10a303a28ed075624db3978" + * + * @throws BadRequest + */ + private function parseBoundaryFromHeaders(string $contentType): string { + try { + [$mimeType, $boundary] = explode(';', $contentType); + [$boundaryKey, $boundaryValue] = explode('=', $boundary); + } catch (\Exception $e) { + throw new BadRequest("Error while parsing boundary in Content-Type header.", Http::STATUS_BAD_REQUEST, $e); + } + + $boundaryValue = trim($boundaryValue); + + // Remove potential quotes around boundary value. + if (substr($boundaryValue, 0, 1) == '"' && substr($boundaryValue, -1) == '"') { + $boundaryValue = substr($boundaryValue, 1, -1); + } + + if (trim($mimeType) !== 'multipart/related') { + throw new BadRequest('Content-Type must be multipart/related'); + } + + if (trim($boundaryKey) !== 'boundary') { + throw new BadRequest('Boundary is invalid'); + } + + return $boundaryValue; + } + + /** + * Check whether the stream's cursor is sitting right before the provided string. + * + * @throws Exception + */ + private function isAt(string $expectedContent): bool { + $expectedContentLength = strlen($expectedContent); + + $content = fread($this->stream, $expectedContentLength); + if ($content === false) { + throw new Exception('An error occurred while checking content'); + } + + $seekBackResult = fseek($this->stream, -$expectedContentLength, SEEK_CUR); + if ($seekBackResult === -1) { + throw new Exception("Unknown error while seeking content", Http::STATUS_INTERNAL_SERVER_ERROR); + } + + return $expectedContent === $content; + } + + + /** + * Check whether the stream's cursor is sitting right before the boundary. + */ + private function isAtBoundary(): bool { + return $this->isAt($this->boundary); + } + + /** + * Check whether the stream's cursor is sitting right before the last boundary. + */ + public function isAtLastBoundary(): bool { + return $this->isAt($this->lastBoundary); + } + + /** + * Parse and return the next part of the multipart headers. + * + * Example: + * --boundary_azertyuiop + * Header1: value + * Header2: value + * + * Content of + * the part + * + */ + public function parseNextPart(): array { + $this->readBoundary(); + + $headers = $this->readPartHeaders(); + + $content = $this->readPartContent($headers["content-length"], $headers["x-file-md5"]); + + return [$headers, $content]; + } + + /** + * Read the boundary and check its content. + * + * @throws BadRequest + */ + private function readBoundary(): string { + if (!$this->isAtBoundary()) { + throw new BadRequest("Boundary not found where it should be."); + } + + return fread($this->stream, strlen($this->boundary)); + } + + /** + * Return the headers of a part of the multipart body. + * + * @throws Exception + * @throws BadRequest + * @throws LengthRequired + */ + private function readPartHeaders(): array { + $headers = []; + + while (($line = fgets($this->stream)) !== "\r\n") { + if ($line === false) { + throw new Exception('An error occurred while reading headers of a part'); + } + + try { + [$key, $value] = explode(':', $line, 2); + $headers[strtolower(trim($key))] = trim($value); + } catch (\Exception $e) { + throw new BadRequest('An error occurred while parsing headers of a part', Http::STATUS_BAD_REQUEST, $e); + } + } + + if (!isset($headers["content-length"])) { + throw new LengthRequired("The Content-Length header must not be null."); + } + + if (!isset($headers["x-file-md5"])) { + throw new BadRequest("The X-File-MD5 header must not be null."); + } + + return $headers; + } + + /** + * Return the content of a part of the multipart body. + * + * @throws Exception + * @throws BadRequest + */ + private function readPartContent(int $length, string $md5): string { + $computedMd5 = $this->computeMd5Hash($length); + + if ($md5 !== $computedMd5) { + throw new BadRequest("Computed md5 hash is incorrect."); + } + + $content = stream_get_line($this->stream, $length); + + if ($content === false) { + throw new Exception("Fail to read part's content."); + } + + if (feof($this->stream)) { + throw new Exception("Unexpected EOF while reading stream."); + } + + // Read '\r\n'. + stream_get_contents($this->stream, 2); + + return $content; + } + + /** + * Compute the MD5 hash of the next x bytes. + */ + private function computeMd5Hash(int $length): string { + $context = hash_init('md5'); + hash_update_stream($context, $this->stream, $length); + fseek($this->stream, -$length, SEEK_CUR); + return hash_final($context); + } +} diff --git a/apps/dav/lib/Capabilities.php b/apps/dav/lib/Capabilities.php index 5d4e3c05077..ce60bccfd0b 100644 --- a/apps/dav/lib/Capabilities.php +++ b/apps/dav/lib/Capabilities.php @@ -3,6 +3,7 @@ * @copyright Copyright (c) 2016, ownCloud GmbH * * @author Thomas Müller <thomas.mueller@tmit.eu> + * @author Louis Chemineau <louis@chmn.me> * * @license AGPL-3.0 * @@ -28,6 +29,7 @@ class Capabilities implements ICapability { return [ 'dav' => [ 'chunking' => '1.0', + 'bulkupload' => '1.0', ] ]; } diff --git a/apps/dav/lib/Server.php b/apps/dav/lib/Server.php index e9634f670d3..055c37f8472 100644 --- a/apps/dav/lib/Server.php +++ b/apps/dav/lib/Server.php @@ -34,6 +34,7 @@ */ namespace OCA\DAV; +use Psr\Log\LoggerInterface; use OCA\DAV\AppInfo\PluginManager; use OCA\DAV\CalDAV\BirthdayService; use OCA\DAV\CardDAV\HasPhotoPlugin; @@ -62,6 +63,7 @@ use OCA\DAV\DAV\PublicAuth; use OCA\DAV\Events\SabrePluginAuthInitEvent; use OCA\DAV\Files\BrowserErrorPagePlugin; use OCA\DAV\Files\LazySearchBackend; +use OCA\DAV\BulkUpload\BulkUploadPlugin; use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin; use OCA\DAV\SystemTag\SystemTagPlugin; use OCA\DAV\Upload\ChunkingPlugin; @@ -294,6 +296,10 @@ class Server { \OC::$server->getShareManager(), $view )); + $logger = \OC::$server->get(LoggerInterface::class); + $this->server->addPlugin( + new BulkUploadPlugin($userFolder, $logger) + ); } $this->server->addPlugin(new \OCA\DAV\CalDAV\BirthdayCalendar\EnablePlugin( \OC::$server->getConfig(), diff --git a/apps/dav/tests/benchmarks/benchmark.sh b/apps/dav/tests/benchmarks/benchmark.sh new file mode 100755 index 00000000000..27d7c4ecbc7 --- /dev/null +++ b/apps/dav/tests/benchmarks/benchmark.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -eu + +# benchmark.sh + +export KB=1000 +export MB=$((KB*1000)) + +MAX_UPLOAD_SIZE=$((512*KB)) + +export CONCURRENCY=5 +export BANDWIDTH=$((100*MB/CONCURRENCY)) + +FILE_SIZES=($((1*KB)) $((10*KB)) $((100*KB))) + +echo "Concurrency: $CONCURRENCY" +echo "Bandwidth: $BANDWIDTH" + +md_output="# Bulk upload benchmark\n" +md_output+="\n" +md_output+="- Concurrency: $CONCURRENCY\n" +md_output+="- Bandwidth: ${BANDWIDTH}B\n" +md_output+="\n" +md_output+="| Nb | Size (B) | Bundle (sec) | Single (sec) |\n" +md_output+="|---|---|---|---|\n" + +requests_count='1 2 3 4 5' + +for size in "${FILE_SIZES[@]}" +do + nb=$((MAX_UPLOAD_SIZE/size)) + + echo "- Upload of $nb tiny file of ${size}B" + echo " - Bundled" + start=$(date +%s) + echo "$requests_count" | xargs -d ' ' -P $CONCURRENCY -I{} ./bulk_upload.sh "$nb" "$size" + end=$(date +%s) + bulk_exec_time=$((end-start)) + echo "${bulk_exec_time}s" + + echo " - Single" + start=$(date +%s) + echo "$requests_count" | xargs -d ' ' -P $CONCURRENCY -I{} ./single_upload.sh "$nb" "$size" + end=$(date +%s) + single_exec_time=$((end-start)) + echo "${single_exec_time}s" + + md_output+="| $nb | $size | $bulk_exec_time | $single_exec_time |\n" +done + +echo -en "$md_output"
\ No newline at end of file diff --git a/apps/dav/tests/benchmarks/bulk_upload.sh b/apps/dav/tests/benchmarks/bulk_upload.sh new file mode 100755 index 00000000000..862ddfe461f --- /dev/null +++ b/apps/dav/tests/benchmarks/bulk_upload.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -eu + +# bulk_upload.sh <nb-of-files> <size-of-files> + +KB=${KB:-100} +MB=${MB:-$((KB*1000))} + +NB=$1 +SIZE=$2 + +CONCURRENCY=${CONCURRENCY:-1} +BANDWIDTH=${BANDWIDTH:-$((100*MB/CONCURRENCY))} + +USER="admin" +PASS="password" +SERVER="nextcloud.test" +UPLOAD_PATH="/tmp/bulk_upload_request_$(openssl rand --hex 8).txt" +BOUNDARY="boundary_$(openssl rand --hex 8)" +LOCAL_FOLDER="/tmp/bulk_upload/${BOUNDARY}_${NB}_${SIZE}" +REMOTE_FOLDER="/bulk_upload/${BOUNDARY}_${NB}_${SIZE}" + +mkdir --parent "$LOCAL_FOLDER" + +for ((i=1; i<="$NB"; i++)) +do + file_name=$(openssl rand --hex 8) + file_local_path="$LOCAL_FOLDER/$file_name.txt" + file_remote_path="$REMOTE_FOLDER/$file_name.txt" + head -c "$SIZE" /dev/urandom > "$file_local_path" + file_mtime=$(stat -c %Y "$file_local_path") + file_hash=$(md5sum "$file_local_path" | awk '{ print $1 }') + file_size=$(du -sb "$file_local_path" | awk '{ print $1 }') + + { + echo -en "--$BOUNDARY\r\n" + # echo -en "Content-ID: $file_name\r\n" + echo -en "X-File-Path: $file_remote_path\r\n" + echo -en "X-File-Mtime: $file_mtime\r\n" + # echo -en "X-File-Id: $file_id\r\n" + echo -en "X-File-Md5: $file_hash\r\n" + echo -en "Content-Length: $file_size\r\n" + echo -en "\r\n" >> "$UPLOAD_PATH" + + cat "$file_local_path" + echo -en "\r\n" >> "$UPLOAD_PATH" + } >> "$UPLOAD_PATH" +done + +echo -en "--$BOUNDARY--\r\n" >> "$UPLOAD_PATH" + +echo "Creating folder /bulk_upload" +curl \ + -X MKCOL \ + -k \ + "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/bulk_upload" > /dev/null + +echo "Creating folder $REMOTE_FOLDER" +curl \ + -X MKCOL \ + -k \ + "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/$REMOTE_FOLDER" + +echo "Uploading $NB files with total size: $(du -sh "$UPLOAD_PATH" | cut -d ' ' -f1)" +echo "Local file is: $UPLOAD_PATH" +curl \ + -X POST \ + -k \ + --progress-bar \ + --limit-rate "${BANDWIDTH}k" \ + --cookie "XDEBUG_PROFILE=true;path=/;" \ + -H "Content-Type: multipart/related; boundary=$BOUNDARY" \ + --data-binary "@$UPLOAD_PATH" \ + "https://$USER:$PASS@$SERVER/remote.php/dav/bulk" + +rm -rf "${LOCAL_FOLDER:?}" +rm "$UPLOAD_PATH" diff --git a/apps/dav/tests/benchmarks/single_upload.sh b/apps/dav/tests/benchmarks/single_upload.sh new file mode 100755 index 00000000000..ec57e66668d --- /dev/null +++ b/apps/dav/tests/benchmarks/single_upload.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -eu + +# single_upload.sh <nb-of-files> <size-of-files> + +export KB=${KB:-100} +export MB=${MB:-$((KB*1000))} + +export NB=$1 +export SIZE=$2 + +export CONCURRENCY=${CONCURRENCY:-1} +export BANDWIDTH=${BANDWIDTH:-$((100*MB/CONCURRENCY))} + +export USER="admin" +export PASS="password" +export SERVER="nextcloud.test" +export UPLOAD_ID="single_$(openssl rand --hex 8)" +export LOCAL_FOLDER="/tmp/single_upload/${UPLOAD_ID}_${NB}_${SIZE}" +export REMOTE_FOLDER="/single_upload/${UPLOAD_ID}_${NB}_${SIZE}" + +mkdir --parent "$LOCAL_FOLDER" + +curl \ + -X MKCOL \ + -k \ + "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/bulk_upload" > /dev/null + +curl \ + -X MKCOL \ + -k \ + --cookie "XDEBUG_SESSION=true;path=/;" \ + "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/$REMOTE_FOLDER" + +upload_file() { + file_name=$(openssl rand --hex 8) + file_local_path="$LOCAL_FOLDER/$file_name.txt" + file_remote_path="$REMOTE_FOLDER/$file_name.txt" + head -c "$SIZE" /dev/urandom > "$file_local_path" + + curl \ + -X PUT \ + -k \ + --limit-rate "${BANDWIDTH}k" \ + --data-binary @"$file_local_path" "https://$USER:$PASS@$SERVER/remote.php/webdav/$file_remote_path" +} +export -f upload_file + +file_list='' +for ((i=1; i<"$NB"; i++)) +do + file_list+="$i " +done +file_list+=$NB + +echo "$file_list" | xargs -d ' ' -P "$((CONCURRENCY/5))" -I{} bash -c "upload_file {}" + +printf "\n" + +rm -rf "${LOCAL_FOLDER:?}"/*
\ No newline at end of file diff --git a/apps/dav/tests/unit/CapabilitiesTest.php b/apps/dav/tests/unit/CapabilitiesTest.php index 719b62115d9..399467f6ed8 100644 --- a/apps/dav/tests/unit/CapabilitiesTest.php +++ b/apps/dav/tests/unit/CapabilitiesTest.php @@ -35,6 +35,7 @@ class CapabilitiesTest extends TestCase { $expected = [ 'dav' => [ 'chunking' => '1.0', + 'bulkupload' => '1.0', ], ]; $this->assertSame($expected, $capabilities->getCapabilities()); diff --git a/apps/dav/tests/unit/Files/MultipartRequestParserTest.php b/apps/dav/tests/unit/Files/MultipartRequestParserTest.php new file mode 100644 index 00000000000..ec9e2d0a383 --- /dev/null +++ b/apps/dav/tests/unit/Files/MultipartRequestParserTest.php @@ -0,0 +1,281 @@ +<?php +/** + * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me> + * + * @author Louis Chemineau <louis@chmn.me> + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OCA\DAV\Tests\unit\DAV; + +use Test\TestCase; +use \OCA\DAV\BulkUpload\MultipartRequestParser; + +class MultipartRequestParserTest extends TestCase { + private function getValidBodyObject() { + return [ + [ + "headers" => [ + "Content-Length" => 7, + "X-File-MD5" => "4f2377b4d911f7ec46325fe603c3af03", + "X-File-Path" => "/coucou.txt" + ], + "content" => "Coucou\n" + ] + ]; + } + + private function getMultipartParser(array $parts, array $headers = [], string $boundary = "boundary_azertyuiop"): MultipartRequestParser { + $request = $this->getMockBuilder('Sabre\HTTP\RequestInterface') + ->disableOriginalConstructor() + ->getMock(); + + $headers = array_merge(['Content-Type' => 'multipart/related; boundary='.$boundary], $headers); + $request->expects($this->any()) + ->method('getHeader') + ->willReturnCallback(function (string $key) use (&$headers) { + return $headers[$key]; + }); + + $body = ""; + foreach ($parts as $part) { + $body .= '--'.$boundary."\r\n"; + + foreach ($part['headers'] as $headerKey => $headerPart) { + $body .= $headerKey.": ".$headerPart."\r\n"; + } + + $body .= "\r\n"; + $body .= $part['content']."\r\n"; + } + + $body .= '--'.$boundary."--"; + + $stream = fopen('php://temp','r+'); + fwrite($stream, $body); + rewind($stream); + + $request->expects($this->any()) + ->method('getBody') + ->willReturn($stream); + + return new MultipartRequestParser($request); + } + + + /** + * Test validation of the request's body type + */ + public function testBodyTypeValidation() { + $bodyStream = "I am not a stream, but pretend to be"; + $request = $this->getMockBuilder('Sabre\HTTP\RequestInterface') + ->disableOriginalConstructor() + ->getMock(); + $request->expects($this->any()) + ->method('getBody') + ->willReturn($bodyStream); + + $this->expectExceptionMessage('Body should be of type resource'); + new MultipartRequestParser($request); + } + + /** + * Test with valid request. + * - valid boundary + * - valid md5 hash + * - valid content-length + * - valid file content + * - valid file path + */ + public function testValidRequest() { + $multipartParser = $this->getMultipartParser( + $this->getValidBodyObject() + ); + + [$headers, $content] = $multipartParser->parseNextPart(); + + $this->assertSame((int)$headers["content-length"], 7, "Content-Length header should be the same as provided."); + $this->assertSame($headers["x-file-md5"], "4f2377b4d911f7ec46325fe603c3af03", "X-File-MD5 header should be the same as provided."); + $this->assertSame($headers["x-file-path"], "/coucou.txt", "X-File-Path header should be the same as provided."); + + $this->assertSame($content, "Coucou\n", "Content should be the same"); + } + + /** + * Test with invalid md5 hash. + */ + public function testInvalidMd5Hash() { + $bodyObject = $this->getValidBodyObject(); + $bodyObject["0"]["headers"]["X-File-MD5"] = "f2377b4d911f7ec46325fe603c3af03"; + $multipartParser = $this->getMultipartParser( + $bodyObject + ); + + $this->expectExceptionMessage('Computed md5 hash is incorrect.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with a null md5 hash. + */ + public function testNullMd5Hash() { + $bodyObject = $this->getValidBodyObject(); + unset($bodyObject["0"]["headers"]["X-File-MD5"]); + $multipartParser = $this->getMultipartParser( + $bodyObject + ); + + $this->expectExceptionMessage('The X-File-MD5 header must not be null.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with a null Content-Length. + */ + public function testNullContentLength() { + $bodyObject = $this->getValidBodyObject(); + unset($bodyObject["0"]["headers"]["Content-Length"]); + $multipartParser = $this->getMultipartParser( + $bodyObject + ); + + $this->expectExceptionMessage('The Content-Length header must not be null.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with a lower Content-Length. + */ + public function testLowerContentLength() { + $bodyObject = $this->getValidBodyObject(); + $bodyObject["0"]["headers"]["Content-Length"] = 6; + $multipartParser = $this->getMultipartParser( + $bodyObject + ); + + $this->expectExceptionMessage('Computed md5 hash is incorrect.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with a higher Content-Length. + */ + public function testHigherContentLength() { + $bodyObject = $this->getValidBodyObject(); + $bodyObject["0"]["headers"]["Content-Length"] = 8; + $multipartParser = $this->getMultipartParser( + $bodyObject + ); + + $this->expectExceptionMessage('Computed md5 hash is incorrect.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with wrong boundary in body. + */ + public function testWrongBoundary() { + $bodyObject = $this->getValidBodyObject(); + $multipartParser = $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/related; boundary=boundary_poiuytreza'] + ); + + $this->expectExceptionMessage('Boundary not found where it should be.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with no boundary in request headers. + */ + public function testNoBoundaryInHeader() { + $bodyObject = $this->getValidBodyObject(); + $this->expectExceptionMessage('Error while parsing boundary in Content-Type header.'); + $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/related'] + ); + } + + /** + * Test with no boundary in the request's headers. + */ + public function testNoBoundaryInBody() { + $bodyObject = $this->getValidBodyObject(); + $multipartParser = $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/related; boundary=boundary_azertyuiop'], + '' + ); + + $this->expectExceptionMessage('Boundary not found where it should be.'); + $multipartParser->parseNextPart(); + } + + /** + * Test with a boundary with quotes in the request's headers. + */ + public function testBoundaryWithQuotes() { + $bodyObject = $this->getValidBodyObject(); + $multipartParser = $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/related; boundary="boundary_azertyuiop"'], + ); + + $multipartParser->parseNextPart(); + + // Dummy assertion, we just want to test that the parsing works. + $this->assertTrue(true); + } + + /** + * Test with a wrong Content-Type in the request's headers. + */ + public function testWrongContentType() { + $bodyObject = $this->getValidBodyObject(); + $this->expectExceptionMessage('Content-Type must be multipart/related'); + $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/form-data; boundary="boundary_azertyuiop"'], + ); + } + + /** + * Test with a wrong key after the content type in the request's headers. + */ + public function testWrongKeyInContentType() { + $bodyObject = $this->getValidBodyObject(); + $this->expectExceptionMessage('Boundary is invalid'); + $this->getMultipartParser( + $bodyObject, + ['Content-Type' => 'multipart/related; wrongkey="boundary_azertyuiop"'], + ); + } + + /** + * Test with a null Content-Type in the request's headers. + */ + public function testNullContentType() { + $bodyObject = $this->getValidBodyObject(); + $this->expectExceptionMessage('Content-Type can not be null'); + $this->getMultipartParser( + $bodyObject, + ['Content-Type' => null], + + ); + } +} diff --git a/build/integration/features/bootstrap/WebDav.php b/build/integration/features/bootstrap/WebDav.php index 31ca68ba92b..9f5e79a3ac6 100644 --- a/build/integration/features/bootstrap/WebDav.php +++ b/build/integration/features/bootstrap/WebDav.php @@ -538,6 +538,57 @@ trait WebDav { } /** + * @Given user :user uploads bulked files :name1 with :content1 and :name2 with :content2 and :name3 with :content3 + * @param string $user + * @param string $name1 + * @param string $content1 + * @param string $name2 + * @param string $content2 + * @param string $name3 + * @param string $content3 + */ + public function userUploadsChunkedFiles($user, $name1, $content1, $name2, $content2, $name3, $content3) { + $boundary = "boundary_azertyuiop"; + + $body = ""; + $body .= '--'.$boundary."\r\n"; + $body .= "X-File-Path: ".$name1."\r\n"; + $body .= "X-File-MD5: f6a6263167c92de8644ac998b3c4e4d1\r\n"; + $body .= "Content-Length: ".strlen($content1)."\r\n"; + $body .= "\r\n"; + $body .= $content1."\r\n"; + $body .= '--'.$boundary."\r\n"; + $body .= "X-File-Path: ".$name2."\r\n"; + $body .= "X-File-MD5: 87c7d4068be07d390a1fffd21bf1e944\r\n"; + $body .= "Content-Length: ".strlen($content2)."\r\n"; + $body .= "\r\n"; + $body .= $content2."\r\n"; + $body .= '--'.$boundary."\r\n"; + $body .= "X-File-Path: ".$name3."\r\n"; + $body .= "X-File-MD5: e86a1cf0678099986a901c79086f5617\r\n"; + $body .= "Content-Length: ".strlen($content3)."\r\n"; + $body .= "\r\n"; + $body .= $content3."\r\n"; + $body .= '--'.$boundary."--\r\n"; + + $stream = fopen('php://temp','r+'); + fwrite($stream, $body); + rewind($stream); + + $client = new GClient(); + $options = [ + 'auth' => [$user, $this->regularUser], + 'headers' => [ + 'Content-Type' => 'multipart/related; boundary='.$boundary, + 'Content-Length' => (string)strlen($body), + ], + 'body' => $body + ]; + + return $client->request("POST", substr($this->baseUrl, 0, -4) . "remote.php/dav/bulk", $options); + } + + /** * @Given user :user creates a new chunking upload with id :id */ public function userCreatesANewChunkingUploadWithId($user, $id) { diff --git a/build/integration/features/webdav-related.feature b/build/integration/features/webdav-related.feature index 66652e6fa26..c98ecc56ec7 100644 --- a/build/integration/features/webdav-related.feature +++ b/build/integration/features/webdav-related.feature @@ -608,3 +608,14 @@ Feature: webdav-related And user "user0" uploads new chunk file "3" with "CCCCC" to id "chunking-42" When user "user0" moves new chunk file with id "chunking-42" to "/myChunkedFile.txt" with size 15 Then the HTTP status code should be "201" + + Scenario: Upload bulked files + Given user "user0" exists + And user "user0" uploads bulked files "A.txt" with "AAAAA" and "B.txt" with "BBBBB" and "C.txt" with "CCCCC" + When As an "user0" + Then Downloading file "/A.txt" + And Downloaded content should be "AAAAA" + And Downloading file "/B.txt" + And Downloaded content should be "BBBBB" + And Downloading file "/C.txt" + And Downloaded content should be "CCCCC" |