aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--apps/dav/composer/composer/autoload_classmap.php2
-rw-r--r--apps/dav/composer/composer/autoload_static.php2
-rw-r--r--apps/dav/lib/BulkUpload/BulkUploadPlugin.php100
-rw-r--r--apps/dav/lib/BulkUpload/MultipartRequestParser.php239
-rw-r--r--apps/dav/lib/Capabilities.php2
-rw-r--r--apps/dav/lib/Server.php6
-rwxr-xr-xapps/dav/tests/benchmarks/benchmark.sh52
-rwxr-xr-xapps/dav/tests/benchmarks/bulk_upload.sh78
-rwxr-xr-xapps/dav/tests/benchmarks/single_upload.sh61
-rw-r--r--apps/dav/tests/unit/CapabilitiesTest.php1
-rw-r--r--apps/dav/tests/unit/Files/MultipartRequestParserTest.php281
-rw-r--r--build/integration/features/bootstrap/WebDav.php51
-rw-r--r--build/integration/features/webdav-related.feature11
13 files changed, 886 insertions, 0 deletions
diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php
index 62c41a0828d..1a536c98272 100644
--- a/apps/dav/composer/composer/autoload_classmap.php
+++ b/apps/dav/composer/composer/autoload_classmap.php
@@ -22,6 +22,8 @@ return array(
'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => $baseDir . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php',
+ 'OCA\\DAV\\BulkUpload\\BulkUploadPlugin' => $baseDir . '/../lib/BulkUpload/BulkUploadPlugin.php',
+ 'OCA\\DAV\\BulkUpload\\MultipartRequestParser' => $baseDir . '/../lib/BulkUpload/MultipartRequestParser.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php',
diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php
index 5d5f57eb51b..b65d4477800 100644
--- a/apps/dav/composer/composer/autoload_static.php
+++ b/apps/dav/composer/composer/autoload_static.php
@@ -37,6 +37,8 @@ class ComposerStaticInitDAV
'OCA\\DAV\\BackgroundJob\\RegisterRegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/BackgroundJob/RegisterRegenerateBirthdayCalendars.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php',
+ 'OCA\\DAV\\BulkUpload\\BulkUploadPlugin' => __DIR__ . '/..' . '/../lib/BulkUpload/BulkUploadPlugin.php',
+ 'OCA\\DAV\\BulkUpload\\MultipartRequestParser' => __DIR__ . '/..' . '/../lib/BulkUpload/MultipartRequestParser.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php',
diff --git a/apps/dav/lib/BulkUpload/BulkUploadPlugin.php b/apps/dav/lib/BulkUpload/BulkUploadPlugin.php
new file mode 100644
index 00000000000..0766ae37a17
--- /dev/null
+++ b/apps/dav/lib/BulkUpload/BulkUploadPlugin.php
@@ -0,0 +1,100 @@
+<?php
+/**
+ * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me>
+ *
+ * @author Louis Chemineau <louis@chmn.me>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\DAV\BulkUpload;
+
+use Psr\Log\LoggerInterface;
+use Sabre\DAV\Server;
+use Sabre\DAV\ServerPlugin;
+use Sabre\HTTP\RequestInterface;
+use Sabre\HTTP\ResponseInterface;
+use OCP\Files\Folder;
+use OCP\AppFramework\Http;
+
+class BulkUploadPlugin extends ServerPlugin {
+
+ /** @var Folder */
+ private $userFolder;
+
+ /** @var LoggerInterface */
+ private $logger;
+
+ public function __construct(Folder $userFolder, LoggerInterface $logger) {
+ $this->userFolder = $userFolder;
+ $this->logger = $logger;
+ }
+
+ /**
+ * Register listener on POST requests with the httpPost method.
+ */
+ public function initialize(Server $server): void {
+ $server->on('method:POST', [$this, 'httpPost'], 10);
+ }
+
+ /**
+ * Handle POST requests on /dav/bulk
+ * - parsing is done with a MultipartContentsParser object
+ * - writing is done with the userFolder service
+ *
+ * Will respond with an object containing an ETag for every written files.
+ */
+ public function httpPost(RequestInterface $request, ResponseInterface $response): bool {
+ // Limit bulk upload to the /dav/bulk endpoint
+ if ($request->getPath() !== "bulk") {
+ return true;
+ }
+
+ $multiPartParser = new MultipartRequestParser($request);
+ $writtenFiles = [];
+
+ while (!$multiPartParser->isAtLastBoundary()) {
+ try {
+ [$headers, $content] = $multiPartParser->parseNextPart();
+ } catch (\Exception $e) {
+ // Return early if an error occurs during parsing.
+ $this->logger->error($e->getMessage());
+ $response->setStatus(Http::STATUS_BAD_REQUEST);
+ $response->setBody(json_encode($writtenFiles));
+ return false;
+ }
+
+ try {
+ $node = $this->userFolder->newFile($headers['x-file-path'], $content);
+ $writtenFiles[$headers['x-file-path']] = [
+ "error" => false,
+ "etag" => $node->getETag(),
+ ];
+ } catch (\Exception $e) {
+ $this->logger->error($e->getMessage(), ['path' => $headers['x-file-path']]);
+ $writtenFiles[$headers['x-file-path']] = [
+ "error" => true,
+ "message" => $e->getMessage(),
+ ];
+ }
+ }
+
+ $response->setStatus(Http::STATUS_OK);
+ $response->setBody(json_encode($writtenFiles));
+
+ return false;
+ }
+}
diff --git a/apps/dav/lib/BulkUpload/MultipartRequestParser.php b/apps/dav/lib/BulkUpload/MultipartRequestParser.php
new file mode 100644
index 00000000000..7554447fc93
--- /dev/null
+++ b/apps/dav/lib/BulkUpload/MultipartRequestParser.php
@@ -0,0 +1,239 @@
+<?php
+/**
+ * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me>
+ *
+ * @author Louis Chemineau <louis@chmn.me>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\DAV\BulkUpload;
+
+use Sabre\HTTP\RequestInterface;
+use Sabre\DAV\Exception;
+use Sabre\DAV\Exception\BadRequest;
+use Sabre\DAV\Exception\LengthRequired;
+use OCP\AppFramework\Http;
+
+class MultipartRequestParser {
+
+ /** @var resource */
+ private $stream;
+
+ /** @var string */
+ private $boundary = "";
+
+ /** @var string */
+ private $lastBoundary = "";
+
+ /**
+ * @throws BadRequest
+ */
+ public function __construct(RequestInterface $request) {
+ $stream = $request->getBody();
+ $contentType = $request->getHeader('Content-Type');
+
+ if (!is_resource($stream)) {
+ throw new BadRequest('Body should be of type resource');
+ }
+
+ if ($contentType === null) {
+ throw new BadRequest("Content-Type can not be null");
+ }
+
+ $this->stream = $stream;
+
+ $boundary = $this->parseBoundaryFromHeaders($contentType);
+ $this->boundary = '--'.$boundary."\r\n";
+ $this->lastBoundary = '--'.$boundary."--\r\n";
+ }
+
+ /**
+ * Parse the boundary from the Content-Type header.
+ * Example: Content-Type: "multipart/related; boundary=boundary_bf38b9b4b10a303a28ed075624db3978"
+ *
+ * @throws BadRequest
+ */
+ private function parseBoundaryFromHeaders(string $contentType): string {
+ try {
+ [$mimeType, $boundary] = explode(';', $contentType);
+ [$boundaryKey, $boundaryValue] = explode('=', $boundary);
+ } catch (\Exception $e) {
+ throw new BadRequest("Error while parsing boundary in Content-Type header.", Http::STATUS_BAD_REQUEST, $e);
+ }
+
+ $boundaryValue = trim($boundaryValue);
+
+ // Remove potential quotes around boundary value.
+ if (substr($boundaryValue, 0, 1) == '"' && substr($boundaryValue, -1) == '"') {
+ $boundaryValue = substr($boundaryValue, 1, -1);
+ }
+
+ if (trim($mimeType) !== 'multipart/related') {
+ throw new BadRequest('Content-Type must be multipart/related');
+ }
+
+ if (trim($boundaryKey) !== 'boundary') {
+ throw new BadRequest('Boundary is invalid');
+ }
+
+ return $boundaryValue;
+ }
+
+ /**
+ * Check whether the stream's cursor is sitting right before the provided string.
+ *
+ * @throws Exception
+ */
+ private function isAt(string $expectedContent): bool {
+ $expectedContentLength = strlen($expectedContent);
+
+ $content = fread($this->stream, $expectedContentLength);
+ if ($content === false) {
+ throw new Exception('An error occurred while checking content');
+ }
+
+ $seekBackResult = fseek($this->stream, -$expectedContentLength, SEEK_CUR);
+ if ($seekBackResult === -1) {
+ throw new Exception("Unknown error while seeking content", Http::STATUS_INTERNAL_SERVER_ERROR);
+ }
+
+ return $expectedContent === $content;
+ }
+
+
+ /**
+ * Check whether the stream's cursor is sitting right before the boundary.
+ */
+ private function isAtBoundary(): bool {
+ return $this->isAt($this->boundary);
+ }
+
+ /**
+ * Check whether the stream's cursor is sitting right before the last boundary.
+ */
+ public function isAtLastBoundary(): bool {
+ return $this->isAt($this->lastBoundary);
+ }
+
+ /**
+ * Parse and return the next part of the multipart headers.
+ *
+ * Example:
+ * --boundary_azertyuiop
+ * Header1: value
+ * Header2: value
+ *
+ * Content of
+ * the part
+ *
+ */
+ public function parseNextPart(): array {
+ $this->readBoundary();
+
+ $headers = $this->readPartHeaders();
+
+ $content = $this->readPartContent($headers["content-length"], $headers["x-file-md5"]);
+
+ return [$headers, $content];
+ }
+
+ /**
+ * Read the boundary and check its content.
+ *
+ * @throws BadRequest
+ */
+ private function readBoundary(): string {
+ if (!$this->isAtBoundary()) {
+ throw new BadRequest("Boundary not found where it should be.");
+ }
+
+ return fread($this->stream, strlen($this->boundary));
+ }
+
+ /**
+ * Return the headers of a part of the multipart body.
+ *
+ * @throws Exception
+ * @throws BadRequest
+ * @throws LengthRequired
+ */
+ private function readPartHeaders(): array {
+ $headers = [];
+
+ while (($line = fgets($this->stream)) !== "\r\n") {
+ if ($line === false) {
+ throw new Exception('An error occurred while reading headers of a part');
+ }
+
+ try {
+ [$key, $value] = explode(':', $line, 2);
+ $headers[strtolower(trim($key))] = trim($value);
+ } catch (\Exception $e) {
+ throw new BadRequest('An error occurred while parsing headers of a part', Http::STATUS_BAD_REQUEST, $e);
+ }
+ }
+
+ if (!isset($headers["content-length"])) {
+ throw new LengthRequired("The Content-Length header must not be null.");
+ }
+
+ if (!isset($headers["x-file-md5"])) {
+ throw new BadRequest("The X-File-MD5 header must not be null.");
+ }
+
+ return $headers;
+ }
+
+ /**
+ * Return the content of a part of the multipart body.
+ *
+ * @throws Exception
+ * @throws BadRequest
+ */
+ private function readPartContent(int $length, string $md5): string {
+ $computedMd5 = $this->computeMd5Hash($length);
+
+ if ($md5 !== $computedMd5) {
+ throw new BadRequest("Computed md5 hash is incorrect.");
+ }
+
+ $content = stream_get_line($this->stream, $length);
+
+ if ($content === false) {
+ throw new Exception("Fail to read part's content.");
+ }
+
+ if (feof($this->stream)) {
+ throw new Exception("Unexpected EOF while reading stream.");
+ }
+
+ // Read '\r\n'.
+ stream_get_contents($this->stream, 2);
+
+ return $content;
+ }
+
+ /**
+ * Compute the MD5 hash of the next x bytes.
+ */
+ private function computeMd5Hash(int $length): string {
+ $context = hash_init('md5');
+ hash_update_stream($context, $this->stream, $length);
+ fseek($this->stream, -$length, SEEK_CUR);
+ return hash_final($context);
+ }
+}
diff --git a/apps/dav/lib/Capabilities.php b/apps/dav/lib/Capabilities.php
index 5d4e3c05077..ce60bccfd0b 100644
--- a/apps/dav/lib/Capabilities.php
+++ b/apps/dav/lib/Capabilities.php
@@ -3,6 +3,7 @@
* @copyright Copyright (c) 2016, ownCloud GmbH
*
* @author Thomas Müller <thomas.mueller@tmit.eu>
+ * @author Louis Chemineau <louis@chmn.me>
*
* @license AGPL-3.0
*
@@ -28,6 +29,7 @@ class Capabilities implements ICapability {
return [
'dav' => [
'chunking' => '1.0',
+ 'bulkupload' => '1.0',
]
];
}
diff --git a/apps/dav/lib/Server.php b/apps/dav/lib/Server.php
index e9634f670d3..055c37f8472 100644
--- a/apps/dav/lib/Server.php
+++ b/apps/dav/lib/Server.php
@@ -34,6 +34,7 @@
*/
namespace OCA\DAV;
+use Psr\Log\LoggerInterface;
use OCA\DAV\AppInfo\PluginManager;
use OCA\DAV\CalDAV\BirthdayService;
use OCA\DAV\CardDAV\HasPhotoPlugin;
@@ -62,6 +63,7 @@ use OCA\DAV\DAV\PublicAuth;
use OCA\DAV\Events\SabrePluginAuthInitEvent;
use OCA\DAV\Files\BrowserErrorPagePlugin;
use OCA\DAV\Files\LazySearchBackend;
+use OCA\DAV\BulkUpload\BulkUploadPlugin;
use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
use OCA\DAV\SystemTag\SystemTagPlugin;
use OCA\DAV\Upload\ChunkingPlugin;
@@ -294,6 +296,10 @@ class Server {
\OC::$server->getShareManager(),
$view
));
+ $logger = \OC::$server->get(LoggerInterface::class);
+ $this->server->addPlugin(
+ new BulkUploadPlugin($userFolder, $logger)
+ );
}
$this->server->addPlugin(new \OCA\DAV\CalDAV\BirthdayCalendar\EnablePlugin(
\OC::$server->getConfig(),
diff --git a/apps/dav/tests/benchmarks/benchmark.sh b/apps/dav/tests/benchmarks/benchmark.sh
new file mode 100755
index 00000000000..27d7c4ecbc7
--- /dev/null
+++ b/apps/dav/tests/benchmarks/benchmark.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -eu
+
+# benchmark.sh
+
+export KB=1000
+export MB=$((KB*1000))
+
+MAX_UPLOAD_SIZE=$((512*KB))
+
+export CONCURRENCY=5
+export BANDWIDTH=$((100*MB/CONCURRENCY))
+
+FILE_SIZES=($((1*KB)) $((10*KB)) $((100*KB)))
+
+echo "Concurrency: $CONCURRENCY"
+echo "Bandwidth: $BANDWIDTH"
+
+md_output="# Bulk upload benchmark\n"
+md_output+="\n"
+md_output+="- Concurrency: $CONCURRENCY\n"
+md_output+="- Bandwidth: ${BANDWIDTH}B\n"
+md_output+="\n"
+md_output+="| Nb | Size (B) | Bundle (sec) | Single (sec) |\n"
+md_output+="|---|---|---|---|\n"
+
+requests_count='1 2 3 4 5'
+
+for size in "${FILE_SIZES[@]}"
+do
+ nb=$((MAX_UPLOAD_SIZE/size))
+
+ echo "- Upload of $nb tiny file of ${size}B"
+ echo " - Bundled"
+ start=$(date +%s)
+ echo "$requests_count" | xargs -d ' ' -P $CONCURRENCY -I{} ./bulk_upload.sh "$nb" "$size"
+ end=$(date +%s)
+ bulk_exec_time=$((end-start))
+ echo "${bulk_exec_time}s"
+
+ echo " - Single"
+ start=$(date +%s)
+ echo "$requests_count" | xargs -d ' ' -P $CONCURRENCY -I{} ./single_upload.sh "$nb" "$size"
+ end=$(date +%s)
+ single_exec_time=$((end-start))
+ echo "${single_exec_time}s"
+
+ md_output+="| $nb | $size | $bulk_exec_time | $single_exec_time |\n"
+done
+
+echo -en "$md_output" \ No newline at end of file
diff --git a/apps/dav/tests/benchmarks/bulk_upload.sh b/apps/dav/tests/benchmarks/bulk_upload.sh
new file mode 100755
index 00000000000..862ddfe461f
--- /dev/null
+++ b/apps/dav/tests/benchmarks/bulk_upload.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eu
+
+# bulk_upload.sh <nb-of-files> <size-of-files>
+
+KB=${KB:-100}
+MB=${MB:-$((KB*1000))}
+
+NB=$1
+SIZE=$2
+
+CONCURRENCY=${CONCURRENCY:-1}
+BANDWIDTH=${BANDWIDTH:-$((100*MB/CONCURRENCY))}
+
+USER="admin"
+PASS="password"
+SERVER="nextcloud.test"
+UPLOAD_PATH="/tmp/bulk_upload_request_$(openssl rand --hex 8).txt"
+BOUNDARY="boundary_$(openssl rand --hex 8)"
+LOCAL_FOLDER="/tmp/bulk_upload/${BOUNDARY}_${NB}_${SIZE}"
+REMOTE_FOLDER="/bulk_upload/${BOUNDARY}_${NB}_${SIZE}"
+
+mkdir --parent "$LOCAL_FOLDER"
+
+for ((i=1; i<="$NB"; i++))
+do
+ file_name=$(openssl rand --hex 8)
+ file_local_path="$LOCAL_FOLDER/$file_name.txt"
+ file_remote_path="$REMOTE_FOLDER/$file_name.txt"
+ head -c "$SIZE" /dev/urandom > "$file_local_path"
+ file_mtime=$(stat -c %Y "$file_local_path")
+ file_hash=$(md5sum "$file_local_path" | awk '{ print $1 }')
+ file_size=$(du -sb "$file_local_path" | awk '{ print $1 }')
+
+ {
+ echo -en "--$BOUNDARY\r\n"
+ # echo -en "Content-ID: $file_name\r\n"
+ echo -en "X-File-Path: $file_remote_path\r\n"
+ echo -en "X-File-Mtime: $file_mtime\r\n"
+ # echo -en "X-File-Id: $file_id\r\n"
+ echo -en "X-File-Md5: $file_hash\r\n"
+ echo -en "Content-Length: $file_size\r\n"
+ echo -en "\r\n" >> "$UPLOAD_PATH"
+
+ cat "$file_local_path"
+ echo -en "\r\n" >> "$UPLOAD_PATH"
+ } >> "$UPLOAD_PATH"
+done
+
+echo -en "--$BOUNDARY--\r\n" >> "$UPLOAD_PATH"
+
+echo "Creating folder /bulk_upload"
+curl \
+ -X MKCOL \
+ -k \
+ "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/bulk_upload" > /dev/null
+
+echo "Creating folder $REMOTE_FOLDER"
+curl \
+ -X MKCOL \
+ -k \
+ "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/$REMOTE_FOLDER"
+
+echo "Uploading $NB files with total size: $(du -sh "$UPLOAD_PATH" | cut -d ' ' -f1)"
+echo "Local file is: $UPLOAD_PATH"
+curl \
+ -X POST \
+ -k \
+ --progress-bar \
+ --limit-rate "${BANDWIDTH}k" \
+ --cookie "XDEBUG_PROFILE=true;path=/;" \
+ -H "Content-Type: multipart/related; boundary=$BOUNDARY" \
+ --data-binary "@$UPLOAD_PATH" \
+ "https://$USER:$PASS@$SERVER/remote.php/dav/bulk"
+
+rm -rf "${LOCAL_FOLDER:?}"
+rm "$UPLOAD_PATH"
diff --git a/apps/dav/tests/benchmarks/single_upload.sh b/apps/dav/tests/benchmarks/single_upload.sh
new file mode 100755
index 00000000000..ec57e66668d
--- /dev/null
+++ b/apps/dav/tests/benchmarks/single_upload.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -eu
+
+# single_upload.sh <nb-of-files> <size-of-files>
+
+export KB=${KB:-100}
+export MB=${MB:-$((KB*1000))}
+
+export NB=$1
+export SIZE=$2
+
+export CONCURRENCY=${CONCURRENCY:-1}
+export BANDWIDTH=${BANDWIDTH:-$((100*MB/CONCURRENCY))}
+
+export USER="admin"
+export PASS="password"
+export SERVER="nextcloud.test"
+export UPLOAD_ID="single_$(openssl rand --hex 8)"
+export LOCAL_FOLDER="/tmp/single_upload/${UPLOAD_ID}_${NB}_${SIZE}"
+export REMOTE_FOLDER="/single_upload/${UPLOAD_ID}_${NB}_${SIZE}"
+
+mkdir --parent "$LOCAL_FOLDER"
+
+curl \
+ -X MKCOL \
+ -k \
+ "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/bulk_upload" > /dev/null
+
+curl \
+ -X MKCOL \
+ -k \
+ --cookie "XDEBUG_SESSION=true;path=/;" \
+ "https://$USER:$PASS@$SERVER/remote.php/dav/files/$USER/$REMOTE_FOLDER"
+
+upload_file() {
+ file_name=$(openssl rand --hex 8)
+ file_local_path="$LOCAL_FOLDER/$file_name.txt"
+ file_remote_path="$REMOTE_FOLDER/$file_name.txt"
+ head -c "$SIZE" /dev/urandom > "$file_local_path"
+
+ curl \
+ -X PUT \
+ -k \
+ --limit-rate "${BANDWIDTH}k" \
+ --data-binary @"$file_local_path" "https://$USER:$PASS@$SERVER/remote.php/webdav/$file_remote_path"
+}
+export -f upload_file
+
+file_list=''
+for ((i=1; i<"$NB"; i++))
+do
+ file_list+="$i "
+done
+file_list+=$NB
+
+echo "$file_list" | xargs -d ' ' -P "$((CONCURRENCY/5))" -I{} bash -c "upload_file {}"
+
+printf "\n"
+
+rm -rf "${LOCAL_FOLDER:?}"/* \ No newline at end of file
diff --git a/apps/dav/tests/unit/CapabilitiesTest.php b/apps/dav/tests/unit/CapabilitiesTest.php
index 719b62115d9..399467f6ed8 100644
--- a/apps/dav/tests/unit/CapabilitiesTest.php
+++ b/apps/dav/tests/unit/CapabilitiesTest.php
@@ -35,6 +35,7 @@ class CapabilitiesTest extends TestCase {
$expected = [
'dav' => [
'chunking' => '1.0',
+ 'bulkupload' => '1.0',
],
];
$this->assertSame($expected, $capabilities->getCapabilities());
diff --git a/apps/dav/tests/unit/Files/MultipartRequestParserTest.php b/apps/dav/tests/unit/Files/MultipartRequestParserTest.php
new file mode 100644
index 00000000000..ec9e2d0a383
--- /dev/null
+++ b/apps/dav/tests/unit/Files/MultipartRequestParserTest.php
@@ -0,0 +1,281 @@
+<?php
+/**
+ * @copyright Copyright (c) 2021, Louis Chemineau <louis@chmn.me>
+ *
+ * @author Louis Chemineau <louis@chmn.me>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\DAV\Tests\unit\DAV;
+
+use Test\TestCase;
+use \OCA\DAV\BulkUpload\MultipartRequestParser;
+
+class MultipartRequestParserTest extends TestCase {
+ private function getValidBodyObject() {
+ return [
+ [
+ "headers" => [
+ "Content-Length" => 7,
+ "X-File-MD5" => "4f2377b4d911f7ec46325fe603c3af03",
+ "X-File-Path" => "/coucou.txt"
+ ],
+ "content" => "Coucou\n"
+ ]
+ ];
+ }
+
+ private function getMultipartParser(array $parts, array $headers = [], string $boundary = "boundary_azertyuiop"): MultipartRequestParser {
+ $request = $this->getMockBuilder('Sabre\HTTP\RequestInterface')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $headers = array_merge(['Content-Type' => 'multipart/related; boundary='.$boundary], $headers);
+ $request->expects($this->any())
+ ->method('getHeader')
+ ->willReturnCallback(function (string $key) use (&$headers) {
+ return $headers[$key];
+ });
+
+ $body = "";
+ foreach ($parts as $part) {
+ $body .= '--'.$boundary."\r\n";
+
+ foreach ($part['headers'] as $headerKey => $headerPart) {
+ $body .= $headerKey.": ".$headerPart."\r\n";
+ }
+
+ $body .= "\r\n";
+ $body .= $part['content']."\r\n";
+ }
+
+ $body .= '--'.$boundary."--";
+
+ $stream = fopen('php://temp','r+');
+ fwrite($stream, $body);
+ rewind($stream);
+
+ $request->expects($this->any())
+ ->method('getBody')
+ ->willReturn($stream);
+
+ return new MultipartRequestParser($request);
+ }
+
+
+ /**
+ * Test validation of the request's body type
+ */
+ public function testBodyTypeValidation() {
+ $bodyStream = "I am not a stream, but pretend to be";
+ $request = $this->getMockBuilder('Sabre\HTTP\RequestInterface')
+ ->disableOriginalConstructor()
+ ->getMock();
+ $request->expects($this->any())
+ ->method('getBody')
+ ->willReturn($bodyStream);
+
+ $this->expectExceptionMessage('Body should be of type resource');
+ new MultipartRequestParser($request);
+ }
+
+ /**
+ * Test with valid request.
+ * - valid boundary
+ * - valid md5 hash
+ * - valid content-length
+ * - valid file content
+ * - valid file path
+ */
+ public function testValidRequest() {
+ $multipartParser = $this->getMultipartParser(
+ $this->getValidBodyObject()
+ );
+
+ [$headers, $content] = $multipartParser->parseNextPart();
+
+ $this->assertSame((int)$headers["content-length"], 7, "Content-Length header should be the same as provided.");
+ $this->assertSame($headers["x-file-md5"], "4f2377b4d911f7ec46325fe603c3af03", "X-File-MD5 header should be the same as provided.");
+ $this->assertSame($headers["x-file-path"], "/coucou.txt", "X-File-Path header should be the same as provided.");
+
+ $this->assertSame($content, "Coucou\n", "Content should be the same");
+ }
+
+ /**
+ * Test with invalid md5 hash.
+ */
+ public function testInvalidMd5Hash() {
+ $bodyObject = $this->getValidBodyObject();
+ $bodyObject["0"]["headers"]["X-File-MD5"] = "f2377b4d911f7ec46325fe603c3af03";
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject
+ );
+
+ $this->expectExceptionMessage('Computed md5 hash is incorrect.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with a null md5 hash.
+ */
+ public function testNullMd5Hash() {
+ $bodyObject = $this->getValidBodyObject();
+ unset($bodyObject["0"]["headers"]["X-File-MD5"]);
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject
+ );
+
+ $this->expectExceptionMessage('The X-File-MD5 header must not be null.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with a null Content-Length.
+ */
+ public function testNullContentLength() {
+ $bodyObject = $this->getValidBodyObject();
+ unset($bodyObject["0"]["headers"]["Content-Length"]);
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject
+ );
+
+ $this->expectExceptionMessage('The Content-Length header must not be null.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with a lower Content-Length.
+ */
+ public function testLowerContentLength() {
+ $bodyObject = $this->getValidBodyObject();
+ $bodyObject["0"]["headers"]["Content-Length"] = 6;
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject
+ );
+
+ $this->expectExceptionMessage('Computed md5 hash is incorrect.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with a higher Content-Length.
+ */
+ public function testHigherContentLength() {
+ $bodyObject = $this->getValidBodyObject();
+ $bodyObject["0"]["headers"]["Content-Length"] = 8;
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject
+ );
+
+ $this->expectExceptionMessage('Computed md5 hash is incorrect.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with wrong boundary in body.
+ */
+ public function testWrongBoundary() {
+ $bodyObject = $this->getValidBodyObject();
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/related; boundary=boundary_poiuytreza']
+ );
+
+ $this->expectExceptionMessage('Boundary not found where it should be.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with no boundary in request headers.
+ */
+ public function testNoBoundaryInHeader() {
+ $bodyObject = $this->getValidBodyObject();
+ $this->expectExceptionMessage('Error while parsing boundary in Content-Type header.');
+ $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/related']
+ );
+ }
+
+ /**
+ * Test with no boundary in the request's headers.
+ */
+ public function testNoBoundaryInBody() {
+ $bodyObject = $this->getValidBodyObject();
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/related; boundary=boundary_azertyuiop'],
+ ''
+ );
+
+ $this->expectExceptionMessage('Boundary not found where it should be.');
+ $multipartParser->parseNextPart();
+ }
+
+ /**
+ * Test with a boundary with quotes in the request's headers.
+ */
+ public function testBoundaryWithQuotes() {
+ $bodyObject = $this->getValidBodyObject();
+ $multipartParser = $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/related; boundary="boundary_azertyuiop"'],
+ );
+
+ $multipartParser->parseNextPart();
+
+ // Dummy assertion, we just want to test that the parsing works.
+ $this->assertTrue(true);
+ }
+
+ /**
+ * Test with a wrong Content-Type in the request's headers.
+ */
+ public function testWrongContentType() {
+ $bodyObject = $this->getValidBodyObject();
+ $this->expectExceptionMessage('Content-Type must be multipart/related');
+ $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/form-data; boundary="boundary_azertyuiop"'],
+ );
+ }
+
+ /**
+ * Test with a wrong key after the content type in the request's headers.
+ */
+ public function testWrongKeyInContentType() {
+ $bodyObject = $this->getValidBodyObject();
+ $this->expectExceptionMessage('Boundary is invalid');
+ $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => 'multipart/related; wrongkey="boundary_azertyuiop"'],
+ );
+ }
+
+ /**
+ * Test with a null Content-Type in the request's headers.
+ */
+ public function testNullContentType() {
+ $bodyObject = $this->getValidBodyObject();
+ $this->expectExceptionMessage('Content-Type can not be null');
+ $this->getMultipartParser(
+ $bodyObject,
+ ['Content-Type' => null],
+
+ );
+ }
+}
diff --git a/build/integration/features/bootstrap/WebDav.php b/build/integration/features/bootstrap/WebDav.php
index 31ca68ba92b..9f5e79a3ac6 100644
--- a/build/integration/features/bootstrap/WebDav.php
+++ b/build/integration/features/bootstrap/WebDav.php
@@ -538,6 +538,57 @@ trait WebDav {
}
/**
+ * @Given user :user uploads bulked files :name1 with :content1 and :name2 with :content2 and :name3 with :content3
+ * @param string $user
+ * @param string $name1
+ * @param string $content1
+ * @param string $name2
+ * @param string $content2
+ * @param string $name3
+ * @param string $content3
+ */
+ public function userUploadsChunkedFiles($user, $name1, $content1, $name2, $content2, $name3, $content3) {
+ $boundary = "boundary_azertyuiop";
+
+ $body = "";
+ $body .= '--'.$boundary."\r\n";
+ $body .= "X-File-Path: ".$name1."\r\n";
+ $body .= "X-File-MD5: f6a6263167c92de8644ac998b3c4e4d1\r\n";
+ $body .= "Content-Length: ".strlen($content1)."\r\n";
+ $body .= "\r\n";
+ $body .= $content1."\r\n";
+ $body .= '--'.$boundary."\r\n";
+ $body .= "X-File-Path: ".$name2."\r\n";
+ $body .= "X-File-MD5: 87c7d4068be07d390a1fffd21bf1e944\r\n";
+ $body .= "Content-Length: ".strlen($content2)."\r\n";
+ $body .= "\r\n";
+ $body .= $content2."\r\n";
+ $body .= '--'.$boundary."\r\n";
+ $body .= "X-File-Path: ".$name3."\r\n";
+ $body .= "X-File-MD5: e86a1cf0678099986a901c79086f5617\r\n";
+ $body .= "Content-Length: ".strlen($content3)."\r\n";
+ $body .= "\r\n";
+ $body .= $content3."\r\n";
+ $body .= '--'.$boundary."--\r\n";
+
+ $stream = fopen('php://temp','r+');
+ fwrite($stream, $body);
+ rewind($stream);
+
+ $client = new GClient();
+ $options = [
+ 'auth' => [$user, $this->regularUser],
+ 'headers' => [
+ 'Content-Type' => 'multipart/related; boundary='.$boundary,
+ 'Content-Length' => (string)strlen($body),
+ ],
+ 'body' => $body
+ ];
+
+ return $client->request("POST", substr($this->baseUrl, 0, -4) . "remote.php/dav/bulk", $options);
+ }
+
+ /**
* @Given user :user creates a new chunking upload with id :id
*/
public function userCreatesANewChunkingUploadWithId($user, $id) {
diff --git a/build/integration/features/webdav-related.feature b/build/integration/features/webdav-related.feature
index 66652e6fa26..c98ecc56ec7 100644
--- a/build/integration/features/webdav-related.feature
+++ b/build/integration/features/webdav-related.feature
@@ -608,3 +608,14 @@ Feature: webdav-related
And user "user0" uploads new chunk file "3" with "CCCCC" to id "chunking-42"
When user "user0" moves new chunk file with id "chunking-42" to "/myChunkedFile.txt" with size 15
Then the HTTP status code should be "201"
+
+ Scenario: Upload bulked files
+ Given user "user0" exists
+ And user "user0" uploads bulked files "A.txt" with "AAAAA" and "B.txt" with "BBBBB" and "C.txt" with "CCCCC"
+ When As an "user0"
+ Then Downloading file "/A.txt"
+ And Downloaded content should be "AAAAA"
+ And Downloading file "/B.txt"
+ And Downloaded content should be "BBBBB"
+ And Downloading file "/C.txt"
+ And Downloaded content should be "CCCCC"