feat(s3): Use multipart upload for chunked uploading

This allows to stream file chunks directly to S3 during upload.

Signed-off-by: Julius Härtl <jus@bitgrid.net>

2 files changed, 129 insertions, 0 deletions

diff --git a/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php b/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php
new file mode 100644
index 00000000000..f46982f3112
--- /dev/null
+++ b/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php
@@ -0,0 +1,59 @@
+<?php
+/*
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+declare(strict_types=1);
+
+
+namespace OCP\Files\ObjectStore;
+
+use Aws\Result;
+
+/**
+ * @since 26.0.0
+ */
+interface IObjectStoreMultiPartUpload {
+	/**
+	 * @since 26.0.0
+	 */
+	public function initiateMultipartUpload(string $urn): string;
+
+	/**
+	 * @since 26.0.0
+	 */
+	public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result;
+
+	/**
+	 * @since 26.0.0
+	 */
+	public function completeMultipartUpload(string $urn, string $uploadId, array $result): int;
+
+	/**
+	 * @since 26.0.0
+	 */
+	public function abortMultipartUpload(string $urn, string $uploadId): void;
+
+	/**
+	 * @since 26.0.0
+	 */
+	public function getMultipartUploads(string $urn, string $uploadId): array;
+}
diff --git a/lib/public/Files/Storage/IChunkedFileWrite.php b/lib/public/Files/Storage/IChunkedFileWrite.php
new file mode 100644
index 00000000000..01f5cbbb20a
--- /dev/null
+++ b/lib/public/Files/Storage/IChunkedFileWrite.php
@@ -0,0 +1,70 @@
+<?php
+/*
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+declare(strict_types=1);
+
+
+namespace OCP\Files\Storage;
+
+use OCP\Files\GenericFileException;
+
+/**
+ * @since 26.0.0
+ */
+interface IChunkedFileWrite extends IStorage {
+	/**
+	 * @param string $targetPath Relative target path in the storage
+	 * @return string writeToken to be used with the other methods to uniquely identify the file write operation
+	 * @throws GenericFileException
+	 * @since 26.0.0
+	 */
+	public function startChunkedWrite(string $targetPath): string;
+
+	/**
+	 * @param string $targetPath
+	 * @param string $writeToken
+	 * @param string $chunkId
+	 * @param resource $data
+	 * @param int|null $size
+	 * @throws GenericFileException
+	 * @since 26.0.0
+	 */
+	public function putChunkedWritePart(string $targetPath, string $writeToken, string $chunkId, $data, int $size = null): ?array;
+
+	/**
+	 * @param string $targetPath
+	 * @param string $writeToken
+	 * @return int
+	 * @throws GenericFileException
+	 * @since 26.0.0
+	 */
+	public function completeChunkedWrite(string $targetPath, string $writeToken): int;
+
+	/**
+	 * @param string $targetPath
+	 * @param string $writeToken
+	 * @throws GenericFileException
+	 * @since 26.0.0
+	 */
+	public function cancelChunkedWrite(string $targetPath, string $writeToken): void;
+}