Merge pull request #6602 from nextcloud/s3-multipart-upload

Add multipart upload for s3

1 files changed, 44 insertions, 1 deletions

diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
index 3ba4da92b98..2cbe20d9801 100644
--- a/lib/private/Files/ObjectStore/S3ObjectTrait.php
+++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -21,9 +21,13 @@
 
 namespace OC\Files\ObjectStore;
 
+use Aws\Exception\MultipartUploadException;
+use Aws\S3\MultipartUploader;
 use Aws\S3\S3Client;
 use Psr\Http\Message\StreamInterface;
 
+const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
+
 trait S3ObjectTrait {
 	/**
 	 * Returns the connection
@@ -60,6 +64,17 @@ trait S3ObjectTrait {
 	 * @since 7.0.0
 	 */
 	function writeObject($urn, $stream) {
+		$stat = fstat($stream);
+
+		if ($stat['size'] && $stat['size'] < S3_UPLOAD_PART_SIZE) {
+			$this->singlePartUpload($urn, $stream);
+		} else {
+			$this->multiPartUpload($urn, $stream);
+		}
+
+	}
+
+	protected function singlePartUpload($urn, $stream) {
 		$this->getConnection()->putObject([
 			'Bucket' => $this->bucket,
 			'Key' => $urn,
@@ -67,6 +82,34 @@ trait S3ObjectTrait {
 		]);
 	}
 
+	protected function multiPartUpload($urn, $stream) {
+		$uploader = new MultipartUploader($this->getConnection(), $stream, [
+			'bucket' => $this->bucket,
+			'key' => $urn,
+			'part_size' => S3_UPLOAD_PART_SIZE
+		]);
+
+		$tries = 0;
+
+		do {
+			try {
+				$result = $uploader->upload();
+			} catch (MultipartUploadException $e) {
+				\OC::$server->getLogger()->logException($e);
+				rewind($stream);
+				$tries++;
+
+				if ($tries < 5) {
+					$uploader = new MultipartUploader($this->getConnection(), $stream, [
+						'state' => $e->getState()
+					]);
+				} else {
+					$this->getConnection()->abortMultipartUpload($e->getState()->getId());
+				}
+			}
+		} while (!isset($result) && $tries < 5);
+	}
+
 	/**
 	 * @param string $urn the unified resource name used to identify the object
 	 * @return void
@@ -79,4 +122,4 @@ trait S3ObjectTrait {
 			'Key' => $urn
 		]);
 	}
-}
\ No newline at end of file
+}