Merge pull request #6602 from nextcloud/s3-multipart-upload

Add multipart upload for s3

2 files changed, 61 insertions, 5 deletions

diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
index 3ba4da92b98..2cbe20d9801 100644
--- a/lib/private/Files/ObjectStore/S3ObjectTrait.php
+++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -21,9 +21,13 @@
 
 namespace OC\Files\ObjectStore;
 
+use Aws\Exception\MultipartUploadException;
+use Aws\S3\MultipartUploader;
 use Aws\S3\S3Client;
 use Psr\Http\Message\StreamInterface;
 
+const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
+
 trait S3ObjectTrait {
 	/**
 	 * Returns the connection
@@ -60,6 +64,17 @@ trait S3ObjectTrait {
 	 * @since 7.0.0
 	 */
 	function writeObject($urn, $stream) {
+		$stat = fstat($stream);
+
+		if ($stat['size'] && $stat['size'] < S3_UPLOAD_PART_SIZE) {
+			$this->singlePartUpload($urn, $stream);
+		} else {
+			$this->multiPartUpload($urn, $stream);
+		}
+
+	}
+
+	protected function singlePartUpload($urn, $stream) {
 		$this->getConnection()->putObject([
 			'Bucket' => $this->bucket,
 			'Key' => $urn,
@@ -67,6 +82,34 @@ trait S3ObjectTrait {
 		]);
 	}
 
+	protected function multiPartUpload($urn, $stream) {
+		$uploader = new MultipartUploader($this->getConnection(), $stream, [
+			'bucket' => $this->bucket,
+			'key' => $urn,
+			'part_size' => S3_UPLOAD_PART_SIZE
+		]);
+
+		$tries = 0;
+
+		do {
+			try {
+				$result = $uploader->upload();
+			} catch (MultipartUploadException $e) {
+				\OC::$server->getLogger()->logException($e);
+				rewind($stream);
+				$tries++;
+
+				if ($tries < 5) {
+					$uploader = new MultipartUploader($this->getConnection(), $stream, [
+						'state' => $e->getState()
+					]);
+				} else {
+					$this->getConnection()->abortMultipartUpload($e->getState()->getId());
+				}
+			}
+		} while (!isset($result) && $tries < 5);
+	}
+
 	/**
 	 * @param string $urn the unified resource name used to identify the object
 	 * @return void
@@ -79,4 +122,4 @@ trait S3ObjectTrait {
 			'Key' => $urn
 		]);
 	}
-}
\ No newline at end of file
+}
diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php
index b93e9beebdc..14167656fb5 100644
--- a/tests/lib/Files/ObjectStore/S3Test.php
+++ b/tests/lib/Files/ObjectStore/S3Test.php
@@ -23,19 +23,32 @@ namespace Test\Files\ObjectStore;
 
 use OC\Files\ObjectStore\S3;
 
+class MultiPartUploadS3 extends S3 {
+	public function multiPartUpload($urn, $stream) {
+		parent::multiPartUpload($urn, $stream);
+	}
+}
+
 /**
  * @group PRIMARY-s3
  */
 class S3Test extends ObjectStoreTest {
-	/**
-	 * @return \OCP\Files\ObjectStore\IObjectStore
-	 */
 	protected function getInstance() {
 		$config = \OC::$server->getConfig()->getSystemValue('objectstore');
 		if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
 			$this->markTestSkipped('objectstore not configured for s3');
 		}
 
-		return new S3($config['arguments']);
+		return new MultiPartUploadS3($config['arguments']);
+	}
+
+	public function testMultiPartUploader() {
+		$s3 = $this->getInstance();
+
+		$s3->multiPartUpload('multiparttest', fopen(__FILE__, 'r'));
+
+		$result = $s3->readObject('multiparttest');
+
+		$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
 	}
 }