1 files changed, 430 insertions, 275 deletions

diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php
index 7c536443a0e..5dc9e114532 100644
--- a/apps/files_external/lib/Lib/Storage/AmazonS3.php
+++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php
@@ -1,77 +1,68 @@
 <?php
+
 /**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author André Gaul <gaul@web-yard.de>
- * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
- * @author Christian Berendt <berendt@b1-systems.de>
- * @author Christopher T. Johnson <ctjctj@gmail.com>
- * @author Johan Björk <johanimon@gmail.com>
- * @author Jörn Friedrich Dreyer <jfd@butonic.de>
- * @author Martin Mattel <martin.mattel@diemattels.at>
- * @author Michael Gapczynski <GapczynskiM@gmail.com>
- * @author Morris Jobke <hey@morrisjobke.de>
- * @author Philipp Kapfer <philipp.kapfer@gmx.at>
- * @author Robin Appelman <robin@icewind.nl>
- * @author Robin McCorkell <robin@mccorkell.me.uk>
- * @author Thomas Müller <thomas.mueller@tmit.eu>
- * @author Vincent Petry <pvince81@owncloud.com>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program.  If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
  */
-
 namespace OCA\Files_External\Lib\Storage;
 
-use Aws\Result;
-use Aws\S3\S3Client;
 use Aws\S3\Exception\S3Exception;
 use Icewind\Streams\CallbackWrapper;
+use Icewind\Streams\CountWrapper;
 use Icewind\Streams\IteratorDirectory;
-use OC\Cache\CappedMemoryCache;
+use OC\Files\Cache\CacheEntry;
 use OC\Files\ObjectStore\S3ConnectionTrait;
 use OC\Files\ObjectStore\S3ObjectTrait;
+use OC\Files\Storage\Common;
+use OCP\Cache\CappedMemoryCache;
 use OCP\Constants;
-
-class AmazonS3 extends \OC\Files\Storage\Common {
+use OCP\Files\FileInfo;
+use OCP\Files\IMimeTypeDetector;
+use OCP\ICache;
+use OCP\ICacheFactory;
+use OCP\ITempManager;
+use OCP\Server;
+use Psr\Log\LoggerInterface;
+
+class AmazonS3 extends Common {
 	use S3ConnectionTrait;
 	use S3ObjectTrait;
 
-	public function needsPartFile() {
+	private LoggerInterface $logger;
+
+	public function needsPartFile(): bool {
 		return false;
 	}
 
-	/**
-	 * @var int in seconds
-	 */
-	private $rescanDelay = 10;
+	/** @var CappedMemoryCache<array|false> */
+	private CappedMemoryCache $objectCache;
 
-	/** @var CappedMemoryCache|Result[] */
-	private $objectCache;
+	/** @var CappedMemoryCache<bool> */
+	private CappedMemoryCache $directoryCache;
 
-	public function __construct($parameters) {
+	/** @var CappedMemoryCache<array> */
+	private CappedMemoryCache $filesCache;
+
+	private IMimeTypeDetector $mimeDetector;
+	private ?bool $versioningEnabled = null;
+	private ICache $memCache;
+
+	public function __construct(array $parameters) {
 		parent::__construct($parameters);
 		$this->parseParams($parameters);
+		$this->id = 'amazon::external::' . md5($this->params['hostname'] . ':' . $this->params['bucket'] . ':' . $this->params['key']);
 		$this->objectCache = new CappedMemoryCache();
+		$this->directoryCache = new CappedMemoryCache();
+		$this->filesCache = new CappedMemoryCache();
+		$this->mimeDetector = Server::get(IMimeTypeDetector::class);
+		/** @var ICacheFactory $cacheFactory */
+		$cacheFactory = Server::get(ICacheFactory::class);
+		$this->memCache = $cacheFactory->createLocal('s3-external');
+		$this->logger = Server::get(LoggerInterface::class);
 	}
 
-	/**
-	 * @param string $path
-	 * @return string correctly encoded path
-	 */
-	private function normalizePath($path) {
+	private function normalizePath(string $path): string {
 		$path = trim($path, '/');
 
 		if (!$path) {
@@ -81,43 +72,50 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $path;
 	}
 
-	private function isRoot($path) {
+	private function isRoot(string $path): bool {
 		return $path === '.';
 	}
 
-	private function cleanKey($path) {
+	private function cleanKey(string $path): string {
 		if ($this->isRoot($path)) {
 			return '/';
 		}
 		return $path;
 	}
 
-	private function clearCache() {
+	private function clearCache(): void {
 		$this->objectCache = new CappedMemoryCache();
+		$this->directoryCache = new CappedMemoryCache();
+		$this->filesCache = new CappedMemoryCache();
 	}
 
-	private function invalidateCache($key) {
+	private function invalidateCache(string $key): void {
 		unset($this->objectCache[$key]);
 		$keys = array_keys($this->objectCache->getData());
 		$keyLength = strlen($key);
 		foreach ($keys as $existingKey) {
-			if (substr($existingKey, 0, $keyLength) === $keys) {
+			if (substr($existingKey, 0, $keyLength) === $key) {
 				unset($this->objectCache[$existingKey]);
 			}
 		}
+		unset($this->filesCache[$key]);
+		$keys = array_keys($this->directoryCache->getData());
+		$keyLength = strlen($key);
+		foreach ($keys as $existingKey) {
+			if (substr($existingKey, 0, $keyLength) === $key) {
+				unset($this->directoryCache[$existingKey]);
+			}
+		}
+		unset($this->directoryCache[$key]);
 	}
 
-	/**
-	 * @param $key
-	 * @return Result|boolean
-	 */
-	private function headObject($key) {
+	private function headObject(string $key): array|false {
 		if (!isset($this->objectCache[$key])) {
 			try {
-				$this->objectCache[$key] = $this->getConnection()->headObject(array(
+				$this->objectCache[$key] = $this->getConnection()->headObject([
 					'Bucket' => $this->bucket,
 					'Key' => $key
-				));
+				] + $this->getSSECParameters())->toArray();
 			} catch (S3Exception $e) {
 				if ($e->getStatusCode() >= 500) {
 					throw $e;
@@ -126,59 +124,79 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			}
 		}
 
+		if (is_array($this->objectCache[$key]) && !isset($this->objectCache[$key]['Key'])) {
+			/** @psalm-suppress InvalidArgument Psalm doesn't understand nested arrays well */
+			$this->objectCache[$key]['Key'] = $key;
+		}
 		return $this->objectCache[$key];
 	}
 
 	/**
-	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
-	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
+	 * Return true if directory exists
+	 *
+	 * There are no folders in s3. A folder like structure could be archived
+	 * by prefixing files with the folder name.
 	 *
-	 * @param array $params
+	 * Implementation from flysystem-aws-s3-v3:
+	 * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
+	 *
+	 * @throws \Exception
 	 */
-	public function updateLegacyId(array $params) {
-		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
+	private function doesDirectoryExist(string $path): bool {
+		if ($path === '.' || $path === '') {
+			return true;
+		}
+		$path = rtrim($path, '/') . '/';
 
-		// find by old id or bucket
-		$stmt = \OC::$server->getDatabaseConnection()->prepare(
-			'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
-		);
-		$stmt->execute(array($oldId, $this->id));
-		while ($row = $stmt->fetch()) {
-			$storages[$row['id']] = $row['numeric_id'];
+		if (isset($this->directoryCache[$path])) {
+			return $this->directoryCache[$path];
 		}
+		try {
+			// Maybe this isn't an actual key, but a prefix.
+			// Do a prefix listing of objects to determine.
+			$result = $this->getConnection()->listObjectsV2([
+				'Bucket' => $this->bucket,
+				'Prefix' => $path,
+				'MaxKeys' => 1,
+			]);
+
+			if (isset($result['Contents'])) {
+				$this->directoryCache[$path] = true;
+				return true;
+			}
 
-		if (isset($storages[$this->id]) && isset($storages[$oldId])) {
-			// if both ids exist, delete the old storage and corresponding filecache entries
-			\OC\Files\Cache\Storage::remove($oldId);
-		} else if (isset($storages[$oldId])) {
-			// if only the old id exists do an update
-			$stmt = \OC::$server->getDatabaseConnection()->prepare(
-				'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
-			);
-			$stmt->execute(array($this->id, $oldId));
+			// empty directories have their own object
+			$object = $this->headObject($path);
+
+			if ($object) {
+				$this->directoryCache[$path] = true;
+				return true;
+			}
+		} catch (S3Exception $e) {
+			if ($e->getStatusCode() >= 400 && $e->getStatusCode() < 500) {
+				$this->directoryCache[$path] = false;
+			}
+			throw $e;
 		}
-		// only the bucket based id may exist, do nothing
+
+
+		$this->directoryCache[$path] = false;
+		return false;
 	}
 
-	/**
-	 * Remove a file or folder
-	 *
-	 * @param string $path
-	 * @return bool
-	 */
-	protected function remove($path) {
+	protected function remove(string $path): bool {
 		// remember fileType to reduce http calls
 		$fileType = $this->filetype($path);
 		if ($fileType === 'dir') {
 			return $this->rmdir($path);
-		} else if ($fileType === 'file') {
+		} elseif ($fileType === 'file') {
 			return $this->unlink($path);
 		} else {
 			return false;
 		}
 	}
 
-	public function mkdir($path) {
+	public function mkdir(string $path): bool {
 		$path = $this->normalizePath($path);
 
 		if ($this->is_dir($path)) {
@@ -186,15 +204,18 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->getConnection()->putObject(array(
+			$this->getConnection()->putObject([
 				'Bucket' => $this->bucket,
 				'Key' => $path . '/',
 				'Body' => '',
-				'ContentType' => 'httpd/unix-directory'
-			));
+				'ContentType' => FileInfo::MIMETYPE_FOLDER
+			] + $this->getSSECParameters());
 			$this->testTimeout();
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 
@@ -203,12 +224,12 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return true;
 	}
 
-	public function file_exists($path) {
+	public function file_exists(string $path): bool {
 		return $this->filetype($path) !== false;
 	}
 
 
-	public function rmdir($path) {
+	public function rmdir(string $path): bool {
 		$path = $this->normalizePath($path);
 
 		if ($this->isRoot($path)) {
@@ -223,128 +244,136 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $this->batchDelete($path);
 	}
 
-	protected function clearBucket() {
+	protected function clearBucket(): bool {
 		$this->clearCache();
-		try {
-			$this->getConnection()->clearBucket($this->bucket);
-			return true;
-			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
-		} catch (\Exception $e) {
-			return $this->batchDelete();
-		}
-		return false;
+		return $this->batchDelete();
 	}
 
-	private function batchDelete($path = null) {
-		$params = array(
+	private function batchDelete(?string $path = null): bool {
+		// TODO explore using https://docs.aws.amazon.com/aws-sdk-php/v3/api/class-Aws.S3.BatchDelete.html
+		$params = [
 			'Bucket' => $this->bucket
-		);
+		];
 		if ($path !== null) {
 			$params['Prefix'] = $path . '/';
 		}
 		try {
+			$connection = $this->getConnection();
 			// Since there are no real directories on S3, we need
 			// to delete all objects prefixed with the path.
 			do {
 				// instead of the iterator, manually loop over the list ...
-				$objects = $this->getConnection()->listObjects($params);
+				$objects = $connection->listObjects($params);
 				// ... so we can delete the files in batches
-				$this->getConnection()->deleteObjects(array(
-					'Bucket' => $this->bucket,
-					'Objects' => $objects['Contents']
-				));
-				$this->testTimeout();
+				if (isset($objects['Contents'])) {
+					$connection->deleteObjects([
+						'Bucket' => $this->bucket,
+						'Delete' => [
+							'Objects' => $objects['Contents']
+						]
+					]);
+					$this->testTimeout();
+				}
 				// we reached the end when the list is no longer truncated
 			} while ($objects['IsTruncated']);
+			if ($path !== '' && $path !== null) {
+				$this->deleteObject($path);
+			}
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 		return true;
 	}
 
-	public function opendir($path) {
+	public function opendir(string $path) {
+		try {
+			$content = iterator_to_array($this->getDirectoryContent($path));
+			return IteratorDirectory::wrap(array_map(function (array $item) {
+				return $item['name'];
+			}, $content));
+		} catch (S3Exception $e) {
+			return false;
+		}
+	}
+
+	public function stat(string $path): array|false {
 		$path = $this->normalizePath($path);
 
-		if ($this->isRoot($path)) {
-			$path = '';
+		if ($this->is_dir($path)) {
+			$stat = $this->getDirectoryMetaData($path);
 		} else {
-			$path .= '/';
+			$object = $this->headObject($path);
+			if ($object === false) {
+				return false;
+			}
+			$stat = $this->objectToMetaData($object);
 		}
+		$stat['atime'] = time();
 
-		try {
-			$files = array();
-			$results = $this->getConnection()->getPaginator('ListObjects', [
-				'Bucket' => $this->bucket,
-				'Delimiter' => '/',
-				'Prefix' => $path,
-			]);
+		return $stat;
+	}
 
-			foreach ($results as $result) {
-				// sub folders
-				if (is_array($result['CommonPrefixes'])) {
-					foreach ($result['CommonPrefixes'] as $prefix) {
-						$files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));
-					}
-				}
-				foreach ($result['Contents'] as $object) {
-					if (isset($object['Key']) && $object['Key'] === $path) {
-						// it's the directory itself, skip
-						continue;
-					}
-					$file = basename(
-						isset($object['Key']) ? $object['Key'] : $object['Prefix']
-					);
-					$files[] = $file;
-				}
-			}
+	/**
+	 * Return content length for object
+	 *
+	 * When the information is already present (e.g. opendir has been called before)
+	 * this value is return. Otherwise a headObject is emitted.
+	 */
+	private function getContentLength(string $path): int {
+		if (isset($this->filesCache[$path])) {
+			return (int)$this->filesCache[$path]['ContentLength'];
+		}
 
-			return IteratorDirectory::wrap($files);
-		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
-			return false;
+		$result = $this->headObject($path);
+		if (isset($result['ContentLength'])) {
+			return (int)$result['ContentLength'];
 		}
+
+		return 0;
 	}
 
-	public function stat($path) {
-		$path = $this->normalizePath($path);
+	/**
+	 * Return last modified for object
+	 *
+	 * When the information is already present (e.g. opendir has been called before)
+	 * this value is return. Otherwise a headObject is emitted.
+	 */
+	private function getLastModified(string $path): string {
+		if (isset($this->filesCache[$path])) {
+			return $this->filesCache[$path]['LastModified'];
+		}
 
-		try {
-			$stat = array();
-			if ($this->is_dir($path)) {
-				//folders don't really exist
-				$stat['size'] = -1; //unknown
-				$stat['mtime'] = time() - $this->rescanDelay * 1000;
-			} else {
-				$result = $this->headObject($path);
+		$result = $this->headObject($path);
+		if (isset($result['LastModified'])) {
+			return $result['LastModified'];
+		}
 
-				$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
-				if (isset($result['Metadata']['lastmodified'])) {
-					$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
-				} else {
-					$stat['mtime'] = strtotime($result['LastModified']);
-				}
-			}
-			$stat['atime'] = time();
+		return 'now';
+	}
 
-			return $stat;
-		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+	public function is_dir(string $path): bool {
+		$path = $this->normalizePath($path);
+
+		if (isset($this->filesCache[$path])) {
 			return false;
 		}
-	}
 
-	public function is_dir($path) {
-		$path = $this->normalizePath($path);
 		try {
-			return $this->isRoot($path) || $this->headObject($path . '/');
+			return $this->doesDirectoryExist($path);
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 	}
 
-	public function filetype($path) {
+	public function filetype(string $path): string|false {
 		$path = $this->normalizePath($path);
 
 		if ($this->isRoot($path)) {
@@ -352,21 +381,27 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			if ($this->headObject($path)) {
+			if (isset($this->directoryCache[$path]) && $this->directoryCache[$path]) {
+				return 'dir';
+			}
+			if (isset($this->filesCache[$path]) || $this->headObject($path)) {
 				return 'file';
 			}
-			if ($this->headObject($path . '/')) {
+			if ($this->doesDirectoryExist($path)) {
 				return 'dir';
 			}
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 
 		return false;
 	}
 
-	public function getPermissions($path) {
+	public function getPermissions(string $path): int {
 		$type = $this->filetype($path);
 		if (!$type) {
 			return 0;
@@ -374,7 +409,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
 	}
 
-	public function unlink($path) {
+	public function unlink(string $path): bool {
 		$path = $this->normalizePath($path);
 
 		if ($this->is_dir($path)) {
@@ -385,31 +420,43 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			$this->deleteObject($path);
 			$this->invalidateCache($path);
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 
 		return true;
 	}
 
-	public function fopen($path, $mode) {
+	public function fopen(string $path, string $mode) {
 		$path = $this->normalizePath($path);
 
 		switch ($mode) {
 			case 'r':
 			case 'rb':
+				// Don't try to fetch empty files
+				$stat = $this->stat($path);
+				if (is_array($stat) && isset($stat['size']) && $stat['size'] === 0) {
+					return fopen('php://memory', $mode);
+				}
+
 				try {
 					return $this->readObject($path);
-				} catch (S3Exception $e) {
-					\OCP\Util::logException('files_external', $e);
+				} catch (\Exception $e) {
+					$this->logger->error($e->getMessage(), [
+						'app' => 'files_external',
+						'exception' => $e,
+					]);
 					return false;
 				}
 			case 'w':
 			case 'wb':
-				$tmpFile = \OCP\Files::tmpFile();
+				$tmpFile = Server::get(ITempManager::class)->getTemporaryFile();
 
 				$handle = fopen($tmpFile, 'w');
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile): void {
 					$this->writeBack($tmpFile, $path);
 				});
 			case 'a':
@@ -427,24 +474,21 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 				} else {
 					$ext = '';
 				}
-				$tmpFile = \OCP\Files::tmpFile($ext);
+				$tmpFile = Server::get(ITempManager::class)->getTemporaryFile($ext);
 				if ($this->file_exists($path)) {
 					$source = $this->readObject($path);
 					file_put_contents($tmpFile, $source);
 				}
 
 				$handle = fopen($tmpFile, $mode);
-				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile): void {
 					$this->writeBack($tmpFile, $path);
 				});
 		}
 		return false;
 	}
 
-	public function touch($path, $mtime = null) {
-		$path = $this->normalizePath($path);
-
-		$metadata = array();
+	public function touch(string $path, ?int $mtime = null): bool {
 		if (is_null($mtime)) {
 			$mtime = time();
 		}
@@ -452,34 +496,26 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			'lastmodified' => gmdate(\DateTime::RFC1123, $mtime)
 		];
 
-		$fileType = $this->filetype($path);
 		try {
-			if ($fileType !== false) {
-				if ($fileType === 'dir' && !$this->isRoot($path)) {
-					$path .= '/';
-				}
-				$this->getConnection()->copyObject([
-					'Bucket' => $this->bucket,
-					'Key' => $this->cleanKey($path),
-					'Metadata' => $metadata,
-					'CopySource' => $this->bucket . '/' . $path,
-					'MetadataDirective' => 'REPLACE',
-				]);
-				$this->testTimeout();
-			} else {
-				$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
-				$this->getConnection()->putObject([
-					'Bucket' => $this->bucket,
-					'Key' => $this->cleanKey($path),
-					'Metadata' => $metadata,
-					'Body' => '',
-					'ContentType' => $mimeType,
-					'MetadataDirective' => 'REPLACE',
-				]);
-				$this->testTimeout();
+			if ($this->file_exists($path)) {
+				return false;
 			}
+
+			$mimeType = $this->mimeDetector->detectPath($path);
+			$this->getConnection()->putObject([
+				'Bucket' => $this->bucket,
+				'Key' => $this->cleanKey($path),
+				'Metadata' => $metadata,
+				'Body' => '',
+				'ContentType' => $mimeType,
+				'MetadataDirective' => 'REPLACE',
+			] + $this->getSSECParameters());
+			$this->testTimeout();
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 
@@ -487,78 +523,69 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return true;
 	}
 
-	public function copy($path1, $path2) {
-		$path1 = $this->normalizePath($path1);
-		$path2 = $this->normalizePath($path2);
+	public function copy(string $source, string $target, ?bool $isFile = null): bool {
+		$source = $this->normalizePath($source);
+		$target = $this->normalizePath($target);
 
-		if ($this->is_file($path1)) {
+		if ($isFile === true || $this->is_file($source)) {
 			try {
-				$this->getConnection()->copyObject(array(
-					'Bucket' => $this->bucket,
-					'Key' => $this->cleanKey($path2),
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
-				));
+				$this->copyObject($source, $target, [
+					'StorageClass' => $this->storageClass,
+				]);
 				$this->testTimeout();
 			} catch (S3Exception $e) {
-				\OCP\Util::logException('files_external', $e);
+				$this->logger->error($e->getMessage(), [
+					'app' => 'files_external',
+					'exception' => $e,
+				]);
 				return false;
 			}
 		} else {
-			$this->remove($path2);
+			$this->remove($target);
 
 			try {
-				$this->getConnection()->copyObject(array(
-					'Bucket' => $this->bucket,
-					'Key' => $path2 . '/',
-					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
-				));
+				$this->mkdir($target);
 				$this->testTimeout();
 			} catch (S3Exception $e) {
-				\OCP\Util::logException('files_external', $e);
+				$this->logger->error($e->getMessage(), [
+					'app' => 'files_external',
+					'exception' => $e,
+				]);
 				return false;
 			}
 
-			$dh = $this->opendir($path1);
-			if (is_resource($dh)) {
-				while (($file = readdir($dh)) !== false) {
-					if (\OC\Files\Filesystem::isIgnoredDir($file)) {
-						continue;
-					}
-
-					$source = $path1 . '/' . $file;
-					$target = $path2 . '/' . $file;
-					$this->copy($source, $target);
-				}
+			foreach ($this->getDirectoryContent($source) as $item) {
+				$childSource = $source . '/' . $item['name'];
+				$childTarget = $target . '/' . $item['name'];
+				$this->copy($childSource, $childTarget, $item['mimetype'] !== FileInfo::MIMETYPE_FOLDER);
 			}
 		}
 
-		$this->invalidateCache($path2);
+		$this->invalidateCache($target);
 
 		return true;
 	}
 
-	public function rename($path1, $path2) {
-		$path1 = $this->normalizePath($path1);
-		$path2 = $this->normalizePath($path2);
+	public function rename(string $source, string $target): bool {
+		$source = $this->normalizePath($source);
+		$target = $this->normalizePath($target);
 
-		if ($this->is_file($path1)) {
-
-			if ($this->copy($path1, $path2) === false) {
+		if ($this->is_file($source)) {
+			if ($this->copy($source, $target) === false) {
 				return false;
 			}
 
-			if ($this->unlink($path1) === false) {
-				$this->unlink($path2);
+			if ($this->unlink($source) === false) {
+				$this->unlink($target);
 				return false;
 			}
 		} else {
-
-			if ($this->copy($path1, $path2) === false) {
+			if ($this->copy($source, $target) === false) {
 				return false;
 			}
 
-			if ($this->rmdir($path1) === false) {
-				$this->rmdir($path2);
+			if ($this->rmdir($source) === false) {
+				$this->rmdir($target);
 				return false;
 			}
 		}
@@ -566,31 +593,30 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return true;
 	}
 
-	public function test() {
-		$test = $this->getConnection()->getBucketAcl(array(
-			'Bucket' => $this->bucket,
-		));
-		if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
-			return true;
-		}
-		return false;
+	public function test(): bool {
+		$this->getConnection()->headBucket([
+			'Bucket' => $this->bucket
+		]);
+		return true;
 	}
 
-	public function getId() {
+	public function getId(): string {
 		return $this->id;
 	}
 
-	public function writeBack($tmpFile, $path) {
+	public function writeBack(string $tmpFile, string $path): bool {
 		try {
 			$source = fopen($tmpFile, 'r');
-			$this->writeObject($path, $source);
+			$this->writeObject($path, $source, $this->mimeDetector->detectPath($path));
 			$this->invalidateCache($path);
-			fclose($source);
 
 			unlink($tmpFile);
 			return true;
 		} catch (S3Exception $e) {
-			\OCP\Util::logException('files_external', $e);
+			$this->logger->error($e->getMessage(), [
+				'app' => 'files_external',
+				'exception' => $e,
+			]);
 			return false;
 		}
 	}
@@ -598,8 +624,137 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 	/**
 	 * check if curl is installed
 	 */
-	public static function checkDependencies() {
+	public static function checkDependencies(): bool {
 		return true;
 	}
 
+	public function getDirectoryContent(string $directory): \Traversable {
+		$path = $this->normalizePath($directory);
+
+		if ($this->isRoot($path)) {
+			$path = '';
+		} else {
+			$path .= '/';
+		}
+
+		$results = $this->getConnection()->getPaginator('ListObjectsV2', [
+			'Bucket' => $this->bucket,
+			'Delimiter' => '/',
+			'Prefix' => $path,
+		]);
+
+		foreach ($results as $result) {
+			// sub folders
+			if (is_array($result['CommonPrefixes'])) {
+				foreach ($result['CommonPrefixes'] as $prefix) {
+					$dir = $this->getDirectoryMetaData($prefix['Prefix']);
+					if ($dir) {
+						yield $dir;
+					}
+				}
+			}
+			if (is_array($result['Contents'])) {
+				foreach ($result['Contents'] as $object) {
+					$this->objectCache[$object['Key']] = $object;
+					if ($object['Key'] !== $path) {
+						yield $this->objectToMetaData($object);
+					}
+				}
+			}
+		}
+	}
+
+	private function objectToMetaData(array $object): array {
+		return [
+			'name' => basename($object['Key']),
+			'mimetype' => $this->mimeDetector->detectPath($object['Key']),
+			'mtime' => strtotime($object['LastModified']),
+			'storage_mtime' => strtotime($object['LastModified']),
+			'etag' => trim($object['ETag'], '"'),
+			'permissions' => Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE,
+			'size' => (int)($object['Size'] ?? $object['ContentLength']),
+		];
+	}
+
+	private function getDirectoryMetaData(string $path): ?array {
+		$path = trim($path, '/');
+		// when versioning is enabled, delete markers are returned as part of CommonPrefixes
+		// resulting in "ghost" folders, verify that each folder actually exists
+		if ($this->versioningEnabled() && !$this->doesDirectoryExist($path)) {
+			return null;
+		}
+		$cacheEntry = $this->getCache()->get($path);
+		if ($cacheEntry instanceof CacheEntry) {
+			return $cacheEntry->getData();
+		} else {
+			return [
+				'name' => basename($path),
+				'mimetype' => FileInfo::MIMETYPE_FOLDER,
+				'mtime' => time(),
+				'storage_mtime' => time(),
+				'etag' => uniqid(),
+				'permissions' => Constants::PERMISSION_ALL,
+				'size' => -1,
+			];
+		}
+	}
+
+	public function versioningEnabled(): bool {
+		if ($this->versioningEnabled === null) {
+			$cached = $this->memCache->get('versioning-enabled::' . $this->getBucket());
+			if ($cached === null) {
+				$this->versioningEnabled = $this->getVersioningStatusFromBucket();
+				$this->memCache->set('versioning-enabled::' . $this->getBucket(), $this->versioningEnabled, 60);
+			} else {
+				$this->versioningEnabled = $cached;
+			}
+		}
+		return $this->versioningEnabled;
+	}
+
+	protected function getVersioningStatusFromBucket(): bool {
+		try {
+			$result = $this->getConnection()->getBucketVersioning(['Bucket' => $this->getBucket()]);
+			return $result->get('Status') === 'Enabled';
+		} catch (S3Exception $s3Exception) {
+			// This is needed for compatibility with Storj gateway which does not support versioning yet
+			if ($s3Exception->getAwsErrorCode() === 'NotImplemented' || $s3Exception->getAwsErrorCode() === 'AccessDenied') {
+				return false;
+			}
+			throw $s3Exception;
+		}
+	}
+
+	public function hasUpdated(string $path, int $time): bool {
+		// for files we can get the proper mtime
+		if ($path !== '' && $object = $this->headObject($path)) {
+			$stat = $this->objectToMetaData($object);
+			return $stat['mtime'] > $time;
+		} else {
+			// for directories, the only real option we have is to do a prefix listing and iterate over all objects
+			// however, since this is just as expensive as just re-scanning the directory, we can simply return true
+			// and have the scanner figure out if anything has actually changed
+			return true;
+		}
+	}
+
+	public function writeStream(string $path, $stream, ?int $size = null): int {
+		if ($size === null) {
+			$size = 0;
+			// track the number of bytes read from the input stream to return as the number of written bytes.
+			$stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void {
+				$size = $writtenSize;
+			});
+		}
+
+		if (!is_resource($stream)) {
+			throw new \InvalidArgumentException('Invalid stream provided');
+		}
+
+		$path = $this->normalizePath($path);
+		$this->writeObject($path, $stream, $this->mimeDetector->detectPath($path));
+		$this->invalidateCache($path);
+
+		return $size;
+	}
 }