aboutsummaryrefslogtreecommitdiffstats
path: root/apps/files_external/lib/Lib/Storage/AmazonS3.php
diff options
context:
space:
mode:
Diffstat (limited to 'apps/files_external/lib/Lib/Storage/AmazonS3.php')
-rw-r--r--apps/files_external/lib/Lib/Storage/AmazonS3.php343
1 files changed, 167 insertions, 176 deletions
diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php
index cfd78689fa4..5dc9e114532 100644
--- a/apps/files_external/lib/Lib/Storage/AmazonS3.php
+++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php
@@ -1,103 +1,68 @@
<?php
+
/**
- * @copyright Copyright (c) 2016, ownCloud, Inc.
- *
- * @author André Gaul <gaul@web-yard.de>
- * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
- * @author Christian Berendt <berendt@b1-systems.de>
- * @author Christopher T. Johnson <ctjctj@gmail.com>
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Daniel Kesselberg <mail@danielkesselberg.de>
- * @author enoch <lanxenet@hotmail.com>
- * @author Johan Björk <johanimon@gmail.com>
- * @author Jörn Friedrich Dreyer <jfd@butonic.de>
- * @author Julius Härtl <jus@bitgrid.net>
- * @author Martin Mattel <martin.mattel@diemattels.at>
- * @author Michael Gapczynski <GapczynskiM@gmail.com>
- * @author Morris Jobke <hey@morrisjobke.de>
- * @author Philipp Kapfer <philipp.kapfer@gmx.at>
- * @author Robin Appelman <robin@icewind.nl>
- * @author Robin McCorkell <robin@mccorkell.me.uk>
- * @author Roeland Jago Douma <roeland@famdouma.nl>
- * @author Thomas Müller <thomas.mueller@tmit.eu>
- * @author Vincent Petry <vincent@nextcloud.com>
- *
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program. If not, see <http://www.gnu.org/licenses/>
- *
+ * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-only
*/
namespace OCA\Files_External\Lib\Storage;
-use Aws\Result;
use Aws\S3\Exception\S3Exception;
-use Aws\S3\S3Client;
use Icewind\Streams\CallbackWrapper;
+use Icewind\Streams\CountWrapper;
use Icewind\Streams\IteratorDirectory;
-use OC\Cache\CappedMemoryCache;
use OC\Files\Cache\CacheEntry;
use OC\Files\ObjectStore\S3ConnectionTrait;
use OC\Files\ObjectStore\S3ObjectTrait;
+use OC\Files\Storage\Common;
+use OCP\Cache\CappedMemoryCache;
use OCP\Constants;
use OCP\Files\FileInfo;
use OCP\Files\IMimeTypeDetector;
+use OCP\ICache;
use OCP\ICacheFactory;
-use OCP\IMemcache;
+use OCP\ITempManager;
+use OCP\Server;
+use Psr\Log\LoggerInterface;
-class AmazonS3 extends \OC\Files\Storage\Common {
+class AmazonS3 extends Common {
use S3ConnectionTrait;
use S3ObjectTrait;
- public function needsPartFile() {
+ private LoggerInterface $logger;
+
+ public function needsPartFile(): bool {
return false;
}
- /** @var CappedMemoryCache|Result[] */
- private $objectCache;
-
- /** @var CappedMemoryCache|bool[] */
- private $directoryCache;
+ /** @var CappedMemoryCache<array|false> */
+ private CappedMemoryCache $objectCache;
- /** @var CappedMemoryCache|array */
- private $filesCache;
+ /** @var CappedMemoryCache<bool> */
+ private CappedMemoryCache $directoryCache;
- /** @var IMimeTypeDetector */
- private $mimeDetector;
+ /** @var CappedMemoryCache<array> */
+ private CappedMemoryCache $filesCache;
- /** @var bool|null */
- private $versioningEnabled = null;
+ private IMimeTypeDetector $mimeDetector;
+ private ?bool $versioningEnabled = null;
+ private ICache $memCache;
- /** @var IMemcache */
- private $memCache;
-
- public function __construct($parameters) {
+ public function __construct(array $parameters) {
parent::__construct($parameters);
$this->parseParams($parameters);
$this->id = 'amazon::external::' . md5($this->params['hostname'] . ':' . $this->params['bucket'] . ':' . $this->params['key']);
$this->objectCache = new CappedMemoryCache();
$this->directoryCache = new CappedMemoryCache();
$this->filesCache = new CappedMemoryCache();
- $this->mimeDetector = \OC::$server->get(IMimeTypeDetector::class);
+ $this->mimeDetector = Server::get(IMimeTypeDetector::class);
/** @var ICacheFactory $cacheFactory */
- $cacheFactory = \OC::$server->get(ICacheFactory::class);
+ $cacheFactory = Server::get(ICacheFactory::class);
$this->memCache = $cacheFactory->createLocal('s3-external');
+ $this->logger = Server::get(LoggerInterface::class);
}
- /**
- * @param string $path
- * @return string correctly encoded path
- */
- private function normalizePath($path) {
+ private function normalizePath(string $path): string {
$path = trim($path, '/');
if (!$path) {
@@ -107,24 +72,24 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $path;
}
- private function isRoot($path) {
+ private function isRoot(string $path): bool {
return $path === '.';
}
- private function cleanKey($path) {
+ private function cleanKey(string $path): string {
if ($this->isRoot($path)) {
return '/';
}
return $path;
}
- private function clearCache() {
+ private function clearCache(): void {
$this->objectCache = new CappedMemoryCache();
$this->directoryCache = new CappedMemoryCache();
$this->filesCache = new CappedMemoryCache();
}
- private function invalidateCache($key) {
+ private function invalidateCache(string $key): void {
unset($this->objectCache[$key]);
$keys = array_keys($this->objectCache->getData());
$keyLength = strlen($key);
@@ -144,17 +109,13 @@ class AmazonS3 extends \OC\Files\Storage\Common {
unset($this->directoryCache[$key]);
}
- /**
- * @param $key
- * @return array|false
- */
- private function headObject($key) {
+ private function headObject(string $key): array|false {
if (!isset($this->objectCache[$key])) {
try {
$this->objectCache[$key] = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
'Key' => $key
- ])->toArray();
+ ] + $this->getSSECParameters())->toArray();
} catch (S3Exception $e) {
if ($e->getStatusCode() >= 500) {
throw $e;
@@ -163,8 +124,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
}
- if (is_array($this->objectCache[$key]) && !isset($this->objectCache[$key]["Key"])) {
- $this->objectCache[$key]["Key"] = $key;
+ if (is_array($this->objectCache[$key]) && !isset($this->objectCache[$key]['Key'])) {
+ /** @psalm-suppress InvalidArgument Psalm doesn't understand nested arrays well */
+ $this->objectCache[$key]['Key'] = $key;
}
return $this->objectCache[$key];
}
@@ -178,11 +140,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
* Implementation from flysystem-aws-s3-v3:
* https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
*
- * @param $path
- * @return bool
* @throws \Exception
*/
- private function doesDirectoryExist($path) {
+ private function doesDirectoryExist(string $path): bool {
if ($path === '.' || $path === '') {
return true;
}
@@ -224,13 +184,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return false;
}
- /**
- * Remove a file or folder
- *
- * @param string $path
- * @return bool
- */
- protected function remove($path) {
+ protected function remove(string $path): bool {
// remember fileType to reduce http calls
$fileType = $this->filetype($path);
if ($fileType === 'dir') {
@@ -242,7 +196,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
}
- public function mkdir($path) {
+ public function mkdir(string $path): bool {
$path = $this->normalizePath($path);
if ($this->is_dir($path)) {
@@ -255,10 +209,13 @@ class AmazonS3 extends \OC\Files\Storage\Common {
'Key' => $path . '/',
'Body' => '',
'ContentType' => FileInfo::MIMETYPE_FOLDER
- ]);
+ ] + $this->getSSECParameters());
$this->testTimeout();
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
@@ -267,12 +224,12 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return true;
}
- public function file_exists($path) {
+ public function file_exists(string $path): bool {
return $this->filetype($path) !== false;
}
- public function rmdir($path) {
+ public function rmdir(string $path): bool {
$path = $this->normalizePath($path);
if ($this->isRoot($path)) {
@@ -287,20 +244,13 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $this->batchDelete($path);
}
- protected function clearBucket() {
+ protected function clearBucket(): bool {
$this->clearCache();
- try {
- $this->getConnection()->clearBucket([
- "Bucket" => $this->bucket
- ]);
- return true;
- // clearBucket() is not working with Ceph, so if it fails we try the slower approach
- } catch (\Exception $e) {
- return $this->batchDelete();
- }
+ return $this->batchDelete();
}
- private function batchDelete($path = null) {
+ private function batchDelete(?string $path = null): bool {
+ // TODO explore using https://docs.aws.amazon.com/aws-sdk-php/v3/api/class-Aws.S3.BatchDelete.html
$params = [
'Bucket' => $this->bucket
];
@@ -330,13 +280,16 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->deleteObject($path);
}
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
return true;
}
- public function opendir($path) {
+ public function opendir(string $path) {
try {
$content = iterator_to_array($this->getDirectoryContent($path));
return IteratorDirectory::wrap(array_map(function (array $item) {
@@ -347,7 +300,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
}
- public function stat($path) {
+ public function stat(string $path): array|false {
$path = $this->normalizePath($path);
if ($this->is_dir($path)) {
@@ -369,11 +322,8 @@ class AmazonS3 extends \OC\Files\Storage\Common {
*
* When the information is already present (e.g. opendir has been called before)
* this value is return. Otherwise a headObject is emitted.
- *
- * @param $path
- * @return int|mixed
*/
- private function getContentLength($path) {
+ private function getContentLength(string $path): int {
if (isset($this->filesCache[$path])) {
return (int)$this->filesCache[$path]['ContentLength'];
}
@@ -391,11 +341,8 @@ class AmazonS3 extends \OC\Files\Storage\Common {
*
* When the information is already present (e.g. opendir has been called before)
* this value is return. Otherwise a headObject is emitted.
- *
- * @param $path
- * @return mixed|string
*/
- private function getLastModified($path) {
+ private function getLastModified(string $path): string {
if (isset($this->filesCache[$path])) {
return $this->filesCache[$path]['LastModified'];
}
@@ -408,7 +355,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return 'now';
}
- public function is_dir($path) {
+ public function is_dir(string $path): bool {
$path = $this->normalizePath($path);
if (isset($this->filesCache[$path])) {
@@ -418,12 +365,15 @@ class AmazonS3 extends \OC\Files\Storage\Common {
try {
return $this->doesDirectoryExist($path);
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
}
- public function filetype($path) {
+ public function filetype(string $path): string|false {
$path = $this->normalizePath($path);
if ($this->isRoot($path)) {
@@ -441,14 +391,17 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return 'dir';
}
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
return false;
}
- public function getPermissions($path) {
+ public function getPermissions(string $path): int {
$type = $this->filetype($path);
if (!$type) {
return 0;
@@ -456,7 +409,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE;
}
- public function unlink($path) {
+ public function unlink(string $path): bool {
$path = $this->normalizePath($path);
if ($this->is_dir($path)) {
@@ -467,14 +420,17 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->deleteObject($path);
$this->invalidateCache($path);
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
return true;
}
- public function fopen($path, $mode) {
+ public function fopen(string $path, string $mode) {
$path = $this->normalizePath($path);
switch ($mode) {
@@ -488,16 +444,19 @@ class AmazonS3 extends \OC\Files\Storage\Common {
try {
return $this->readObject($path);
- } catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ } catch (\Exception $e) {
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
case 'w':
case 'wb':
- $tmpFile = \OC::$server->getTempManager()->getTemporaryFile();
+ $tmpFile = Server::get(ITempManager::class)->getTemporaryFile();
$handle = fopen($tmpFile, 'w');
- return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+ return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile): void {
$this->writeBack($tmpFile, $path);
});
case 'a':
@@ -515,21 +474,21 @@ class AmazonS3 extends \OC\Files\Storage\Common {
} else {
$ext = '';
}
- $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
+ $tmpFile = Server::get(ITempManager::class)->getTemporaryFile($ext);
if ($this->file_exists($path)) {
$source = $this->readObject($path);
file_put_contents($tmpFile, $source);
}
$handle = fopen($tmpFile, $mode);
- return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+ return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile): void {
$this->writeBack($tmpFile, $path);
});
}
return false;
}
- public function touch($path, $mtime = null) {
+ public function touch(string $path, ?int $mtime = null): bool {
if (is_null($mtime)) {
$mtime = time();
}
@@ -538,20 +497,25 @@ class AmazonS3 extends \OC\Files\Storage\Common {
];
try {
- if (!$this->file_exists($path)) {
- $mimeType = $this->mimeDetector->detectPath($path);
- $this->getConnection()->putObject([
- 'Bucket' => $this->bucket,
- 'Key' => $this->cleanKey($path),
- 'Metadata' => $metadata,
- 'Body' => '',
- 'ContentType' => $mimeType,
- 'MetadataDirective' => 'REPLACE',
- ]);
- $this->testTimeout();
+ if ($this->file_exists($path)) {
+ return false;
}
+
+ $mimeType = $this->mimeDetector->detectPath($path);
+ $this->getConnection()->putObject([
+ 'Bucket' => $this->bucket,
+ 'Key' => $this->cleanKey($path),
+ 'Metadata' => $metadata,
+ 'Body' => '',
+ 'ContentType' => $mimeType,
+ 'MetadataDirective' => 'REPLACE',
+ ] + $this->getSSECParameters());
+ $this->testTimeout();
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
@@ -559,65 +523,69 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return true;
}
- public function copy($path1, $path2, $isFile = null) {
- $path1 = $this->normalizePath($path1);
- $path2 = $this->normalizePath($path2);
+ public function copy(string $source, string $target, ?bool $isFile = null): bool {
+ $source = $this->normalizePath($source);
+ $target = $this->normalizePath($target);
- if ($isFile === true || $this->is_file($path1)) {
+ if ($isFile === true || $this->is_file($source)) {
try {
- $this->getConnection()->copyObject([
- 'Bucket' => $this->bucket,
- 'Key' => $this->cleanKey($path2),
- 'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
+ $this->copyObject($source, $target, [
+ 'StorageClass' => $this->storageClass,
]);
$this->testTimeout();
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
} else {
- $this->remove($path2);
+ $this->remove($target);
try {
- $this->mkdir($path2);
+ $this->mkdir($target);
$this->testTimeout();
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
- foreach ($this->getDirectoryContent($path1) as $item) {
- $source = $path1 . '/' . $item['name'];
- $target = $path2 . '/' . $item['name'];
- $this->copy($source, $target, $item['mimetype'] !== FileInfo::MIMETYPE_FOLDER);
+ foreach ($this->getDirectoryContent($source) as $item) {
+ $childSource = $source . '/' . $item['name'];
+ $childTarget = $target . '/' . $item['name'];
+ $this->copy($childSource, $childTarget, $item['mimetype'] !== FileInfo::MIMETYPE_FOLDER);
}
}
- $this->invalidateCache($path2);
+ $this->invalidateCache($target);
return true;
}
- public function rename($path1, $path2) {
- $path1 = $this->normalizePath($path1);
- $path2 = $this->normalizePath($path2);
+ public function rename(string $source, string $target): bool {
+ $source = $this->normalizePath($source);
+ $target = $this->normalizePath($target);
- if ($this->is_file($path1)) {
- if ($this->copy($path1, $path2) === false) {
+ if ($this->is_file($source)) {
+ if ($this->copy($source, $target) === false) {
return false;
}
- if ($this->unlink($path1) === false) {
- $this->unlink($path2);
+ if ($this->unlink($source) === false) {
+ $this->unlink($target);
return false;
}
} else {
- if ($this->copy($path1, $path2) === false) {
+ if ($this->copy($source, $target) === false) {
return false;
}
- if ($this->rmdir($path1) === false) {
- $this->rmdir($path2);
+ if ($this->rmdir($source) === false) {
+ $this->rmdir($target);
return false;
}
}
@@ -625,18 +593,18 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return true;
}
- public function test() {
+ public function test(): bool {
$this->getConnection()->headBucket([
'Bucket' => $this->bucket
]);
return true;
}
- public function getId() {
+ public function getId(): string {
return $this->id;
}
- public function writeBack($tmpFile, $path) {
+ public function writeBack(string $tmpFile, string $path): bool {
try {
$source = fopen($tmpFile, 'r');
$this->writeObject($path, $source, $this->mimeDetector->detectPath($path));
@@ -645,7 +613,10 @@ class AmazonS3 extends \OC\Files\Storage\Common {
unlink($tmpFile);
return true;
} catch (S3Exception $e) {
- \OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
+ $this->logger->error($e->getMessage(), [
+ 'app' => 'files_external',
+ 'exception' => $e,
+ ]);
return false;
}
}
@@ -653,11 +624,11 @@ class AmazonS3 extends \OC\Files\Storage\Common {
/**
* check if curl is installed
*/
- public static function checkDependencies() {
+ public static function checkDependencies(): bool {
return true;
}
- public function getDirectoryContent($directory): \Traversable {
+ public function getDirectoryContent(string $directory): \Traversable {
$path = $this->normalizePath($directory);
if ($this->isRoot($path)) {
@@ -699,7 +670,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
'mimetype' => $this->mimeDetector->detectPath($object['Key']),
'mtime' => strtotime($object['LastModified']),
'storage_mtime' => strtotime($object['LastModified']),
- 'etag' => $object['ETag'],
+ 'etag' => trim($object['ETag'], '"'),
'permissions' => Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE,
'size' => (int)($object['Size'] ?? $object['ContentLength']),
];
@@ -747,14 +718,14 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $result->get('Status') === 'Enabled';
} catch (S3Exception $s3Exception) {
// This is needed for compatibility with Storj gateway which does not support versioning yet
- if ($s3Exception->getAwsErrorCode() === 'NotImplemented') {
+ if ($s3Exception->getAwsErrorCode() === 'NotImplemented' || $s3Exception->getAwsErrorCode() === 'AccessDenied') {
return false;
}
throw $s3Exception;
}
}
- public function hasUpdated($path, $time) {
+ public function hasUpdated(string $path, int $time): bool {
// for files we can get the proper mtime
if ($path !== '' && $object = $this->headObject($path)) {
$stat = $this->objectToMetaData($object);
@@ -766,4 +737,24 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return true;
}
}
+
+ public function writeStream(string $path, $stream, ?int $size = null): int {
+ if ($size === null) {
+ $size = 0;
+ // track the number of bytes read from the input stream to return as the number of written bytes.
+ $stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void {
+ $size = $writtenSize;
+ });
+ }
+
+ if (!is_resource($stream)) {
+ throw new \InvalidArgumentException('Invalid stream provided');
+ }
+
+ $path = $this->normalizePath($path);
+ $this->writeObject($path, $stream, $this->mimeDetector->detectPath($path));
+ $this->invalidateCache($path);
+
+ return $size;
+ }
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-15 07:25:45 +0000