aboutsummaryrefslogtreecommitdiffstats
path: root/lib/private/Files/ObjectStore/S3ConnectionTrait.php
diff options
context:
space:
mode:
Diffstat (limited to 'lib/private/Files/ObjectStore/S3ConnectionTrait.php')
-rw-r--r--lib/private/Files/ObjectStore/S3ConnectionTrait.php210
1 files changed, 120 insertions, 90 deletions
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
index c3836749c6d..67b82a44ab7 100644
--- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -1,32 +1,8 @@
<?php
+
/**
- * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
- *
- * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
- * @author Christoph Wurst <christoph@winzerhof-wurst.at>
- * @author Florent <florent@coppint.com>
- * @author James Letendre <James.Letendre@gmail.com>
- * @author Morris Jobke <hey@morrisjobke.de>
- * @author Robin Appelman <robin@icewind.nl>
- * @author Roeland Jago Douma <roeland@famdouma.nl>
- * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com>
- * @author Stephen Cuppett <steve@cuppett.com>
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
+ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Files\ObjectStore;
@@ -36,57 +12,53 @@ use Aws\Credentials\Credentials;
use Aws\Exception\CredentialsException;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
-use GuzzleHttp\Promise;
+use GuzzleHttp\Promise\Create;
use GuzzleHttp\Promise\RejectedPromise;
+use OCP\Files\StorageNotAvailableException;
use OCP\ICertificateManager;
+use OCP\Server;
use Psr\Log\LoggerInterface;
trait S3ConnectionTrait {
- /** @var array */
- protected $params;
-
- /** @var S3Client */
- protected $connection;
-
- /** @var string */
- protected $id;
-
- /** @var string */
- protected $bucket;
-
- /** @var int */
- protected $timeout;
+ use S3ConfigTrait;
- /** @var string */
- protected $proxy;
+ protected string $id;
- /** @var int */
- protected $uploadPartSize;
+ protected bool $test;
- /** @var int */
- private $putSizeLimit;
-
- protected $test;
+ protected ?S3Client $connection = null;
protected function parseParams($params) {
if (empty($params['bucket'])) {
- throw new \Exception("Bucket has to be configured.");
+ throw new \Exception('Bucket has to be configured.');
}
$this->id = 'amazon::' . $params['bucket'];
$this->test = isset($params['test']);
$this->bucket = $params['bucket'];
+ // Default to 5 like the S3 SDK does
+ $this->concurrency = $params['concurrency'] ?? 5;
$this->proxy = $params['proxy'] ?? false;
+ $this->connectTimeout = $params['connect_timeout'] ?? 5;
$this->timeout = $params['timeout'] ?? 15;
+ $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
$this->putSizeLimit = $params['putSizeLimit'] ?? 104857600;
+ $this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000;
+ $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true);
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
+ $params['s3-accelerate'] = $params['hostname'] === 's3-accelerate.amazonaws.com' || $params['hostname'] === 's3-accelerate.dualstack.amazonaws.com';
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
}
- $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists'];
+ $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true;
+
+ if ($params['s3-accelerate']) {
+ $params['verify_bucket_exists'] = false;
+ }
+
$this->params = $params;
}
@@ -105,7 +77,7 @@ trait S3ConnectionTrait {
* @throws \Exception if connection could not be made
*/
public function getConnection() {
- if (!is_null($this->connection)) {
+ if ($this->connection !== null) {
return $this->connection;
}
@@ -121,17 +93,8 @@ trait S3ConnectionTrait {
)
);
- // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage
- if (!isset($this->params['primary_storage'])) {
- /** @var ICertificateManager $certManager */
- $certManager = \OC::$server->get(ICertificateManager::class);
- $certPath = $certManager->getAbsoluteBundlePath();
- } else {
- $certPath = \OC::$SERVERROOT . '/resources/config/ca-bundle.crt';
- }
-
$options = [
- 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest',
+ 'version' => $this->params['version'] ?? 'latest',
'credentials' => $provider,
'endpoint' => $base_url,
'region' => $this->params['region'],
@@ -139,8 +102,23 @@ trait S3ConnectionTrait {
'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()),
'csm' => false,
'use_arn_region' => false,
- 'http' => ['verify' => $certPath],
+ 'http' => [
+ 'verify' => $this->getCertificateBundlePath(),
+ 'connect_timeout' => $this->connectTimeout,
+ ],
+ 'use_aws_shared_config_files' => false,
+ 'retries' => [
+ 'mode' => 'standard',
+ 'max_attempts' => 5,
+ ],
];
+
+ if ($this->params['s3-accelerate']) {
+ $options['use_accelerate_endpoint'] = true;
+ } else {
+ $options['endpoint'] = $base_url;
+ }
+
if ($this->getProxy()) {
$options['http']['proxy'] = $this->getProxy();
}
@@ -149,33 +127,38 @@ trait S3ConnectionTrait {
}
$this->connection = new S3Client($options);
- if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
- $logger = \OC::$server->get(LoggerInterface::class);
- $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.',
- ['app' => 'objectstore']);
- }
+ try {
+ $logger = Server::get(LoggerInterface::class);
+ if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
+ $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.',
+ ['app' => 'objectstore']);
+ }
- if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
- $logger = \OC::$server->get(LoggerInterface::class);
- try {
- $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
- if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
- throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket);
+ if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
+ try {
+ $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
+ if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
+ throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket);
+ }
+ $this->connection->createBucket(['Bucket' => $this->bucket]);
+ $this->testTimeout();
+ } catch (S3Exception $e) {
+ $logger->debug('Invalid remote storage.', [
+ 'exception' => $e,
+ 'app' => 'objectstore',
+ ]);
+ if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') {
+ throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
+ }
}
- $this->connection->createBucket(['Bucket' => $this->bucket]);
- $this->testTimeout();
- } catch (S3Exception $e) {
- $logger->debug('Invalid remote storage.', [
- 'exception' => $e,
- 'app' => 'objectstore',
- ]);
- throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
}
- }
- // google cloud's s3 compatibility doesn't like the EncodingType parameter
- if (strpos($base_url, 'storage.googleapis.com')) {
- $this->connection->getHandlerList()->remove('s3.auto_encode');
+ // google cloud's s3 compatibility doesn't like the EncodingType parameter
+ if (strpos($base_url, 'storage.googleapis.com')) {
+ $this->connection->getHandlerList()->remove('s3.auto_encode');
+ }
+ } catch (S3Exception $e) {
+ throw new StorageNotAvailableException('S3 service is unable to handle request: ' . $e->getMessage());
}
return $this->connection;
@@ -203,14 +186,16 @@ trait S3ConnectionTrait {
/**
* This function creates a credential provider based on user parameter file
*/
- protected function paramCredentialProvider() : callable {
+ protected function paramCredentialProvider(): callable {
return function () {
$key = empty($this->params['key']) ? null : $this->params['key'];
$secret = empty($this->params['secret']) ? null : $this->params['secret'];
+ $sessionToken = empty($this->params['session_token']) ? null : $this->params['session_token'];
if ($key && $secret) {
- return Promise\promise_for(
- new Credentials($key, $secret)
+ return Create::promiseFor(
+ // a null sessionToken match the default signature of the constructor
+ new Credentials($key, $secret, $sessionToken)
);
}
@@ -218,4 +203,49 @@ trait S3ConnectionTrait {
return new RejectedPromise(new CredentialsException($msg));
};
}
+
+ protected function getCertificateBundlePath(): ?string {
+ if ((int)($this->params['use_nextcloud_bundle'] ?? '0')) {
+ // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage
+ if (!isset($this->params['primary_storage'])) {
+ /** @var ICertificateManager $certManager */
+ $certManager = Server::get(ICertificateManager::class);
+ return $certManager->getAbsoluteBundlePath();
+ } else {
+ return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt';
+ }
+ } else {
+ return null;
+ }
+ }
+
+ protected function getSSECKey(): ?string {
+ if (isset($this->params['sse_c_key']) && !empty($this->params['sse_c_key'])) {
+ return $this->params['sse_c_key'];
+ }
+
+ return null;
+ }
+
+ protected function getSSECParameters(bool $copy = false): array {
+ $key = $this->getSSECKey();
+
+ if ($key === null) {
+ return [];
+ }
+
+ $rawKey = base64_decode($key);
+ if ($copy) {
+ return [
+ 'CopySourceSSECustomerAlgorithm' => 'AES256',
+ 'CopySourceSSECustomerKey' => $rawKey,
+ 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true)
+ ];
+ }
+ return [
+ 'SSECustomerAlgorithm' => 'AES256',
+ 'SSECustomerKey' => $rawKey,
+ 'SSECustomerKeyMD5' => md5($rawKey, true)
+ ];
+ }
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-14 20:27:57 +0000