aboutsummaryrefslogtreecommitdiffstats
path: root/lib/private/Files/ObjectStore
diff options
context:
space:
mode:
Diffstat (limited to 'lib/private/Files/ObjectStore')
-rw-r--r--lib/private/Files/ObjectStore/Azure.php1
-rw-r--r--lib/private/Files/ObjectStore/ObjectStoreStorage.php20
-rw-r--r--lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php140
-rw-r--r--lib/private/Files/ObjectStore/S3.php1
-rw-r--r--lib/private/Files/ObjectStore/S3ConfigTrait.php4
-rw-r--r--lib/private/Files/ObjectStore/S3ConnectionTrait.php5
-rw-r--r--lib/private/Files/ObjectStore/S3ObjectTrait.php60
-rw-r--r--lib/private/Files/ObjectStore/S3Signature.php1
-rw-r--r--lib/private/Files/ObjectStore/StorageObjectStore.php1
9 files changed, 199 insertions, 34 deletions
diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php
index 575cc336ba8..2729bb3c037 100644
--- a/lib/private/Files/ObjectStore/Azure.php
+++ b/lib/private/Files/ObjectStore/Azure.php
@@ -1,4 +1,5 @@
<?php
+
/**
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php
index ebe87399ab4..10ee6aec167 100644
--- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php
+++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php
@@ -67,7 +67,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$this->logger = \OCP\Server::get(LoggerInterface::class);
}
- public function mkdir(string $path, bool $force = false): bool {
+ public function mkdir(string $path, bool $force = false, array $metadata = []): bool {
$path = $this->normalizePath($path);
if (!$force && $this->file_exists($path)) {
$this->logger->warning("Tried to create an object store folder that already exists: $path");
@@ -77,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$mTime = time();
$data = [
'mimetype' => 'httpd/unix-directory',
- 'size' => 0,
+ 'size' => $metadata['size'] ?? 0,
'mtime' => $mTime,
'storage_mtime' => $mTime,
'permissions' => \OCP\Constants::PERMISSION_ALL,
@@ -413,16 +413,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
//create a empty file, need to have at least on char to make it
// work with all object storage implementations
$this->file_put_contents($path, ' ');
- $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
- $stat = [
- 'etag' => $this->getETag($path),
- 'mimetype' => $mimeType,
- 'size' => 0,
- 'mtime' => $mtime,
- 'storage_mtime' => $mtime,
- 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
- ];
- $this->getCache()->put($path, $stat);
} catch (\Exception $ex) {
$this->logger->error(
'Could not create object for ' . $path,
@@ -607,8 +597,8 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool {
$sourceCache = $sourceStorage->getCache();
if (
- $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) &&
- $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()
+ $sourceStorage->instanceOfStorage(ObjectStoreStorage::class)
+ && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()
) {
if ($this->getCache()->get($targetInternalPath)) {
$this->unlink($targetInternalPath);
@@ -709,7 +699,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
if ($cache->inCache($to)) {
$cache->remove($to);
}
- $this->mkdir($to);
+ $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]);
foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) {
$this->copyInner($sourceCache, $child, $to . '/' . $child->getName());
diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
new file mode 100644
index 00000000000..fdfe989addc
--- /dev/null
+++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
@@ -0,0 +1,140 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OC\Files\ObjectStore;
+
+use OCP\App\IAppManager;
+use OCP\Files\ObjectStore\IObjectStore;
+use OCP\IConfig;
+use OCP\IUser;
+
+/**
+ * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}}
+ */
+class PrimaryObjectStoreConfig {
+ public function __construct(
+ private readonly IConfig $config,
+ private readonly IAppManager $appManager,
+ ) {
+ }
+
+ /**
+ * @param ObjectStoreConfig $config
+ */
+ public function buildObjectStore(array $config): IObjectStore {
+ return new $config['class']($config['arguments']);
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ public function getObjectStoreConfigForRoot(): ?array {
+ $config = $this->getObjectStoreConfig();
+
+ if ($config && $config['arguments']['multibucket']) {
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+
+ // put the root FS always in first bucket for multibucket configuration
+ $config['arguments']['bucket'] .= '0';
+ }
+ return $config;
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ public function getObjectStoreConfigForUser(IUser $user): ?array {
+ $config = $this->getObjectStoreConfig();
+
+ if ($config && $config['arguments']['multibucket']) {
+ $config['arguments']['bucket'] = $this->getBucketForUser($user, $config);
+ }
+ return $config;
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ private function getObjectStoreConfig(): ?array {
+ $objectStore = $this->config->getSystemValue('objectstore', null);
+ $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null);
+
+ // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config
+ if ($objectStoreMultiBucket) {
+ $objectStoreMultiBucket['arguments']['multibucket'] = true;
+ return $this->validateObjectStoreConfig($objectStoreMultiBucket);
+ } elseif ($objectStore) {
+ return $this->validateObjectStoreConfig($objectStore);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @return ObjectStoreConfig
+ */
+ private function validateObjectStoreConfig(array $config) {
+ if (!isset($config['class'])) {
+ throw new \Exception('No class configured for object store');
+ }
+ if (!isset($config['arguments'])) {
+ $config['arguments'] = [];
+ }
+ $class = $config['class'];
+ $arguments = $config['arguments'];
+ if (!is_array($arguments)) {
+ throw new \Exception('Configured object store arguments are not an array');
+ }
+ if (!isset($arguments['multibucket'])) {
+ $arguments['multibucket'] = false;
+ }
+ if (!is_bool($arguments['multibucket'])) {
+ throw new \Exception('arguments.multibucket must be a boolean in object store configuration');
+ }
+
+ if (!is_string($class)) {
+ throw new \Exception('Configured class for object store is not a string');
+ }
+
+ if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) {
+ [$appId] = explode('\\', $class);
+ $this->appManager->loadApp(strtolower($appId));
+ }
+
+ if (!is_a($class, IObjectStore::class, true)) {
+ throw new \Exception('Configured class for object store is not an object store');
+ }
+ return [
+ 'class' => $class,
+ 'arguments' => $arguments,
+ ];
+ }
+
+ private function getBucketForUser(IUser $user, array $config): string {
+ $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null);
+
+ if ($bucket === null) {
+ /*
+ * Use any provided bucket argument as prefix
+ * and add the mapping from username => bucket
+ */
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+ $mapper = new Mapper($user, $this->config);
+ $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64;
+ $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets);
+
+ $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket);
+ }
+
+ return $bucket;
+ }
+}
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
index 23c061db174..72e1751e23d 100644
--- a/lib/private/Files/ObjectStore/S3.php
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -1,4 +1,5 @@
<?php
+
/**
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php
index 63f14ac2d00..5b086db8f77 100644
--- a/lib/private/Files/ObjectStore/S3ConfigTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php
@@ -18,6 +18,10 @@ trait S3ConfigTrait {
/** Maximum number of concurrent multipart uploads */
protected int $concurrency;
+ /** Timeout, in seconds, for the connection to S3 server, not for the
+ * request. */
+ protected float $connectTimeout;
+
protected int $timeout;
protected string|false $proxy;
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
index b7017583dc2..67b82a44ab7 100644
--- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -1,4 +1,5 @@
<?php
+
/**
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
@@ -39,6 +40,7 @@ trait S3ConnectionTrait {
// Default to 5 like the S3 SDK does
$this->concurrency = $params['concurrency'] ?? 5;
$this->proxy = $params['proxy'] ?? false;
+ $this->connectTimeout = $params['connect_timeout'] ?? 5;
$this->timeout = $params['timeout'] ?? 15;
$this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
@@ -102,8 +104,7 @@ trait S3ConnectionTrait {
'use_arn_region' => false,
'http' => [
'verify' => $this->getCertificateBundlePath(),
- // Timeout for the connection to S3 server, not for the request.
- 'connect_timeout' => 5
+ 'connect_timeout' => $this->connectTimeout,
],
'use_aws_shared_config_files' => false,
'retries' => [
diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
index 61e8158b863..5e6dcf88a42 100644
--- a/lib/private/Files/ObjectStore/S3ObjectTrait.php
+++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -119,28 +119,54 @@ trait S3ObjectTrait {
protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void {
$mimetype = $metaData['mimetype'] ?? null;
unset($metaData['mimetype']);
- $uploader = new MultipartUploader($this->getConnection(), $stream, [
- 'bucket' => $this->bucket,
- 'concurrency' => $this->concurrency,
- 'key' => $urn,
- 'part_size' => $this->uploadPartSize,
- 'params' => [
- 'ContentType' => $mimetype,
- 'Metadata' => $this->buildS3Metadata($metaData),
- 'StorageClass' => $this->storageClass,
- ] + $this->getSSECParameters(),
- ]);
- try {
- $uploader->upload();
- } catch (S3MultipartUploadException $e) {
+ $attempts = 0;
+ $uploaded = false;
+ $concurrency = $this->concurrency;
+ $exception = null;
+ $state = null;
+
+ // retry multipart upload once with concurrency at half on failure
+ while (!$uploaded && $attempts <= 1) {
+ $uploader = new MultipartUploader($this->getConnection(), $stream, [
+ 'bucket' => $this->bucket,
+ 'concurrency' => $concurrency,
+ 'key' => $urn,
+ 'part_size' => $this->uploadPartSize,
+ 'state' => $state,
+ 'params' => [
+ 'ContentType' => $mimetype,
+ 'Metadata' => $this->buildS3Metadata($metaData),
+ 'StorageClass' => $this->storageClass,
+ ] + $this->getSSECParameters(),
+ ]);
+
+ try {
+ $uploader->upload();
+ $uploaded = true;
+ } catch (S3MultipartUploadException $e) {
+ $exception = $e;
+ $attempts++;
+
+ if ($concurrency > 1) {
+ $concurrency = round($concurrency / 2);
+ }
+
+ if ($stream->isSeekable()) {
+ $stream->rewind();
+ }
+ }
+ }
+
+ if (!$uploaded) {
// if anything goes wrong with multipart, make sure that you don´t poison and
// slow down s3 bucket with orphaned fragments
- $uploadInfo = $e->getState()->getId();
- if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) {
+ $uploadInfo = $exception->getState()->getId();
+ if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) {
$this->getConnection()->abortMultipartUpload($uploadInfo);
}
- throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $e);
+
+ throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception);
}
}
diff --git a/lib/private/Files/ObjectStore/S3Signature.php b/lib/private/Files/ObjectStore/S3Signature.php
index 994d65cc219..b80382ff67d 100644
--- a/lib/private/Files/ObjectStore/S3Signature.php
+++ b/lib/private/Files/ObjectStore/S3Signature.php
@@ -1,4 +1,5 @@
<?php
+
/**
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php
index 4361795ec45..888602a62e4 100644
--- a/lib/private/Files/ObjectStore/StorageObjectStore.php
+++ b/lib/private/Files/ObjectStore/StorageObjectStore.php
@@ -1,4 +1,5 @@
<?php
+
/**
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-01 23:11:45 +0000