aboutsummaryrefslogtreecommitdiffstats
path: root/lib/private/Files/ObjectStore
diff options
context:
space:
mode:
Diffstat (limited to 'lib/private/Files/ObjectStore')
-rw-r--r--lib/private/Files/ObjectStore/ObjectStoreStorage.php34
-rw-r--r--lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php140
-rw-r--r--lib/private/Files/ObjectStore/S3.php12
-rw-r--r--lib/private/Files/ObjectStore/S3ConfigTrait.php4
-rw-r--r--lib/private/Files/ObjectStore/S3ConnectionTrait.php4
-rw-r--r--lib/private/Files/ObjectStore/S3ObjectTrait.php103
6 files changed, 247 insertions, 50 deletions
diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php
index 16ef4e7de63..752c3cf4fb7 100644
--- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php
+++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php
@@ -22,6 +22,7 @@ use OCP\Files\FileInfo;
use OCP\Files\GenericFileException;
use OCP\Files\NotFoundException;
use OCP\Files\ObjectStore\IObjectStore;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\Storage\IStorage;
@@ -66,7 +67,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$this->logger = \OCP\Server::get(LoggerInterface::class);
}
- public function mkdir(string $path, bool $force = false): bool {
+ public function mkdir(string $path, bool $force = false, array $metadata = []): bool {
$path = $this->normalizePath($path);
if (!$force && $this->file_exists($path)) {
$this->logger->warning("Tried to create an object store folder that already exists: $path");
@@ -76,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$mTime = time();
$data = [
'mimetype' => 'httpd/unix-directory',
- 'size' => 0,
+ 'size' => $metadata['size'] ?? 0,
'mtime' => $mTime,
'storage_mtime' => $mTime,
'permissions' => \OCP\Constants::PERMISSION_ALL,
@@ -412,16 +413,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
//create a empty file, need to have at least on char to make it
// work with all object storage implementations
$this->file_put_contents($path, ' ');
- $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
- $stat = [
- 'etag' => $this->getETag($path),
- 'mimetype' => $mimeType,
- 'size' => 0,
- 'mtime' => $mtime,
- 'storage_mtime' => $mtime,
- 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
- ];
- $this->getCache()->put($path, $stat);
} catch (\Exception $ex) {
$this->logger->error(
'Could not create object for ' . $path,
@@ -479,6 +470,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$mimetypeDetector = \OC::$server->getMimeTypeDetector();
$mimetype = $mimetypeDetector->detectPath($path);
+ $metadata = [
+ 'mimetype' => $mimetype,
+ 'original-storage' => $this->getId(),
+ 'original-path' => $path,
+ ];
$stat['mimetype'] = $mimetype;
$stat['etag'] = $this->getETag($path);
@@ -507,13 +503,21 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
]);
$size = $writtenSize;
});
- $this->objectStore->writeObject($urn, $countStream, $mimetype);
+ if ($this->objectStore instanceof IObjectStoreMetaData) {
+ $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata);
+ } else {
+ $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']);
+ }
if (is_resource($countStream)) {
fclose($countStream);
}
$stat['size'] = $size;
} else {
- $this->objectStore->writeObject($urn, $stream, $mimetype);
+ if ($this->objectStore instanceof IObjectStoreMetaData) {
+ $this->objectStore->writeObjectWithMetaData($urn, $stream, $metadata);
+ } else {
+ $this->objectStore->writeObject($urn, $stream, $metadata['mimetype']);
+ }
if (is_resource($stream)) {
fclose($stream);
}
@@ -695,7 +699,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
if ($cache->inCache($to)) {
$cache->remove($to);
}
- $this->mkdir($to);
+ $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]);
foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) {
$this->copyInner($sourceCache, $child, $to . '/' . $child->getName());
diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
new file mode 100644
index 00000000000..fdfe989addc
--- /dev/null
+++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
@@ -0,0 +1,140 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OC\Files\ObjectStore;
+
+use OCP\App\IAppManager;
+use OCP\Files\ObjectStore\IObjectStore;
+use OCP\IConfig;
+use OCP\IUser;
+
+/**
+ * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}}
+ */
+class PrimaryObjectStoreConfig {
+ public function __construct(
+ private readonly IConfig $config,
+ private readonly IAppManager $appManager,
+ ) {
+ }
+
+ /**
+ * @param ObjectStoreConfig $config
+ */
+ public function buildObjectStore(array $config): IObjectStore {
+ return new $config['class']($config['arguments']);
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ public function getObjectStoreConfigForRoot(): ?array {
+ $config = $this->getObjectStoreConfig();
+
+ if ($config && $config['arguments']['multibucket']) {
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+
+ // put the root FS always in first bucket for multibucket configuration
+ $config['arguments']['bucket'] .= '0';
+ }
+ return $config;
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ public function getObjectStoreConfigForUser(IUser $user): ?array {
+ $config = $this->getObjectStoreConfig();
+
+ if ($config && $config['arguments']['multibucket']) {
+ $config['arguments']['bucket'] = $this->getBucketForUser($user, $config);
+ }
+ return $config;
+ }
+
+ /**
+ * @return ?ObjectStoreConfig
+ */
+ private function getObjectStoreConfig(): ?array {
+ $objectStore = $this->config->getSystemValue('objectstore', null);
+ $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null);
+
+ // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config
+ if ($objectStoreMultiBucket) {
+ $objectStoreMultiBucket['arguments']['multibucket'] = true;
+ return $this->validateObjectStoreConfig($objectStoreMultiBucket);
+ } elseif ($objectStore) {
+ return $this->validateObjectStoreConfig($objectStore);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * @return ObjectStoreConfig
+ */
+ private function validateObjectStoreConfig(array $config) {
+ if (!isset($config['class'])) {
+ throw new \Exception('No class configured for object store');
+ }
+ if (!isset($config['arguments'])) {
+ $config['arguments'] = [];
+ }
+ $class = $config['class'];
+ $arguments = $config['arguments'];
+ if (!is_array($arguments)) {
+ throw new \Exception('Configured object store arguments are not an array');
+ }
+ if (!isset($arguments['multibucket'])) {
+ $arguments['multibucket'] = false;
+ }
+ if (!is_bool($arguments['multibucket'])) {
+ throw new \Exception('arguments.multibucket must be a boolean in object store configuration');
+ }
+
+ if (!is_string($class)) {
+ throw new \Exception('Configured class for object store is not a string');
+ }
+
+ if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) {
+ [$appId] = explode('\\', $class);
+ $this->appManager->loadApp(strtolower($appId));
+ }
+
+ if (!is_a($class, IObjectStore::class, true)) {
+ throw new \Exception('Configured class for object store is not an object store');
+ }
+ return [
+ 'class' => $class,
+ 'arguments' => $arguments,
+ ];
+ }
+
+ private function getBucketForUser(IUser $user, array $config): string {
+ $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null);
+
+ if ($bucket === null) {
+ /*
+ * Use any provided bucket argument as prefix
+ * and add the mapping from username => bucket
+ */
+ if (!isset($config['arguments']['bucket'])) {
+ $config['arguments']['bucket'] = '';
+ }
+ $mapper = new Mapper($user, $this->config);
+ $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64;
+ $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets);
+
+ $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket);
+ }
+
+ return $bucket;
+ }
+}
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
index e970fb6ac14..23c061db174 100644
--- a/lib/private/Files/ObjectStore/S3.php
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -95,6 +95,16 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaD
]);
}
+ private function parseS3Metadata(array $metadata): array {
+ $result = [];
+ foreach ($metadata as $key => $value) {
+ if (str_starts_with($key, 'x-amz-meta-')) {
+ $result[substr($key, strlen('x-amz-meta-'))] = $value;
+ }
+ }
+ return $result;
+ }
+
public function getObjectMetaData(string $urn): array {
$object = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
@@ -104,7 +114,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaD
'mtime' => $object['LastModified'],
'etag' => trim($object['ETag'], '"'),
'size' => (int)($object['Size'] ?? $object['ContentLength']),
- ];
+ ] + $this->parseS3Metadata($object['Metadata'] ?? []);
}
public function listObjects(string $prefix = ''): \Iterator {
diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php
index 63f14ac2d00..5b086db8f77 100644
--- a/lib/private/Files/ObjectStore/S3ConfigTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php
@@ -18,6 +18,10 @@ trait S3ConfigTrait {
/** Maximum number of concurrent multipart uploads */
protected int $concurrency;
+ /** Timeout, in seconds, for the connection to S3 server, not for the
+ * request. */
+ protected float $connectTimeout;
+
protected int $timeout;
protected string|false $proxy;
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
index b7017583dc2..062d2e4bde4 100644
--- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php
+++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -39,6 +39,7 @@ trait S3ConnectionTrait {
// Default to 5 like the S3 SDK does
$this->concurrency = $params['concurrency'] ?? 5;
$this->proxy = $params['proxy'] ?? false;
+ $this->connectTimeout = $params['connect_timeout'] ?? 5;
$this->timeout = $params['timeout'] ?? 15;
$this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
@@ -102,8 +103,7 @@ trait S3ConnectionTrait {
'use_arn_region' => false,
'http' => [
'verify' => $this->getCertificateBundlePath(),
- // Timeout for the connection to S3 server, not for the request.
- 'connect_timeout' => 5
+ 'connect_timeout' => $this->connectTimeout,
],
'use_aws_shared_config_files' => false,
'retries' => [
diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
index 9d7cfa644e6..5e6dcf88a42 100644
--- a/lib/private/Files/ObjectStore/S3ObjectTrait.php
+++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -77,22 +77,32 @@ trait S3ObjectTrait {
return $fh;
}
+ private function buildS3Metadata(array $metadata): array {
+ $result = [];
+ foreach ($metadata as $key => $value) {
+ $result['x-amz-meta-' . $key] = $value;
+ }
+ return $result;
+ }
/**
* Single object put helper
*
* @param string $urn the unified resource name used to identify the object
* @param StreamInterface $stream stream with the data to write
- * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
+ * @param array $metaData the metadata to set for the object
* @throws \Exception when something goes wrong, message will be logged
*/
- protected function writeSingle(string $urn, StreamInterface $stream, ?string $mimetype = null): void {
+ protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void {
+ $mimetype = $metaData['mimetype'] ?? null;
+ unset($metaData['mimetype']);
$this->getConnection()->putObject([
'Bucket' => $this->bucket,
'Key' => $urn,
'Body' => $stream,
'ACL' => 'private',
'ContentType' => $mimetype,
+ 'Metadata' => $this->buildS3Metadata($metaData),
'StorageClass' => $this->storageClass,
] + $this->getSSECParameters());
}
@@ -103,43 +113,72 @@ trait S3ObjectTrait {
*
* @param string $urn the unified resource name used to identify the object
* @param StreamInterface $stream stream with the data to write
- * @param string|null $mimetype the mimetype to set for the remove object
+ * @param array $metaData the metadata to set for the object
* @throws \Exception when something goes wrong, message will be logged
*/
- protected function writeMultiPart(string $urn, StreamInterface $stream, ?string $mimetype = null): void {
- $uploader = new MultipartUploader($this->getConnection(), $stream, [
- 'bucket' => $this->bucket,
- 'concurrency' => $this->concurrency,
- 'key' => $urn,
- 'part_size' => $this->uploadPartSize,
- 'params' => [
- 'ContentType' => $mimetype,
- 'StorageClass' => $this->storageClass,
- ] + $this->getSSECParameters(),
- ]);
+ protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void {
+ $mimetype = $metaData['mimetype'] ?? null;
+ unset($metaData['mimetype']);
+
+ $attempts = 0;
+ $uploaded = false;
+ $concurrency = $this->concurrency;
+ $exception = null;
+ $state = null;
+
+ // retry multipart upload once with concurrency at half on failure
+ while (!$uploaded && $attempts <= 1) {
+ $uploader = new MultipartUploader($this->getConnection(), $stream, [
+ 'bucket' => $this->bucket,
+ 'concurrency' => $concurrency,
+ 'key' => $urn,
+ 'part_size' => $this->uploadPartSize,
+ 'state' => $state,
+ 'params' => [
+ 'ContentType' => $mimetype,
+ 'Metadata' => $this->buildS3Metadata($metaData),
+ 'StorageClass' => $this->storageClass,
+ ] + $this->getSSECParameters(),
+ ]);
+
+ try {
+ $uploader->upload();
+ $uploaded = true;
+ } catch (S3MultipartUploadException $e) {
+ $exception = $e;
+ $attempts++;
+
+ if ($concurrency > 1) {
+ $concurrency = round($concurrency / 2);
+ }
- try {
- $uploader->upload();
- } catch (S3MultipartUploadException $e) {
+ if ($stream->isSeekable()) {
+ $stream->rewind();
+ }
+ }
+ }
+
+ if (!$uploaded) {
// if anything goes wrong with multipart, make sure that you don´t poison and
// slow down s3 bucket with orphaned fragments
- $uploadInfo = $e->getState()->getId();
- if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) {
+ $uploadInfo = $exception->getState()->getId();
+ if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) {
$this->getConnection()->abortMultipartUpload($uploadInfo);
}
- throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $e);
+
+ throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception);
}
}
-
- /**
- * @param string $urn the unified resource name used to identify the object
- * @param resource $stream stream with the data to write
- * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
- * @throws \Exception when something goes wrong, message will be logged
- * @since 7.0.0
- */
public function writeObject($urn, $stream, ?string $mimetype = null) {
+ $metaData = [];
+ if ($mimetype) {
+ $metaData['mimetype'] = $mimetype;
+ }
+ $this->writeObjectWithMetaData($urn, $stream, $metaData);
+ }
+
+ public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void {
$canSeek = fseek($stream, 0, SEEK_CUR) === 0;
$psrStream = Utils::streamFor($stream);
@@ -154,16 +193,16 @@ trait S3ObjectTrait {
$buffer->seek(0);
if ($buffer->getSize() < $this->putSizeLimit) {
// buffer is fully seekable, so use it directly for the small upload
- $this->writeSingle($urn, $buffer, $mimetype);
+ $this->writeSingle($urn, $buffer, $metaData);
} else {
$loadStream = new Psr7\AppendStream([$buffer, $psrStream]);
- $this->writeMultiPart($urn, $loadStream, $mimetype);
+ $this->writeMultiPart($urn, $loadStream, $metaData);
}
} else {
if ($size < $this->putSizeLimit) {
- $this->writeSingle($urn, $psrStream, $mimetype);
+ $this->writeSingle($urn, $psrStream, $metaData);
} else {
- $this->writeMultiPart($urn, $psrStream, $mimetype);
+ $this->writeMultiPart($urn, $psrStream, $metaData);
}
}
$psrStream->close();
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-01 04:52:09 +0000