diff options
Diffstat (limited to 'lib/private/Files/ObjectStore')
4 files changed, 166 insertions, 43 deletions
diff --git a/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php new file mode 100644 index 00000000000..369182b069d --- /dev/null +++ b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php @@ -0,0 +1,13 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\ObjectStore; + +class InvalidObjectStoreConfigurationException extends \Exception { + +} diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 10ee6aec167..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -475,6 +475,9 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil 'original-storage' => $this->getId(), 'original-path' => $path, ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -496,32 +499,27 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - if ($this->objectStore instanceof IObjectStoreMetaData) { - $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); - } else { - $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } - if (is_resource($countStream)) { - fclose($countStream); - } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - if ($this->objectStore instanceof IObjectStoreMetaData) { - $this->objectStore->writeObjectWithMetaData($urn, $stream, $metadata); - } else { - $this->objectStore->writeObject($urn, $stream, $metadata['mimetype']); - } - if (is_resource($stream)) { - fclose($stream); - } + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -545,7 +543,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil ] ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { @@ -561,7 +559,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index fdfe989addc..ffc33687340 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -34,9 +34,13 @@ class PrimaryObjectStoreConfig { * @return ?ObjectStoreConfig */ public function getObjectStoreConfigForRoot(): ?array { - $config = $this->getObjectStoreConfig(); + if (!$this->hasObjectStore()) { + return null; + } + + $config = $this->getObjectStoreConfiguration('root'); - if ($config && $config['arguments']['multibucket']) { + if ($config['arguments']['multibucket']) { if (!isset($config['arguments']['bucket'])) { $config['arguments']['bucket'] = ''; } @@ -51,38 +55,102 @@ class PrimaryObjectStoreConfig { * @return ?ObjectStoreConfig */ public function getObjectStoreConfigForUser(IUser $user): ?array { - $config = $this->getObjectStoreConfig(); + if (!$this->hasObjectStore()) { + return null; + } - if ($config && $config['arguments']['multibucket']) { + $store = $this->getObjectStoreForUser($user); + $config = $this->getObjectStoreConfiguration($store); + + if ($config['arguments']['multibucket']) { $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); } return $config; } /** - * @return ?ObjectStoreConfig + * @param string $name + * @return ObjectStoreConfig */ - private function getObjectStoreConfig(): ?array { + public function getObjectStoreConfiguration(string $name): array { + $configs = $this->getObjectStoreConfigs(); + $name = $this->resolveAlias($name); + if (!isset($configs[$name])) { + throw new \Exception("Object store configuration for '$name' not found"); + } + if (is_string($configs[$name])) { + throw new \Exception("Object store configuration for '{$configs[$name]}' not found"); + } + return $configs[$name]; + } + + public function resolveAlias(string $name): string { + $configs = $this->getObjectStoreConfigs(); + + while (isset($configs[$name]) && is_string($configs[$name])) { + $name = $configs[$name]; + } + return $name; + } + + public function hasObjectStore(): bool { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + return $objectStore || $objectStoreMultiBucket; + } + + public function hasMultipleObjectStorages(): bool { + $objectStore = $this->config->getSystemValue('objectstore', []); + return isset($objectStore['default']); + } + + /** + * @return ?array<string, ObjectStoreConfig|string> + * @throws InvalidObjectStoreConfigurationException + */ + public function getObjectStoreConfigs(): ?array { $objectStore = $this->config->getSystemValue('objectstore', null); $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config if ($objectStoreMultiBucket) { $objectStoreMultiBucket['arguments']['multibucket'] = true; - return $this->validateObjectStoreConfig($objectStoreMultiBucket); + return [ + 'default' => 'server1', + 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), + 'root' => 'server1', + ]; } elseif ($objectStore) { - return $this->validateObjectStoreConfig($objectStore); + if (!isset($objectStore['default'])) { + $objectStore = [ + 'default' => 'server1', + 'root' => 'server1', + 'server1' => $objectStore, + ]; + } + if (!isset($objectStore['root'])) { + $objectStore['root'] = 'default'; + } + + if (!is_string($objectStore['default'])) { + throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); + } + return array_map($this->validateObjectStoreConfig(...), $objectStore); } else { return null; } } /** - * @return ObjectStoreConfig + * @param array|string $config + * @return string|ObjectStoreConfig */ - private function validateObjectStoreConfig(array $config) { + private function validateObjectStoreConfig(array|string $config): array|string { + if (is_string($config)) { + return $config; + } if (!isset($config['class'])) { - throw new \Exception('No class configured for object store'); + throw new InvalidObjectStoreConfigurationException('No class configured for object store'); } if (!isset($config['arguments'])) { $config['arguments'] = []; @@ -90,17 +158,17 @@ class PrimaryObjectStoreConfig { $class = $config['class']; $arguments = $config['arguments']; if (!is_array($arguments)) { - throw new \Exception('Configured object store arguments are not an array'); + throw new InvalidObjectStoreConfigurationException('Configured object store arguments are not an array'); } if (!isset($arguments['multibucket'])) { $arguments['multibucket'] = false; } if (!is_bool($arguments['multibucket'])) { - throw new \Exception('arguments.multibucket must be a boolean in object store configuration'); + throw new InvalidObjectStoreConfigurationException('arguments.multibucket must be a boolean in object store configuration'); } if (!is_string($class)) { - throw new \Exception('Configured class for object store is not a string'); + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not a string'); } if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { @@ -109,7 +177,7 @@ class PrimaryObjectStoreConfig { } if (!is_a($class, IObjectStore::class, true)) { - throw new \Exception('Configured class for object store is not an object store'); + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not an object store'); } return [ 'class' => $class, @@ -117,8 +185,8 @@ class PrimaryObjectStoreConfig { ]; } - private function getBucketForUser(IUser $user, array $config): string { - $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + public function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->getSetBucketForUser($user); if ($bucket === null) { /* @@ -129,7 +197,7 @@ class PrimaryObjectStoreConfig { $config['arguments']['bucket'] = ''; } $mapper = new Mapper($user, $this->config); - $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $numBuckets = $config['arguments']['num_buckets'] ?? 64; $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); @@ -137,4 +205,21 @@ class PrimaryObjectStoreConfig { return $bucket; } + + public function getSetBucketForUser(IUser $user): ?string { + return $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + } + + public function getObjectStoreForUser(IUser $user): string { + if ($this->hasMultipleObjectStorages()) { + $value = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null); + if ($value === null) { + $value = $this->resolveAlias('default'); + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $value); + } + return $value; + } else { + return 'default'; + } + } } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 5e6dcf88a42..89405de2e8e 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -6,6 +6,8 @@ */ namespace OC\Files\ObjectStore; +use Aws\Command; +use Aws\Exception\MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; @@ -96,7 +98,9 @@ trait S3ObjectTrait { protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { $mimetype = $metaData['mimetype'] ?? null; unset($metaData['mimetype']); - $this->getConnection()->putObject([ + unset($metaData['size']); + + $args = [ 'Bucket' => $this->bucket, 'Key' => $urn, 'Body' => $stream, @@ -104,7 +108,13 @@ trait S3ObjectTrait { 'ContentType' => $mimetype, 'Metadata' => $this->buildS3Metadata($metaData), 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters()); + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); } @@ -119,12 +129,15 @@ trait S3ObjectTrait { protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { $mimetype = $metaData['mimetype'] ?? null; unset($metaData['mimetype']); + unset($metaData['size']); $attempts = 0; $uploaded = false; $concurrency = $this->concurrency; $exception = null; $state = null; + $size = $stream->getSize(); + $totalWritten = 0; // retry multipart upload once with concurrency at half on failure while (!$uploaded && $attempts <= 1) { @@ -139,6 +152,15 @@ trait S3ObjectTrait { 'Metadata' => $this->buildS3Metadata($metaData), 'StorageClass' => $this->storageClass, ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, ]); try { @@ -155,6 +177,9 @@ trait S3ObjectTrait { if ($stream->isSeekable()) { $stream->rewind(); } + } catch (MultipartUploadException $e) { + $exception = $e; + break; } } @@ -180,7 +205,9 @@ trait S3ObjectTrait { public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { $canSeek = fseek($stream, 0, SEEK_CUR) === 0; - $psrStream = Utils::streamFor($stream); + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); $size = $psrStream->getSize(); |