diff options
Diffstat (limited to 'lib/private/Files/ObjectStore')
18 files changed, 1358 insertions, 723 deletions
diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php index bdc41f9ed95..aaaee044bac 100644 --- a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -3,43 +3,27 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> - * - * @author Morris Jobke <hey@morrisjobke.de> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { + private string $internalId; - /** @var string */ - private $internalId; - - public function __construct($params) { - if (!isset($params['internal-id'])) { + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (!isset($parameters['internal-id'])) { throw new \Exception('missing id in parameters'); } - $this->internalId = (string)$params['internal-id']; - parent::__construct($params); + $this->internalId = (string)$parameters['internal-id']; + parent::__construct($parameters); } - public function getId() { + public function getId(): string { return 'object::appdata::preview:' . $this->internalId; } } diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php index 2ef13d60c56..2729bb3c037 100644 --- a/lib/private/Files/ObjectStore/Azure.php +++ b/lib/private/Files/ObjectStore/Azure.php @@ -1,29 +1,13 @@ <?php + /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; use MicrosoftAzure\Storage\Blob\BlobRestProxy; +use MicrosoftAzure\Storage\Blob\Models\CreateBlockBlobOptions; use MicrosoftAzure\Storage\Common\Exceptions\ServiceException; use OCP\Files\ObjectStore\IObjectStore; @@ -38,13 +22,13 @@ class Azure implements IObjectStore { private $blobClient = null; /** @var string|null */ private $endpoint = null; - /** @var bool */ + /** @var bool */ private $autoCreate = false; /** * @param array $parameters */ - public function __construct($parameters) { + public function __construct(array $parameters) { $this->containerName = $parameters['container']; $this->accountName = $parameters['account_name']; $this->accountKey = $parameters['account_key']; @@ -62,7 +46,7 @@ class Azure implements IObjectStore { private function getBlobClient() { if (!$this->blobClient) { $protocol = $this->endpoint ? substr($this->endpoint, 0, strpos($this->endpoint, ':')) : 'https'; - $connectionString = "DefaultEndpointsProtocol=" . $protocol . ";AccountName=" . $this->accountName . ";AccountKey=" . $this->accountKey; + $connectionString = 'DefaultEndpointsProtocol=' . $protocol . ';AccountName=' . $this->accountName . ';AccountKey=' . $this->accountKey; if ($this->endpoint) { $connectionString .= ';BlobEndpoint=' . $this->endpoint; } @@ -100,13 +84,12 @@ class Azure implements IObjectStore { return $blob->getContentStream(); } - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @throws \Exception when something goes wrong, message will be logged - */ - public function writeObject($urn, $stream) { - $this->getBlobClient()->createBlockBlob($this->containerName, $urn, $stream); + public function writeObject($urn, $stream, ?string $mimetype = null) { + $options = new CreateBlockBlobOptions(); + if ($mimetype) { + $options->setContentType($mimetype); + } + $this->getBlobClient()->createBlockBlob($this->containerName, $urn, $stream, $options); } /** diff --git a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php index 7a95665b10d..4e2d10705fe 100644 --- a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php @@ -1,69 +1,42 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; -use OC\User\User; +use Exception; +use OCP\Files\IHomeStorage; +use OCP\IUser; -class HomeObjectStoreStorage extends ObjectStoreStorage implements \OCP\Files\IHomeStorage { +class HomeObjectStoreStorage extends ObjectStoreStorage implements IHomeStorage { + protected IUser $user; /** * The home user storage requires a user object to create a unique storage id - * @param array $params + * + * @param array $parameters + * @throws Exception */ - public function __construct($params) { - if (! isset($params['user']) || ! $params['user'] instanceof User) { - throw new \Exception('missing user object in parameters'); + public function __construct(array $parameters) { + if (! isset($parameters['user']) || ! $parameters['user'] instanceof IUser) { + throw new Exception('missing user object in parameters'); } - $this->user = $params['user']; - parent::__construct($params); + $this->user = $parameters['user']; + parent::__construct($parameters); } - public function getId() { + public function getId(): string { return 'object::user:' . $this->user->getUID(); } - /** - * get the owner of a path - * - * @param string $path The path to get the owner - * @return false|string uid - */ - public function getOwner($path) { - if (is_object($this->user)) { - return $this->user->getUID(); - } - return false; + public function getOwner(string $path): string|false { + return $this->user->getUID(); } - /** - * @param string $path, optional - * @return \OC\User\User - */ - public function getUser($path = null) { + public function getUser(): IUser { return $this->user; } } diff --git a/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php new file mode 100644 index 00000000000..369182b069d --- /dev/null +++ b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php @@ -0,0 +1,13 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\ObjectStore; + +class InvalidObjectStoreConfigurationException extends \Exception { + +} diff --git a/lib/private/Files/ObjectStore/Mapper.php b/lib/private/Files/ObjectStore/Mapper.php index a5186877738..e1174a285a6 100644 --- a/lib/private/Files/ObjectStore/Mapper.php +++ b/lib/private/Files/ObjectStore/Mapper.php @@ -1,28 +1,13 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; +use OCP\IConfig; use OCP\IUser; /** @@ -36,13 +21,18 @@ class Mapper { /** @var IUser */ private $user; + /** @var IConfig */ + private $config; + /** * Mapper constructor. * * @param IUser $user + * @param IConfig $config */ - public function __construct(IUser $user) { + public function __construct(IUser $user, IConfig $config) { $this->user = $user; + $this->config = $config; } /** @@ -50,8 +40,15 @@ class Mapper { * @return string */ public function getBucket($numBuckets = 64) { + // Get the bucket config and shift if provided. + // Allow us to prevent writing in old filled buckets + $config = $this->config->getSystemValue('objectstore_multibucket'); + $minBucket = is_array($config) && isset($config['arguments']['min_bucket']) + ? (int)$config['arguments']['min_bucket'] + : 0; + $hash = md5($this->user->getUID()); $num = hexdec(substr($hash, 0, 4)); - return (string)($num % $numBuckets); + return (string)(($num % ($numBuckets - $minBucket)) + $minBucket); } } diff --git a/lib/private/Files/ObjectStore/NoopScanner.php b/lib/private/Files/ObjectStore/NoopScanner.php deleted file mode 100644 index 9195e7f8d9f..00000000000 --- a/lib/private/Files/ObjectStore/NoopScanner.php +++ /dev/null @@ -1,82 +0,0 @@ -<?php -/** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * - */ - -namespace OC\Files\ObjectStore; - -use OC\Files\Cache\Scanner; -use OC\Files\Storage\Storage; - -class NoopScanner extends Scanner { - public function __construct(Storage $storage) { - //we don't need the storage, so do nothing here - } - - /** - * scan a single file and store it in the cache - * - * @param string $file - * @param int $reuseExisting - * @param int $parentId - * @param array|null $cacheData existing data in the cache for the file to be scanned - * @return array an array of metadata of the scanned file - */ - public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { - return []; - } - - /** - * scan a folder and all it's children - * - * @param string $path - * @param bool $recursive - * @param int $reuse - * @return array with the meta data of the scanned file or folder - */ - public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { - return []; - } - - /** - * scan all the files and folders in a folder - * - * @param string $path - * @param bool $recursive - * @param int $reuse - * @param array $folderData existing cache data for the folder to be scanned - * @return int the size of the scanned folder or -1 if the size is unknown at this stage - */ - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { - return 0; - } - - /** - * walk over any folders that are not fully scanned yet and scan them - */ - public function backgroundScan() { - //noop - } -} diff --git a/lib/private/Files/ObjectStore/ObjectStoreScanner.php b/lib/private/Files/ObjectStore/ObjectStoreScanner.php new file mode 100644 index 00000000000..5c3992b8458 --- /dev/null +++ b/lib/private/Files/ObjectStore/ObjectStoreScanner.php @@ -0,0 +1,79 @@ +<?php + +/** + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\Files\ObjectStore; + +use OC\Files\Cache\Scanner; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\FileInfo; + +class ObjectStoreScanner extends Scanner { + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { + return null; + } + + public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + return null; + } + + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + return 0; + } + + public function backgroundScan() { + $lastPath = null; + // find any path marked as unscanned and run the scanner until no more paths are unscanned (or we get stuck) + // we sort by path DESC to ensure that contents of a folder are handled before the parent folder + while (($path = $this->getIncomplete()) !== false && $path !== $lastPath) { + $this->runBackgroundScanJob(function () use ($path) { + $item = $this->cache->get($path); + if ($item && $item->getMimeType() !== FileInfo::MIMETYPE_FOLDER) { + $fh = $this->storage->fopen($path, 'r'); + if ($fh) { + $stat = fstat($fh); + if ($stat['size']) { + $this->cache->update($item->getId(), ['size' => $stat['size']]); + } + } + } + }, $path); + // FIXME: this won't proceed with the next item, needs revamping of getIncomplete() + // to make this possible + $lastPath = $path; + } + } + + /** + * Unlike the default Cache::getIncomplete this one sorts by path. + * + * This is needed since self::backgroundScan doesn't fix child entries when running on a parent folder. + * By sorting by path we ensure that we encounter the child entries first. + * + * @return false|string + * @throws \OCP\DB\Exception + */ + private function getIncomplete() { + $query = $this->connection->getQueryBuilder(); + $query->select('path') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->orderBy('path', 'DESC') + ->setMaxResults(1); + + $result = $query->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + + if ($path === false) { + return false; + } + + // Make sure Oracle does not continue with null for empty strings + return (string)$path; + } +} diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 3378f00c4dd..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -1,98 +1,83 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bjoern Schiessle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Marcel Klehr <mklehr@gmx.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tigran Mkrtchyan <tigran.mkrtchyan@desy.de> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; +use Aws\S3\Exception\S3Exception; +use Aws\S3\Exception\S3MultipartUploadException; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\CountWrapper; use Icewind\Streams\IteratorDirectory; +use OC\Files\Cache\Cache; use OC\Files\Cache\CacheEntry; use OC\Files\Storage\PolyFill\CopyDirectory; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; use OCP\Files\FileInfo; +use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; +use OCP\Files\Storage\IChunkedFileWrite; +use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; -class ObjectStoreStorage extends \OC\Files\Storage\Common { +class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; - /** - * @var \OCP\Files\ObjectStore\IObjectStore $objectStore - */ - protected $objectStore; - /** - * @var string $id - */ - protected $id; - /** - * @var \OC\User\User $user - */ - protected $user; + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; - private $objectPrefix = 'urn:oid:'; + private LoggerInterface $logger; - private $logger; + private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + private bool $preserveCacheItemsOnDelete = false; - public function __construct($params) { - if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { - $this->objectStore = $params['objectstore']; + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) { + $this->objectStore = $parameters['objectstore']; } else { throw new \Exception('missing IObjectStore instance'); } - if (isset($params['storageid'])) { - $this->id = 'object::store:' . $params['storageid']; + if (isset($parameters['storageid'])) { + $this->id = 'object::store:' . $parameters['storageid']; } else { $this->id = 'object::store:' . $this->objectStore->getStorageId(); } - if (isset($params['objectPrefix'])) { - $this->objectPrefix = $params['objectPrefix']; + if (isset($parameters['objectPrefix'])) { + $this->objectPrefix = $parameters['objectPrefix']; } - //initialize cache with root directory in cache - if (!$this->is_dir('/')) { - $this->mkdir('/'); + if (isset($parameters['validateWrites'])) { + $this->validateWrites = (bool)$parameters['validateWrites']; } + $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false); - $this->logger = \OC::$server->getLogger(); + $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir($path) { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); - - if ($this->file_exists($path)) { + if (!$force && $this->file_exists($path)) { + $this->logger->warning("Tried to create an object store folder that already exists: $path"); return false; } $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -109,10 +94,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { if ($parentType === false) { if (!$this->mkdir($parent)) { // something went wrong + $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created"); return false; } } elseif ($parentType === 'file') { // parent is a file + $this->logger->warning("Parent ($parent) is a file"); return false; } // finally create the new dir @@ -125,11 +112,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - /** - * @param string $path - * @return string - */ - private function normalizePath($path) { + private function normalizePath(string $path): string { $path = trim($path, '/'); //FIXME why do we sometimes get a path like 'files//username'? $path = str_replace('//', '/', $path); @@ -145,95 +128,108 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { /** * Object Stores use a NoopScanner because metadata is directly stored in * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. - * - * @param string $path - * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner - * @return \OC\Files\ObjectStore\NoopScanner */ - public function getScanner($path = '', $storage = null) { + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { if (!$storage) { $storage = $this; } if (!isset($this->scanner)) { - $this->scanner = new NoopScanner($storage); + $this->scanner = new ObjectStoreScanner($storage); } + /** @var \OC\Files\ObjectStore\ObjectStoreScanner */ return $this->scanner; } - public function getId() { + public function getId(): string { return $this->id; } - public function rmdir($path) { + public function rmdir(string $path): bool { $path = $this->normalizePath($path); + $entry = $this->getCache()->get($path); - if (!$this->is_dir($path)) { + if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) { return false; } - if (!$this->rmObjects($path)) { - return false; - } - - $this->getCache()->remove($path); - - return true; + return $this->rmObjects($entry); } - private function rmObjects($path) { - $children = $this->getCache()->getFolderContents($path); + private function rmObjects(ICacheEntry $entry): bool { + $children = $this->getCache()->getFolderContentsById($entry->getId()); foreach ($children as $child) { - if ($child['mimetype'] === 'httpd/unix-directory') { - if (!$this->rmObjects($child['path'])) { + if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + if (!$this->rmObjects($child)) { return false; } } else { - if (!$this->unlink($child['path'])) { + if (!$this->rmObject($child)) { return false; } } } + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + return true; } - public function unlink($path) { + public function unlink(string $path): bool { $path = $this->normalizePath($path); - $stat = $this->stat($path); + $entry = $this->getCache()->get($path); - if ($stat && isset($stat['fileid'])) { - if ($stat['mimetype'] === 'httpd/unix-directory') { - return $this->rmdir($path); + if ($entry instanceof ICacheEntry) { + if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + return $this->rmObjects($entry); + } else { + return $this->rmObject($entry); } - try { - $this->objectStore->deleteObject($this->getURN($stat['fileid'])); - } catch (\Exception $ex) { - if ($ex->getCode() !== 404) { - $this->logger->logException($ex, [ + } + return false; + } + + public function rmObject(ICacheEntry $entry): bool { + try { + $this->objectStore->deleteObject($this->getURN($entry->getId())); + } catch (\Exception $ex) { + if ($ex->getCode() !== 404) { + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ 'app' => 'objectstore', - 'message' => 'Could not delete object ' . $this->getURN($stat['fileid']) . ' for ' . $path, - ]); - return false; - } - //removing from cache is ok as it does not exist in the objectstore anyway + 'exception' => $ex, + ] + ); + return false; } - $this->getCache()->remove($path); - return true; + //removing from cache is ok as it does not exist in the objectstore anyway } - return false; + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + return true; } - public function stat($path) { + public function stat(string $path): array|false { $path = $this->normalizePath($path); $cacheEntry = $this->getCache()->get($path); if ($cacheEntry instanceof CacheEntry) { return $cacheEntry->getData(); } else { + if ($path === '') { + $this->mkdir('', true); + $cacheEntry = $this->getCache()->get($path); + if ($cacheEntry instanceof CacheEntry) { + return $cacheEntry->getData(); + } + } return false; } } - public function getPermissions($path) { + public function getPermissions(string $path): int { $stat = $this->stat($path); if (is_array($stat) && isset($stat['permissions'])) { @@ -248,17 +244,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. * You may need a mapping table to store your URN if it cannot be generated from the fileid. * - * @param int $fileId the fileid - * @return null|string the unified resource name used to identify the object + * @return string the unified resource name used to identify the object */ - public function getURN($fileId) { - if (is_numeric($fileId)) { - return $this->objectPrefix . $fileId; - } - return null; + public function getURN(int $fileId): string { + return $this->objectPrefix . $fileId; } - public function opendir($path) { + public function opendir(string $path) { $path = $this->normalizePath($path); try { @@ -270,12 +262,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return IteratorDirectory::wrap($files); } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); return false; } } - public function filetype($path) { + public function filetype(string $path): string|false { $path = $this->normalizePath($path); $stat = $this->stat($path); if ($stat) { @@ -288,7 +280,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - public function fopen($path, $mode) { + public function fopen(string $path, string $mode) { $path = $this->normalizePath($path); if (strrpos($path, '.') !== false) { @@ -302,38 +294,61 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { case 'rb': $stat = $this->stat($path); if (is_array($stat)) { + $filesize = $stat['size'] ?? 0; // Reading 0 sized files is a waste of time - if (isset($stat['size']) && $stat['size'] === 0) { + if ($filesize === 0) { return fopen('php://memory', $mode); } try { - return $this->objectStore->readObject($this->getURN($stat['fileid'])); + $handle = $this->objectStore->readObject($this->getURN($stat['fileid'])); + if ($handle === false) { + return false; // keep backward compatibility + } + $streamStat = fstat($handle); + $actualSize = $streamStat['size'] ?? -1; + if ($actualSize > -1 && $actualSize !== $filesize) { + $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]); + } + return $handle; } catch (NotFoundException $e) { - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw $e; - } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); return false; } } else { return false; } - // no break + // no break case 'w': case 'wb': case 'w+': case 'wb+': + $dirName = dirname($path); + $parentExists = $this->is_dir($dirName); + if (!$parentExists) { + return false; + } + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext); $handle = fopen($tmpFile, $mode); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); + unlink($tmpFile); }); case 'a': case 'ab': @@ -351,17 +366,18 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $handle = fopen($tmpFile, $mode); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); + unlink($tmpFile); }); } return false; } - public function file_exists($path) { + public function file_exists(string $path): bool { $path = $this->normalizePath($path); return (bool)$this->stat($path); } - public function rename($source, $target) { + public function rename(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); $this->remove($target); @@ -370,12 +386,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return true; } - public function getMimeType($path) { + public function getMimeType(string $path): string|false { $path = $this->normalizePath($path); return parent::getMimeType($path); } - public function touch($path, $mtime = null) { + public function touch(string $path, ?int $mtime = null): bool { if (is_null($mtime)) { $mtime = time(); } @@ -397,55 +413,48 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object for ' . $path, - ]); + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); throw $ex; } } return true; } - public function writeBack($tmpFile, $path) { + public function writeBack(string $tmpFile, string $path) { $size = filesize($tmpFile); $this->writeStream($path, fopen($tmpFile, 'r'), $size); } - /** - * external changes are not supported, exclusive access to the object storage is assumed - * - * @param string $path - * @param int $time - * @return false - */ - public function hasUpdated($path, $time) { + public function hasUpdated(string $path, int $time): bool { return false; } - public function needsPartFile() { + public function needsPartFile(): bool { return false; } - public function file_put_contents($path, $data) { - $handle = $this->fopen($path, 'w+'); - $result = fwrite($handle, $data); - fclose($handle); - return $result; + public function file_put_contents(string $path, mixed $data): int { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $data); + rewind($fh); + return $this->writeStream($path, $fh, strlen($data)); } - public function writeStream(string $path, $stream, int $size = null): int { + public function writeStream(string $path, $stream, ?int $size = null): int { + if ($size === null) { + $stats = fstat($stream); + if (is_array($stats) && isset($stats['size'])) { + $size = $stats['size']; + } + } + $stat = $this->stat($path); if (empty($stat)) { // create new file @@ -461,9 +470,18 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); + $stat['checksum'] = ''; $exists = $this->getCache()->inCache($path); $uploadPath = $exists ? $path : $path . '.part'; @@ -471,27 +489,37 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { if ($exists) { $fileId = $stat['fileid']; } else { + $parent = $this->normalizePath(dirname($path)); + if (!$this->is_dir($parent)) { + throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)"); + } $fileId = $this->getCache()->put($uploadPath, $stat); } $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream); + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -499,23 +527,31 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { * Else people lose access to existing files */ $this->getCache()->remove($uploadPath); - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } else { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not update object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { + // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways + $stat['unencrypted_size'] = $stat['size']; $this->getCache()->update($fileId, $stat); } else { - if ($this->objectStore->objectExists($urn)) { + if (!$this->validateWrites || $this->objectStore->objectExists($urn)) { $this->getCache()->move($uploadPath, $path); } else { $this->getCache()->remove($uploadPath); @@ -523,39 +559,148 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { return $this->objectStore; } - public function copy($path1, $path2) { - $path1 = $this->normalizePath($path1); - $path2 = $this->normalizePath($path2); + public function copyFromStorage( + IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime = false, + ): bool { + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { + /** @var CacheEntry $sourceEntry */ + $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath); + $sourceEntryData = $sourceEntry->getData(); + // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current + // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the + // unjailed storage. + if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) { + $sourceEntry['permissions'] = $sourceEntryData['scan_permissions']; + } + $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath); + return true; + } + } + + return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { + $sourceCache = $sourceStorage->getCache(); + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + // Do not import any data when source and target bucket are identical. + return true; + } + if (!$sourceCacheEntry) { + $sourceCacheEntry = $sourceCache->get($sourceInternalPath); + } + + $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry); + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $sourceStorage->rmdir($sourceInternalPath); + } else { + $sourceStorage->unlink($sourceInternalPath); + } + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + + return true; + } + + /** + * Copy the object(s) of a file or folder into this storage, without touching the cache + */ + private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) { + $copiedFiles = []; + try { + foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) { + $sourceStream = $sourceStorage->fopen($file->getPath(), 'r'); + if (!$sourceStream) { + throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})"); + } + $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType()); + if (is_resource($sourceStream)) { + fclose($sourceStream); + } + $copiedFiles[] = $file->getId(); + } + } catch (\Exception $e) { + foreach ($copiedFiles as $fileId) { + try { + $this->objectStore->deleteObject($this->getURN($fileId)); + } catch (\Exception $e) { + // ignore + } + } + throw $e; + } + } + + /** + * @return \Iterator<ICacheEntry> + */ + private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + foreach ($cache->getFolderContentsById($entry->getId()) as $child) { + yield from $this->getAllChildObjects($cache, $child); + } + } else { + yield $entry; + } + } + + public function copy(string $source, string $target): bool { + $source = $this->normalizePath($source); + $target = $this->normalizePath($target); $cache = $this->getCache(); - $sourceEntry = $cache->get($path1); + $sourceEntry = $cache->get($source); if (!$sourceEntry) { throw new NotFoundException('Source object not found'); } - $this->copyInner($sourceEntry, $path2); + $this->copyInner($cache, $sourceEntry, $target); return true; } - private function copyInner(ICacheEntry $sourceEntry, string $to) { + private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) { $cache = $this->getCache(); if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); - foreach ($cache->getFolderContentsById($sourceEntry->getId()) as $child) { - $this->copyInner($child, $to . '/' . $child->getName()); + foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { + $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); } } else { $this->copyFile($sourceEntry, $to); @@ -567,21 +712,104 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $sourceUrn = $this->getURN($sourceEntry->getId()); - $cache->copyFromCache($cache, $sourceEntry, $to); - $targetEntry = $cache->get($to); - - if (!$targetEntry) { - throw new \Exception('Target not in cache after copy'); + if (!$cache instanceof Cache) { + throw new \Exception('Invalid source cache for object store copy'); } - $targetUrn = $this->getURN($targetEntry->getId()); + $targetId = $cache->copyFromCache($cache, $sourceEntry, $to); + + $targetUrn = $this->getURN($targetId); try { $this->objectStore->copyObject($sourceUrn, $targetUrn); + if ($this->handleCopiesAsOwned) { + // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage ! + $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]); + } } catch (\Exception $e) { $cache->remove($to); throw $e; } } + + public function startChunkedWrite(string $targetPath): string { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + return $this->objectStore->initiateMultipartUpload($urn); + } + + /** + * @throws GenericFileException + */ + public function putChunkedWritePart( + string $targetPath, + string $writeToken, + string $chunkId, + $data, + $size = null, + ): ?array { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + + $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size); + + $parts[$chunkId] = [ + 'PartNumber' => $chunkId, + 'ETag' => trim($result->get('ETag'), '"'), + ]; + return $parts[$chunkId]; + } + + public function completeChunkedWrite(string $targetPath, string $writeToken): int { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $parts = $this->objectStore->getMultipartUploads($urn, $writeToken); + $sortedParts = array_values($parts); + sort($sortedParts); + try { + $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts); + $stat = $this->stat($targetPath); + $mtime = time(); + if (is_array($stat)) { + $stat['size'] = $size; + $stat['mtime'] = $mtime; + $stat['mimetype'] = $this->getMimeType($targetPath); + $this->getCache()->update($stat['fileid'], $stat); + } + } catch (S3MultipartUploadException|S3Exception $e) { + $this->objectStore->abortMultipartUpload($urn, $writeToken); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); + throw new GenericFileException('Could not write chunked file'); + } + return $size; + } + + public function cancelChunkedWrite(string $targetPath, string $writeToken): void { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $this->objectStore->abortMultipartUpload($urn, $writeToken); + } + + public function setPreserveCacheOnDelete(bool $preserve) { + $this->preserveCacheItemsOnDelete = $preserve; + } } diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php new file mode 100644 index 00000000000..ffc33687340 --- /dev/null +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -0,0 +1,225 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\ObjectStore; + +use OCP\App\IAppManager; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +use OCP\IUser; + +/** + * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}} + */ +class PrimaryObjectStoreConfig { + public function __construct( + private readonly IConfig $config, + private readonly IAppManager $appManager, + ) { + } + + /** + * @param ObjectStoreConfig $config + */ + public function buildObjectStore(array $config): IObjectStore { + return new $config['class']($config['arguments']); + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForRoot(): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $config = $this->getObjectStoreConfiguration('root'); + + if ($config['arguments']['multibucket']) { + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + // put the root FS always in first bucket for multibucket configuration + $config['arguments']['bucket'] .= '0'; + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForUser(IUser $user): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $store = $this->getObjectStoreForUser($user); + $config = $this->getObjectStoreConfiguration($store); + + if ($config['arguments']['multibucket']) { + $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); + } + return $config; + } + + /** + * @param string $name + * @return ObjectStoreConfig + */ + public function getObjectStoreConfiguration(string $name): array { + $configs = $this->getObjectStoreConfigs(); + $name = $this->resolveAlias($name); + if (!isset($configs[$name])) { + throw new \Exception("Object store configuration for '$name' not found"); + } + if (is_string($configs[$name])) { + throw new \Exception("Object store configuration for '{$configs[$name]}' not found"); + } + return $configs[$name]; + } + + public function resolveAlias(string $name): string { + $configs = $this->getObjectStoreConfigs(); + + while (isset($configs[$name]) && is_string($configs[$name])) { + $name = $configs[$name]; + } + return $name; + } + + public function hasObjectStore(): bool { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + return $objectStore || $objectStoreMultiBucket; + } + + public function hasMultipleObjectStorages(): bool { + $objectStore = $this->config->getSystemValue('objectstore', []); + return isset($objectStore['default']); + } + + /** + * @return ?array<string, ObjectStoreConfig|string> + * @throws InvalidObjectStoreConfigurationException + */ + public function getObjectStoreConfigs(): ?array { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + + // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config + if ($objectStoreMultiBucket) { + $objectStoreMultiBucket['arguments']['multibucket'] = true; + return [ + 'default' => 'server1', + 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), + 'root' => 'server1', + ]; + } elseif ($objectStore) { + if (!isset($objectStore['default'])) { + $objectStore = [ + 'default' => 'server1', + 'root' => 'server1', + 'server1' => $objectStore, + ]; + } + if (!isset($objectStore['root'])) { + $objectStore['root'] = 'default'; + } + + if (!is_string($objectStore['default'])) { + throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); + } + return array_map($this->validateObjectStoreConfig(...), $objectStore); + } else { + return null; + } + } + + /** + * @param array|string $config + * @return string|ObjectStoreConfig + */ + private function validateObjectStoreConfig(array|string $config): array|string { + if (is_string($config)) { + return $config; + } + if (!isset($config['class'])) { + throw new InvalidObjectStoreConfigurationException('No class configured for object store'); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $class = $config['class']; + $arguments = $config['arguments']; + if (!is_array($arguments)) { + throw new InvalidObjectStoreConfigurationException('Configured object store arguments are not an array'); + } + if (!isset($arguments['multibucket'])) { + $arguments['multibucket'] = false; + } + if (!is_bool($arguments['multibucket'])) { + throw new InvalidObjectStoreConfigurationException('arguments.multibucket must be a boolean in object store configuration'); + } + + if (!is_string($class)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not a string'); + } + + if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { + [$appId] = explode('\\', $class); + $this->appManager->loadApp(strtolower($appId)); + } + + if (!is_a($class, IObjectStore::class, true)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not an object store'); + } + return [ + 'class' => $class, + 'arguments' => $arguments, + ]; + } + + public function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->getSetBucketForUser($user); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new Mapper($user, $this->config); + $numBuckets = $config['arguments']['num_buckets'] ?? 64; + $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); + } + + return $bucket; + } + + public function getSetBucketForUser(IUser $user): ?string { + return $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + } + + public function getObjectStoreForUser(IUser $user): string { + if ($this->hasMultipleObjectStorages()) { + $value = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null); + if ($value === null) { + $value = $this->resolveAlias('default'); + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $value); + } + return $value; + } else { + return 'default'; + } + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 3d1a658eb9f..72e1751e23d 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -1,36 +1,24 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; +use Aws\Result; +use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; - public function __construct($parameters) { + public function __construct(array $parameters) { + $parameters['primary_storage'] = true; $this->parseParams($parameters); } @@ -41,4 +29,114 @@ class S3 implements IObjectStore { public function getStorageId() { return $this->id; } + + public function initiateMultipartUpload(string $urn): string { + $upload = $this->getConnection()->createMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ] + $this->getSSECParameters()); + $uploadId = $upload->get('UploadId'); + if ($uploadId === null) { + throw new Exception('No upload id returned'); + } + return (string)$uploadId; + } + + public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result { + return $this->getConnection()->uploadPart([ + 'Body' => $stream, + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'ContentLength' => $size, + 'PartNumber' => $partId, + 'UploadId' => $uploadId, + ] + $this->getSSECParameters()); + } + + public function getMultipartUploads(string $urn, string $uploadId): array { + $parts = []; + $isTruncated = true; + $partNumberMarker = 0; + + while ($isTruncated) { + $result = $this->getConnection()->listParts([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + 'MaxParts' => 1000, + 'PartNumberMarker' => $partNumberMarker, + ] + $this->getSSECParameters()); + $parts = array_merge($parts, $result->get('Parts') ?? []); + $isTruncated = $result->get('IsTruncated'); + $partNumberMarker = $result->get('NextPartNumberMarker'); + } + + return $parts; + } + + public function completeMultipartUpload(string $urn, string $uploadId, array $result): int { + $this->getConnection()->completeMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + 'MultipartUpload' => ['Parts' => $result], + ] + $this->getSSECParameters()); + $stat = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ] + $this->getSSECParameters()); + return (int)$stat->get('ContentLength'); + } + + public function abortMultipartUpload($urn, $uploadId): void { + $this->getConnection()->abortMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + ]); + } + + private function parseS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + if (str_starts_with($key, 'x-amz-meta-')) { + $result[substr($key, strlen('x-amz-meta-'))] = $value; + } + } + return $result; + } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ] + $this->parseS3Metadata($object['Metadata'] ?? []); + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } } diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php new file mode 100644 index 00000000000..5b086db8f77 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -0,0 +1,41 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +/** + * Shared configuration between ConnectionTrait and ObjectTrait to ensure both to be in sync + */ +trait S3ConfigTrait { + protected array $params; + + protected string $bucket; + + /** Maximum number of concurrent multipart uploads */ + protected int $concurrency; + + /** Timeout, in seconds, for the connection to S3 server, not for the + * request. */ + protected float $connectTimeout; + + protected int $timeout; + + protected string|false $proxy; + + protected string $storageClass; + + /** @var int Part size in bytes (float is added for 32bit support) */ + protected int|float $uploadPartSize; + + /** @var int Limit on PUT in bytes (float is added for 32bit support) */ + private int|float $putSizeLimit; + + /** @var int Limit on COPY in bytes (float is added for 32bit support) */ + private int|float $copySizeLimit; + + private bool $useMultipartCopy = true; +} diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index d88ef0ac8e7..67b82a44ab7 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -1,84 +1,64 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Florent <florent@coppint.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com> - * @author Stephen Cuppett <steve@cuppett.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; use Aws\ClientResolver; use Aws\Credentials\CredentialProvider; -use Aws\Credentials\EcsCredentialProvider; use Aws\Credentials\Credentials; use Aws\Exception\CredentialsException; use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; -use GuzzleHttp\Promise; +use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; -use OCP\ILogger; +use OCP\Files\StorageNotAvailableException; +use OCP\ICertificateManager; +use OCP\Server; +use Psr\Log\LoggerInterface; trait S3ConnectionTrait { - /** @var array */ - protected $params; - - /** @var S3Client */ - protected $connection; - - /** @var string */ - protected $id; + use S3ConfigTrait; - /** @var string */ - protected $bucket; + protected string $id; - /** @var int */ - protected $timeout; + protected bool $test; - /** @var int */ - protected $uploadPartSize; - - protected $test; + protected ?S3Client $connection = null; protected function parseParams($params) { if (empty($params['bucket'])) { - throw new \Exception("Bucket has to be configured."); + throw new \Exception('Bucket has to be configured.'); } $this->id = 'amazon::' . $params['bucket']; $this->test = isset($params['test']); $this->bucket = $params['bucket']; - $this->timeout = !isset($params['timeout']) ? 15 : $params['timeout']; - $this->uploadPartSize = !isset($params['uploadPartSize']) ? 524288000 : $params['uploadPartSize']; + // Default to 5 like the S3 SDK does + $this->concurrency = $params['concurrency'] ?? 5; + $this->proxy = $params['proxy'] ?? false; + $this->connectTimeout = $params['connect_timeout'] ?? 5; + $this->timeout = $params['timeout'] ?? 15; + $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; + $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; + $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600; + $this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000; + $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true); $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; + $params['s3-accelerate'] = $params['hostname'] === 's3-accelerate.amazonaws.com' || $params['hostname'] === 's3-accelerate.dualstack.amazonaws.com'; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } - $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; + $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true; + + if ($params['s3-accelerate']) { + $params['verify_bucket_exists'] = false; + } + $this->params = $params; } @@ -86,6 +66,10 @@ trait S3ConnectionTrait { return $this->bucket; } + public function getProxy() { + return $this->proxy; + } + /** * Returns the connection * @@ -93,7 +77,7 @@ trait S3ConnectionTrait { * @throws \Exception if connection could not be made */ public function getConnection() { - if (!is_null($this->connection)) { + if ($this->connection !== null) { return $this->connection; } @@ -101,63 +85,80 @@ trait S3ConnectionTrait { $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; // Adding explicit credential provider to the beginning chain. - // Including environment variables and IAM instance profiles. + // Including default credential provider (skipping AWS shared config files). $provider = CredentialProvider::memoize( CredentialProvider::chain( $this->paramCredentialProvider(), - CredentialProvider::env(), - CredentialProvider::assumeRoleWithWebIdentityCredentialProvider(), - !empty(getenv(EcsCredentialProvider::ENV_URI)) - ? CredentialProvider::ecsCredentials() - : CredentialProvider::instanceProfile() + CredentialProvider::defaultProvider(['use_aws_shared_config_files' => false]) ) ); $options = [ - 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest', + 'version' => $this->params['version'] ?? 'latest', 'credentials' => $provider, 'endpoint' => $base_url, 'region' => $this->params['region'], 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()), 'csm' => false, + 'use_arn_region' => false, + 'http' => [ + 'verify' => $this->getCertificateBundlePath(), + 'connect_timeout' => $this->connectTimeout, + ], + 'use_aws_shared_config_files' => false, + 'retries' => [ + 'mode' => 'standard', + 'max_attempts' => 5, + ], ]; - if (isset($this->params['proxy'])) { - $options['request.options'] = ['proxy' => $this->params['proxy']]; + + if ($this->params['s3-accelerate']) { + $options['use_accelerate_endpoint'] = true; + } else { + $options['endpoint'] = $base_url; + } + + if ($this->getProxy()) { + $options['http']['proxy'] = $this->getProxy(); } if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) { $options['signature_version'] = 'v2'; } $this->connection = new S3Client($options); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - $logger = \OC::$server->getLogger(); - $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', - ['app' => 'objectstore']); - } + try { + $logger = Server::get(LoggerInterface::class); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', + ['app' => 'objectstore']); + } - if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { - $logger = \OC::$server->getLogger(); - try { - $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); + if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { + try { + $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket); + } + $this->connection->createBucket(['Bucket' => $this->bucket]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->debug('Invalid remote storage.', [ + 'exception' => $e, + 'app' => 'objectstore', + ]); + if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') { + throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); + } } - $this->connection->createBucket(['Bucket' => $this->bucket]); - $this->testTimeout(); - } catch (S3Exception $e) { - $logger->logException($e, [ - 'message' => 'Invalid remote storage.', - 'level' => ILogger::DEBUG, - 'app' => 'objectstore', - ]); - throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); } - } - // google cloud's s3 compatibility doesn't like the EncodingType parameter - if (strpos($base_url, 'storage.googleapis.com')) { - $this->connection->getHandlerList()->remove('s3.auto_encode'); + // google cloud's s3 compatibility doesn't like the EncodingType parameter + if (strpos($base_url, 'storage.googleapis.com')) { + $this->connection->getHandlerList()->remove('s3.auto_encode'); + } + } catch (S3Exception $e) { + throw new StorageNotAvailableException('S3 service is unable to handle request: ' . $e->getMessage()); } return $this->connection; @@ -185,14 +186,16 @@ trait S3ConnectionTrait { /** * This function creates a credential provider based on user parameter file */ - protected function paramCredentialProvider() : callable { + protected function paramCredentialProvider(): callable { return function () { $key = empty($this->params['key']) ? null : $this->params['key']; $secret = empty($this->params['secret']) ? null : $this->params['secret']; + $sessionToken = empty($this->params['session_token']) ? null : $this->params['session_token']; if ($key && $secret) { - return Promise\promise_for( - new Credentials($key, $secret) + return Create::promiseFor( + // a null sessionToken match the default signature of the constructor + new Credentials($key, $secret, $sessionToken) ); } @@ -200,4 +203,49 @@ trait S3ConnectionTrait { return new RejectedPromise(new CredentialsException($msg)); }; } + + protected function getCertificateBundlePath(): ?string { + if ((int)($this->params['use_nextcloud_bundle'] ?? '0')) { + // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage + if (!isset($this->params['primary_storage'])) { + /** @var ICertificateManager $certManager */ + $certManager = Server::get(ICertificateManager::class); + return $certManager->getAbsoluteBundlePath(); + } else { + return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt'; + } + } else { + return null; + } + } + + protected function getSSECKey(): ?string { + if (isset($this->params['sse_c_key']) && !empty($this->params['sse_c_key'])) { + return $this->params['sse_c_key']; + } + + return null; + } + + protected function getSSECParameters(bool $copy = false): array { + $key = $this->getSSECKey(); + + if ($key === null) { + return []; + } + + $rawKey = base64_decode($key); + if ($copy) { + return [ + 'CopySourceSSECustomerAlgorithm' => 'AES256', + 'CopySourceSSECustomerKey' => $rawKey, + 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true) + ]; + } + return [ + 'SSECustomerAlgorithm' => 'AES256', + 'SSECustomerKey' => $rawKey, + 'SSECustomerKeyMD5' => md5($rawKey, true) + ]; + } } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 80b8a6f132d..89405de2e8e 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -1,40 +1,25 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Florent <florent@coppint.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; +use Aws\Command; +use Aws\Exception\MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException; +use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; -use Aws\S3\ObjectUploader; use Aws\S3\S3Client; -use Icewind\Streams\CallbackWrapper; +use GuzzleHttp\Psr7; +use GuzzleHttp\Psr7\Utils; use OC\Files\Stream\SeekableHttpStream; +use Psr\Http\Message\StreamInterface; trait S3ObjectTrait { + use S3ConfigTrait; + /** * Returns the connection * @@ -43,19 +28,23 @@ trait S3ObjectTrait { */ abstract protected function getConnection(); + abstract protected function getCertificateBundlePath(): ?string; + abstract protected function getSSECParameters(bool $copy = false): array; + /** * @param string $urn the unified resource name used to identify the object + * * @return resource stream with the read data * @throws \Exception when something goes wrong, message will be logged * @since 7.0.0 */ public function readObject($urn) { - return SeekableHttpStream::open(function ($range) use ($urn) { + $fh = SeekableHttpStream::open(function ($range) use ($urn) { $command = $this->getConnection()->getCommand('GetObject', [ 'Bucket' => $this->bucket, 'Key' => $urn, 'Range' => 'bytes=' . $range, - ]); + ] + $this->getSSECParameters()); $request = \Aws\serialize($command); $headers = []; foreach ($request->getHeaders() as $key => $values) { @@ -65,47 +54,185 @@ trait S3ObjectTrait { } $opts = [ 'http' => [ - 'protocol_version' => 1.1, + 'protocol_version' => $request->getProtocolVersion(), 'header' => $headers, - ], + ] ]; + $bundle = $this->getCertificateBundlePath(); + if ($bundle) { + $opts['ssl'] = [ + 'cafile' => $bundle + ]; + } + + if ($this->getProxy()) { + $opts['http']['proxy'] = $this->getProxy(); + $opts['http']['request_fulluri'] = true; + } $context = stream_context_create($opts); return fopen($request->getUri(), 'r', false, $context); }); + if (!$fh) { + throw new \Exception("Failed to read object $urn"); + } + return $fh; + } + + private function buildS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + $result['x-amz-meta-' . $key] = $value; + } + return $result; } /** + * Single object put helper + * * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write + * @param StreamInterface $stream stream with the data to write + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 */ - public function writeObject($urn, $stream) { - $count = 0; - $countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) { - $count += $read; - }); + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $args = [ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream, + 'ACL' => 'private', + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); + } - $uploader = new MultipartUploader($this->getConnection(), $countStream, [ - 'bucket' => $this->bucket, - 'key' => $urn, - 'part_size' => $this->uploadPartSize, - ]); - try { - $uploader->upload(); - } catch (S3MultipartUploadException $e) { - // This is an empty file so just touch it then - if ($count === 0 && feof($countStream)) { - $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, ''); + /** + * Multipart upload helper that tries to avoid orphaned fragments in S3 + * + * @param string $urn the unified resource name used to identify the object + * @param StreamInterface $stream stream with the data to write + * @param array $metaData the metadata to set for the object + * @throws \Exception when something goes wrong, message will be logged + */ + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $attempts = 0; + $uploaded = false; + $concurrency = $this->concurrency; + $exception = null; + $state = null; + $size = $stream->getSize(); + $totalWritten = 0; + + // retry multipart upload once with concurrency at half on failure + while (!$uploaded && $attempts <= 1) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'concurrency' => $concurrency, + 'key' => $urn, + 'part_size' => $this->uploadPartSize, + 'state' => $state, + 'params' => [ + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, + ]); + + try { $uploader->upload(); - } else { - throw $e; + $uploaded = true; + } catch (S3MultipartUploadException $e) { + $exception = $e; + $attempts++; + + if ($concurrency > 1) { + $concurrency = round($concurrency / 2); + } + + if ($stream->isSeekable()) { + $stream->rewind(); + } + } catch (MultipartUploadException $e) { + $exception = $e; + break; } } - fclose($countStream); + if (!$uploaded) { + // if anything goes wrong with multipart, make sure that you don´t poison and + // slow down s3 bucket with orphaned fragments + $uploadInfo = $exception->getState()->getId(); + if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { + $this->getConnection()->abortMultipartUpload($uploadInfo); + } + + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception); + } + } + + public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } + + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { + $canSeek = fseek($stream, 0, SEEK_CUR) === 0; + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); + + + $size = $psrStream->getSize(); + if ($size === null || !$canSeek) { + // The s3 single-part upload requires the size to be known for the stream. + // So for input streams that don't have a known size, we need to copy (part of) + // the input into a temporary stream so the size can be determined + $buffer = new Psr7\Stream(fopen('php://temp', 'rw+')); + Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); + $buffer->seek(0); + if ($buffer->getSize() < $this->putSizeLimit) { + // buffer is fully seekable, so use it directly for the small upload + $this->writeSingle($urn, $buffer, $metaData); + } else { + $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); + $this->writeMultiPart($urn, $loadStream, $metaData); + } + } else { + if ($size < $this->putSizeLimit) { + $this->writeSingle($urn, $psrStream, $metaData); + } else { + $this->writeMultiPart($urn, $psrStream, $metaData); + } + } + $psrStream->close(); } /** @@ -122,10 +249,34 @@ trait S3ObjectTrait { } public function objectExists($urn) { - return $this->getConnection()->doesObjectExist($this->bucket, $urn); + return $this->getConnection()->doesObjectExist($this->bucket, $urn, $this->getSSECParameters()); } - public function copyObject($from, $to) { - $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to); + public function copyObject($from, $to, array $options = []) { + $sourceMetadata = $this->getConnection()->headObject([ + 'Bucket' => $this->getBucket(), + 'Key' => $from, + ] + $this->getSSECParameters()); + + $size = (int)($sourceMetadata->get('Size') ?? $sourceMetadata->get('ContentLength')); + + if ($this->useMultipartCopy && $size > $this->copySizeLimit) { + $copy = new MultipartCopy($this->getConnection(), [ + 'source_bucket' => $this->getBucket(), + 'source_key' => $from + ], array_merge([ + 'bucket' => $this->getBucket(), + 'key' => $to, + 'acl' => 'private', + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'source_metadata' => $sourceMetadata + ], $options)); + $copy->copy(); + } else { + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([ + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'mup_threshold' => PHP_INT_MAX, + ], $options)); + } } } diff --git a/lib/private/Files/ObjectStore/S3Signature.php b/lib/private/Files/ObjectStore/S3Signature.php index ab8854849fa..b80382ff67d 100644 --- a/lib/private/Files/ObjectStore/S3Signature.php +++ b/lib/private/Files/ObjectStore/S3Signature.php @@ -1,26 +1,8 @@ <?php + /** - * - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; @@ -60,7 +42,7 @@ class S3Signature implements SignatureInterface { public function signRequest( RequestInterface $request, - CredentialsInterface $credentials + CredentialsInterface $credentials, ) { $request = $this->prepareRequest($request, $credentials); $stringToSign = $this->createCanonicalizedString($request); @@ -75,7 +57,7 @@ class S3Signature implements SignatureInterface { RequestInterface $request, CredentialsInterface $credentials, $expires, - array $options = [] + array $options = [], ) { $query = []; // URL encoding already occurs in the URI template expansion. Undo that @@ -107,29 +89,29 @@ class S3Signature implements SignatureInterface { // Move X-Amz-* headers to the query string foreach ($request->getHeaders() as $name => $header) { $name = strtolower($name); - if (strpos($name, 'x-amz-') === 0) { + if (str_starts_with($name, 'x-amz-')) { $query[$name] = implode(',', $header); } } - $queryString = http_build_query($query, null, '&', PHP_QUERY_RFC3986); + $queryString = http_build_query($query, '', '&', PHP_QUERY_RFC3986); return $request->withUri($request->getUri()->withQuery($queryString)); } /** - * @param RequestInterface $request + * @param RequestInterface $request * @param CredentialsInterface $creds * * @return RequestInterface */ private function prepareRequest( RequestInterface $request, - CredentialsInterface $creds + CredentialsInterface $creds, ) { $modify = [ 'remove_headers' => ['X-Amz-Date'], - 'set_headers' => ['Date' => gmdate(\DateTime::RFC2822)] + 'set_headers' => ['Date' => gmdate(\DateTimeInterface::RFC2822)] ]; // Add the security token header if one is being used by the credentials @@ -137,7 +119,7 @@ class S3Signature implements SignatureInterface { $modify['set_headers']['X-Amz-Security-Token'] = $token; } - return Psr7\modify_request($request, $modify); + return Psr7\Utils::modifyRequest($request, $modify); } private function signString($string, CredentialsInterface $credentials) { @@ -148,7 +130,7 @@ class S3Signature implements SignatureInterface { private function createCanonicalizedString( RequestInterface $request, - $expires = null + $expires = null, ) { $buffer = $request->getMethod() . "\n"; @@ -169,7 +151,7 @@ class S3Signature implements SignatureInterface { $headers = []; foreach ($request->getHeaders() as $name => $header) { $name = strtolower($name); - if (strpos($name, 'x-amz-') === 0) { + if (str_starts_with($name, 'x-amz-')) { $value = implode(',', $header); if (strlen($value) > 0) { $headers[$name] = $name . ':' . $value; @@ -201,7 +183,7 @@ class S3Signature implements SignatureInterface { $query = $request->getUri()->getQuery(); if ($query) { - $params = Psr7\parse_query($query); + $params = Psr7\Query::parse($query); $first = true; foreach ($this->signableQueryString as $key) { if (array_key_exists($key, $params)) { diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php index 2076bb3f88b..888602a62e4 100644 --- a/lib/private/Files/ObjectStore/StorageObjectStore.php +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; use OCP\Files\ObjectStore\IObjectStore; @@ -46,8 +28,8 @@ class StorageObjectStore implements IObjectStore { * @return string the container or bucket name where objects are stored * @since 7.0.0 */ - public function getStorageId() { - $this->storage->getId(); + public function getStorageId(): string { + return $this->storage->getId(); } /** @@ -65,13 +47,7 @@ class StorageObjectStore implements IObjectStore { throw new \Exception(); } - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ - public function writeObject($urn, $stream) { + public function writeObject($urn, $stream, ?string $mimetype = null) { $handle = $this->storage->fopen($urn, 'w'); if ($handle) { stream_copy_to_stream($stream, $handle); diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index 1b0888b0700..aa8b3bb34ec 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -1,38 +1,20 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Adrian Brzezinski <adrian.brzezinski@eo.pl> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; use GuzzleHttp\Client; use GuzzleHttp\Exception\BadResponseException; -use function GuzzleHttp\Psr7\stream_for; +use GuzzleHttp\Psr7\Utils; use Icewind\Streams\RetryWrapper; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; use OCP\Files\StorageAuthException; +use Psr\Log\LoggerInterface; const SWIFT_SEGMENT_SIZE = 1073741824; // 1GB @@ -45,11 +27,11 @@ class Swift implements IObjectStore { /** @var SwiftFactory */ private $swiftFactory; - public function __construct($params, SwiftFactory $connectionFactory = null) { + public function __construct($params, ?SwiftFactory $connectionFactory = null) { $this->swiftFactory = $connectionFactory ?: new SwiftFactory( \OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params, - \OC::$server->getLogger() + \OC::$server->get(LoggerInterface::class) ); $this->params = $params; } @@ -74,12 +56,7 @@ class Swift implements IObjectStore { return $this->params['container']; } - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @throws \Exception from openstack lib when something goes wrong - */ - public function writeObject($urn, $stream) { + public function writeObject($urn, $stream, ?string $mimetype = null) { $tmpFile = \OC::$server->getTempManager()->getTemporaryFile('swiftwrite'); file_put_contents($tmpFile, $stream); $handle = fopen($tmpFile, 'rb'); @@ -87,13 +64,15 @@ class Swift implements IObjectStore { if (filesize($tmpFile) < SWIFT_SEGMENT_SIZE) { $this->getContainer()->createObject([ 'name' => $urn, - 'stream' => stream_for($handle), + 'stream' => Utils::streamFor($handle), + 'contentType' => $mimetype, ]); } else { $this->getContainer()->createLargeObject([ 'name' => $urn, - 'stream' => stream_for($handle), + 'stream' => Utils::streamFor($handle), 'segmentSize' => SWIFT_SEGMENT_SIZE, + 'contentType' => $mimetype, ]); } } diff --git a/lib/private/Files/ObjectStore/SwiftFactory.php b/lib/private/Files/ObjectStore/SwiftFactory.php index 54975e8d021..118724159e5 100644 --- a/lib/private/Files/ObjectStore/SwiftFactory.php +++ b/lib/private/Files/ObjectStore/SwiftFactory.php @@ -3,34 +3,9 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Adrian Brzezinski <adrian.brzezinski@eo.pl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Julien Lutran <julien.lutran@corp.ovh.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Volker <skydiablo@gmx.net> - * @author William Pain <pain.william@gmail.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; use GuzzleHttp\Client; @@ -41,7 +16,6 @@ use GuzzleHttp\HandlerStack; use OCP\Files\StorageAuthException; use OCP\Files\StorageNotAvailableException; use OCP\ICache; -use OCP\ILogger; use OpenStack\Common\Auth\Token; use OpenStack\Common\Error\BadResponseError; use OpenStack\Common\Transport\Utils as TransportUtils; @@ -51,13 +25,14 @@ use OpenStack\Identity\v3\Service as IdentityV3Service; use OpenStack\ObjectStore\v1\Models\Container; use OpenStack\OpenStack; use Psr\Http\Message\RequestInterface; +use Psr\Log\LoggerInterface; class SwiftFactory { private $cache; private $params; /** @var Container|null */ private $container = null; - private $logger; + private LoggerInterface $logger; public const DEFAULT_OPTIONS = [ 'autocreate' => false, @@ -66,7 +41,7 @@ class SwiftFactory { 'catalogType' => 'object-store' ]; - public function __construct(ICache $cache, array $params, ILogger $logger) { + public function __construct(ICache $cache, array $params, LoggerInterface $logger) { $this->cache = $cache; $this->params = $params; $this->logger = $logger; @@ -195,7 +170,7 @@ class SwiftFactory { try { /** @var \OpenStack\Identity\v2\Models\Token $token */ $token = $authService->model(\OpenStack\Identity\v2\Models\Token::class, $cachedToken['token']); - $now = new \DateTimeImmutable("now"); + $now = new \DateTimeImmutable('now'); if ($token->expires > $now) { $hasValidCachedToken = true; $this->params['v2cachedToken'] = $token; @@ -204,7 +179,7 @@ class SwiftFactory { $this->logger->debug('Cached token for swift expired'); } } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); } } } @@ -212,20 +187,20 @@ class SwiftFactory { if (!$hasValidCachedToken) { unset($this->params['cachedToken']); try { - list($token, $serviceUrl) = $authService->authenticate($this->params); + [$token, $serviceUrl] = $authService->authenticate($this->params); $this->cacheToken($token, $serviceUrl, $cacheKey); } catch (ConnectException $e) { throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e); } catch (ClientException $e) { $statusCode = $e->getResponse()->getStatusCode(); if ($statusCode === 404) { - throw new StorageAuthException('Keystone not found, verify the keystone url', $e); + throw new StorageAuthException('Keystone not found while connecting to object storage, verify the keystone url', $e); } elseif ($statusCode === 412) { - throw new StorageAuthException('Precondition failed, verify the keystone url', $e); + throw new StorageAuthException('Precondition failed while connecting to object storage, verify the keystone url', $e); } elseif ($statusCode === 401) { - throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e); + throw new StorageAuthException('Authentication failed while connecting to object storage, verify the username, password and possibly tenant', $e); } else { - throw new StorageAuthException('Unknown error', $e); + throw new StorageAuthException('Unknown error while connecting to object storage', $e); } } catch (RequestException $e) { throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e); @@ -280,7 +255,7 @@ class SwiftFactory { /** @var RequestInterface $request */ $request = $e->getRequest(); $host = $request->getUri()->getHost() . ':' . $request->getUri()->getPort(); - \OC::$server->getLogger()->error("Can't connect to object storage server at $host"); + $this->logger->error("Can't connect to object storage server at $host", ['exception' => $e]); throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e); } } diff --git a/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php index 635d0908c9a..266781af142 100644 --- a/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php +++ b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php @@ -3,34 +3,19 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; +use OpenStack\Common\Auth\Token; use OpenStack\Identity\v2\Service; class SwiftV2CachingAuthService extends Service { public function authenticate(array $options = []): array { - if (!empty($options['v2cachedToken'])) { + if (isset($options['v2cachedToken'], $options['v2serviceUrl']) + && $options['v2cachedToken'] instanceof Token + && is_string($options['v2serviceUrl'])) { return [$options['v2cachedToken'], $options['v2serviceUrl']]; } else { return parent::authenticate($options); |