diff options
Diffstat (limited to 'lib/private/Files/ObjectStore')
17 files changed, 869 insertions, 625 deletions
diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php index 2f6db935236..aaaee044bac 100644 --- a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -3,41 +3,27 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> - * - * @author Morris Jobke <hey@morrisjobke.de> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { - /** @var string */ - private $internalId; + private string $internalId; - public function __construct($params) { - if (!isset($params['internal-id'])) { + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (!isset($parameters['internal-id'])) { throw new \Exception('missing id in parameters'); } - $this->internalId = (string)$params['internal-id']; - parent::__construct($params); + $this->internalId = (string)$parameters['internal-id']; + parent::__construct($parameters); } - public function getId() { + public function getId(): string { return 'object::appdata::preview:' . $this->internalId; } } diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php index 553f593b299..2729bb3c037 100644 --- a/lib/private/Files/ObjectStore/Azure.php +++ b/lib/private/Files/ObjectStore/Azure.php @@ -1,24 +1,8 @@ <?php + /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; @@ -38,13 +22,13 @@ class Azure implements IObjectStore { private $blobClient = null; /** @var string|null */ private $endpoint = null; - /** @var bool */ + /** @var bool */ private $autoCreate = false; /** * @param array $parameters */ - public function __construct($parameters) { + public function __construct(array $parameters) { $this->containerName = $parameters['container']; $this->accountName = $parameters['account_name']; $this->accountKey = $parameters['account_key']; @@ -62,7 +46,7 @@ class Azure implements IObjectStore { private function getBlobClient() { if (!$this->blobClient) { $protocol = $this->endpoint ? substr($this->endpoint, 0, strpos($this->endpoint, ':')) : 'https'; - $connectionString = "DefaultEndpointsProtocol=" . $protocol . ";AccountName=" . $this->accountName . ";AccountKey=" . $this->accountKey; + $connectionString = 'DefaultEndpointsProtocol=' . $protocol . ';AccountName=' . $this->accountName . ';AccountKey=' . $this->accountKey; if ($this->endpoint) { $connectionString .= ';BlobEndpoint=' . $this->endpoint; } @@ -100,7 +84,7 @@ class Azure implements IObjectStore { return $blob->getContentStream(); } - public function writeObject($urn, $stream, string $mimetype = null) { + public function writeObject($urn, $stream, ?string $mimetype = null) { $options = new CreateBlockBlobOptions(); if ($mimetype) { $options->setContentType($mimetype); diff --git a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php index b361249ff47..4e2d10705fe 100644 --- a/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php @@ -1,68 +1,42 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Björn Schießle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\ObjectStore; -use OC\User\User; +use Exception; +use OCP\Files\IHomeStorage; use OCP\IUser; -class HomeObjectStoreStorage extends ObjectStoreStorage implements \OCP\Files\IHomeStorage { +class HomeObjectStoreStorage extends ObjectStoreStorage implements IHomeStorage { + protected IUser $user; + /** * The home user storage requires a user object to create a unique storage id - * @param array $params + * + * @param array $parameters + * @throws Exception */ - public function __construct($params) { - if (! isset($params['user']) || ! $params['user'] instanceof User) { - throw new \Exception('missing user object in parameters'); + public function __construct(array $parameters) { + if (! isset($parameters['user']) || ! $parameters['user'] instanceof IUser) { + throw new Exception('missing user object in parameters'); } - $this->user = $params['user']; - parent::__construct($params); + $this->user = $parameters['user']; + parent::__construct($parameters); } - public function getId() { + public function getId(): string { return 'object::user:' . $this->user->getUID(); } - /** - * get the owner of a path - * - * @param string $path The path to get the owner - * @return false|string uid - */ - public function getOwner($path) { - if (is_object($this->user)) { - return $this->user->getUID(); - } - return false; + public function getOwner(string $path): string|false { + return $this->user->getUID(); } - /** - * @param string $path, optional - * @return \OC\User\User - */ - public function getUser($path = null): IUser { + public function getUser(): IUser { return $this->user; } } diff --git a/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php new file mode 100644 index 00000000000..369182b069d --- /dev/null +++ b/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php @@ -0,0 +1,13 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\ObjectStore; + +class InvalidObjectStoreConfigurationException extends \Exception { + +} diff --git a/lib/private/Files/ObjectStore/Mapper.php b/lib/private/Files/ObjectStore/Mapper.php index ef3c6878d81..e1174a285a6 100644 --- a/lib/private/Files/ObjectStore/Mapper.php +++ b/lib/private/Files/ObjectStore/Mapper.php @@ -1,24 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\ObjectStore; diff --git a/lib/private/Files/ObjectStore/ObjectStoreScanner.php b/lib/private/Files/ObjectStore/ObjectStoreScanner.php index d827662ae0b..5c3992b8458 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreScanner.php +++ b/lib/private/Files/ObjectStore/ObjectStoreScanner.php @@ -1,28 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\ObjectStore; @@ -32,14 +13,14 @@ use OCP\Files\FileInfo; class ObjectStoreScanner extends Scanner { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { - return []; + return null; } public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { - return []; + return null; } - protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize) { + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { return 0; } @@ -80,7 +61,7 @@ class ObjectStoreScanner extends Scanner { $query->select('path') ->from('filecache') ->where($query->expr()->eq('storage', $query->createNamedParameter($this->cache->getNumericStorageId(), IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('size', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) ->orderBy('path', 'DESC') ->setMaxResults(1); diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index eb8aaffe1e0..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -1,33 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bjoern Schiessle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Marcel Klehr <mklehr@gmx.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tigran Mkrtchyan <tigran.mkrtchyan@desy.de> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; use Aws\S3\Exception\S3Exception; @@ -40,62 +17,57 @@ use OC\Files\Cache\CacheEntry; use OC\Files\Storage\PolyFill\CopyDirectory; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; use OCP\Files\FileInfo; use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; - /** - * @var \OCP\Files\ObjectStore\IObjectStore $objectStore - */ - protected $objectStore; - /** - * @var string $id - */ - protected $id; - /** - * @var \OC\User\User $user - */ - protected $user; - - private $objectPrefix = 'urn:oid:'; + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; - private $logger; + private LoggerInterface $logger; private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + private bool $preserveCacheItemsOnDelete = false; - /** @var bool */ - protected $validateWrites = true; - - public function __construct($params) { - if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { - $this->objectStore = $params['objectstore']; + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) { + $this->objectStore = $parameters['objectstore']; } else { throw new \Exception('missing IObjectStore instance'); } - if (isset($params['storageid'])) { - $this->id = 'object::store:' . $params['storageid']; + if (isset($parameters['storageid'])) { + $this->id = 'object::store:' . $parameters['storageid']; } else { $this->id = 'object::store:' . $this->objectStore->getStorageId(); } - if (isset($params['objectPrefix'])) { - $this->objectPrefix = $params['objectPrefix']; + if (isset($parameters['objectPrefix'])) { + $this->objectPrefix = $parameters['objectPrefix']; } - if (isset($params['validateWrites'])) { - $this->validateWrites = (bool)$params['validateWrites']; + if (isset($parameters['validateWrites'])) { + $this->validateWrites = (bool)$parameters['validateWrites']; } - $this->handleCopiesAsOwned = (bool)($params['handleCopiesAsOwned'] ?? false); + $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false); - $this->logger = \OC::$server->getLogger(); + $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir($path, bool $force = false) { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); if (!$force && $this->file_exists($path)) { $this->logger->warning("Tried to create an object store folder that already exists: $path"); @@ -105,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -140,11 +112,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - /** - * @param string $path - * @return string - */ - private function normalizePath($path) { + private function normalizePath(string $path): string { $path = trim($path, '/'); //FIXME why do we sometimes get a path like 'files//username'? $path = str_replace('//', '/', $path); @@ -160,26 +128,23 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil /** * Object Stores use a NoopScanner because metadata is directly stored in * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. - * - * @param string $path - * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner - * @return \OC\Files\ObjectStore\ObjectStoreScanner */ - public function getScanner($path = '', $storage = null) { + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { if (!$storage) { $storage = $this; } if (!isset($this->scanner)) { $this->scanner = new ObjectStoreScanner($storage); } + /** @var \OC\Files\ObjectStore\ObjectStoreScanner */ return $this->scanner; } - public function getId() { + public function getId(): string { return $this->id; } - public function rmdir($path) { + public function rmdir(string $path): bool { $path = $this->normalizePath($path); $entry = $this->getCache()->get($path); @@ -204,12 +169,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - $this->getCache()->remove($entry->getPath()); + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } return true; } - public function unlink($path) { + public function unlink(string $path): bool { $path = $this->normalizePath($path); $entry = $this->getCache()->get($path); @@ -228,19 +195,24 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $this->objectStore->deleteObject($this->getURN($entry->getId())); } catch (\Exception $ex) { if ($ex->getCode() !== 404) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), - ]); + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); return false; } //removing from cache is ok as it does not exist in the objectstore anyway } - $this->getCache()->remove($entry->getPath()); + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } return true; } - public function stat($path) { + public function stat(string $path): array|false { $path = $this->normalizePath($path); $cacheEntry = $this->getCache()->get($path); if ($cacheEntry instanceof CacheEntry) { @@ -257,7 +229,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - public function getPermissions($path) { + public function getPermissions(string $path): int { $stat = $this->stat($path); if (is_array($stat) && isset($stat['permissions'])) { @@ -272,17 +244,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. * You may need a mapping table to store your URN if it cannot be generated from the fileid. * - * @param int $fileId the fileid - * @return null|string the unified resource name used to identify the object + * @return string the unified resource name used to identify the object */ - public function getURN($fileId) { - if (is_numeric($fileId)) { - return $this->objectPrefix . $fileId; - } - return null; + public function getURN(int $fileId): string { + return $this->objectPrefix . $fileId; } - public function opendir($path) { + public function opendir(string $path) { $path = $this->normalizePath($path); try { @@ -294,12 +262,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return IteratorDirectory::wrap($files); } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); return false; } } - public function filetype($path) { + public function filetype(string $path): string|false { $path = $this->normalizePath($path); $stat = $this->stat($path); if ($stat) { @@ -312,7 +280,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - public function fopen($path, $mode) { + public function fopen(string $path, string $mode) { $path = $this->normalizePath($path); if (strrpos($path, '.') !== false) { @@ -344,16 +312,22 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } return $handle; } catch (NotFoundException $e) { - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw $e; - } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); return false; } } else { @@ -398,12 +372,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return false; } - public function file_exists($path) { + public function file_exists(string $path): bool { $path = $this->normalizePath($path); return (bool)$this->stat($path); } - public function rename($source, $target) { + public function rename(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); $this->remove($target); @@ -412,12 +386,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return true; } - public function getMimeType($path) { + public function getMimeType(string $path): string|false { $path = $this->normalizePath($path); return parent::getMimeType($path); } - public function touch($path, $mtime = null) { + public function touch(string $path, ?int $mtime = null): bool { if (is_null($mtime)) { $mtime = time(); } @@ -439,58 +413,48 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object for ' . $path, - ]); + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); throw $ex; } } return true; } - public function writeBack($tmpFile, $path) { + public function writeBack(string $tmpFile, string $path) { $size = filesize($tmpFile); $this->writeStream($path, fopen($tmpFile, 'r'), $size); } - /** - * external changes are not supported, exclusive access to the object storage is assumed - * - * @param string $path - * @param int $time - * @return false - */ - public function hasUpdated($path, $time) { + public function hasUpdated(string $path, int $time): bool { return false; } - public function needsPartFile() { + public function needsPartFile(): bool { return false; } - public function file_put_contents($path, $data) { - $handle = $this->fopen($path, 'w+'); - if (!$handle) { - return false; - } - $result = fwrite($handle, $data); - fclose($handle); - return $result; + public function file_put_contents(string $path, mixed $data): int { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $data); + rewind($fh); + return $this->writeStream($path, $fh, strlen($data)); } - public function writeStream(string $path, $stream, int $size = null): int { + public function writeStream(string $path, $stream, ?int $size = null): int { + if ($size === null) { + $stats = fstat($stream); + if (is_array($stats) && isset($stats['size'])) { + $size = $stats['size']; + } + } + $stat = $this->stat($path); if (empty($stat)) { // create new file @@ -506,6 +470,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -517,30 +489,37 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($exists) { $fileId = $stat['fileid']; } else { + $parent = $this->normalizePath(dirname($path)); + if (!$this->is_dir($parent)) { + throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)"); + } $fileId = $this->getCache()->put($uploadPath, $stat); } $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream, $mimetype); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream, $mimetype); - if (is_resource($stream)) { - fclose($stream); - } + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -548,17 +527,23 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil * Else people lose access to existing files */ $this->getCache()->remove($uploadPath); - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } else { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not update object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { @@ -574,7 +559,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { @@ -583,10 +568,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil public function copyFromStorage( IStorage $sourceStorage, - $sourceInternalPath, - $targetInternalPath, - $preserveMtime = false - ) { + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime = false, + ): bool { if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { /** @var ObjectStoreStorage $sourceStorage */ if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { @@ -607,7 +592,90 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); } - public function copy($source, $target) { + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { + $sourceCache = $sourceStorage->getCache(); + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + // Do not import any data when source and target bucket are identical. + return true; + } + if (!$sourceCacheEntry) { + $sourceCacheEntry = $sourceCache->get($sourceInternalPath); + } + + $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry); + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $sourceStorage->rmdir($sourceInternalPath); + } else { + $sourceStorage->unlink($sourceInternalPath); + } + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + + return true; + } + + /** + * Copy the object(s) of a file or folder into this storage, without touching the cache + */ + private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) { + $copiedFiles = []; + try { + foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) { + $sourceStream = $sourceStorage->fopen($file->getPath(), 'r'); + if (!$sourceStream) { + throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})"); + } + $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType()); + if (is_resource($sourceStream)) { + fclose($sourceStream); + } + $copiedFiles[] = $file->getId(); + } + } catch (\Exception $e) { + foreach ($copiedFiles as $fileId) { + try { + $this->objectStore->deleteObject($this->getURN($fileId)); + } catch (\Exception $e) { + // ignore + } + } + throw $e; + } + } + + /** + * @return \Iterator<ICacheEntry> + */ + private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + foreach ($cache->getFolderContentsById($entry->getId()) as $child) { + yield from $this->getAllChildObjects($cache, $child); + } + } else { + yield $entry; + } + } + + public function copy(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); @@ -629,7 +697,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); @@ -645,7 +713,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $sourceUrn = $this->getURN($sourceEntry->getId()); if (!$cache instanceof Cache) { - throw new \Exception("Invalid source cache for object store copy"); + throw new \Exception('Invalid source cache for object store copy'); } $targetId = $cache->copyFromCache($cache, $sourceEntry, $to); @@ -675,7 +743,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } /** - * * @throws GenericFileException */ public function putChunkedWritePart( @@ -683,7 +750,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil string $writeToken, string $chunkId, $data, - $size = null + $size = null, ): ?array { if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { throw new GenericFileException('Object store does not support multipart upload'); @@ -721,10 +788,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } catch (S3MultipartUploadException|S3Exception $e) { $this->objectStore->abortMultipartUpload($urn, $writeToken); - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, - ]); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw new GenericFileException('Could not write chunked file'); } return $size; @@ -738,4 +808,8 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $urn = $this->getURN($cacheEntry->getId()); $this->objectStore->abortMultipartUpload($urn, $writeToken); } + + public function setPreserveCacheOnDelete(bool $preserve) { + $this->preserveCacheItemsOnDelete = $preserve; + } } diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php new file mode 100644 index 00000000000..ffc33687340 --- /dev/null +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -0,0 +1,225 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\Files\ObjectStore; + +use OCP\App\IAppManager; +use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +use OCP\IUser; + +/** + * @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}} + */ +class PrimaryObjectStoreConfig { + public function __construct( + private readonly IConfig $config, + private readonly IAppManager $appManager, + ) { + } + + /** + * @param ObjectStoreConfig $config + */ + public function buildObjectStore(array $config): IObjectStore { + return new $config['class']($config['arguments']); + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForRoot(): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $config = $this->getObjectStoreConfiguration('root'); + + if ($config['arguments']['multibucket']) { + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + // put the root FS always in first bucket for multibucket configuration + $config['arguments']['bucket'] .= '0'; + } + return $config; + } + + /** + * @return ?ObjectStoreConfig + */ + public function getObjectStoreConfigForUser(IUser $user): ?array { + if (!$this->hasObjectStore()) { + return null; + } + + $store = $this->getObjectStoreForUser($user); + $config = $this->getObjectStoreConfiguration($store); + + if ($config['arguments']['multibucket']) { + $config['arguments']['bucket'] = $this->getBucketForUser($user, $config); + } + return $config; + } + + /** + * @param string $name + * @return ObjectStoreConfig + */ + public function getObjectStoreConfiguration(string $name): array { + $configs = $this->getObjectStoreConfigs(); + $name = $this->resolveAlias($name); + if (!isset($configs[$name])) { + throw new \Exception("Object store configuration for '$name' not found"); + } + if (is_string($configs[$name])) { + throw new \Exception("Object store configuration for '{$configs[$name]}' not found"); + } + return $configs[$name]; + } + + public function resolveAlias(string $name): string { + $configs = $this->getObjectStoreConfigs(); + + while (isset($configs[$name]) && is_string($configs[$name])) { + $name = $configs[$name]; + } + return $name; + } + + public function hasObjectStore(): bool { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + return $objectStore || $objectStoreMultiBucket; + } + + public function hasMultipleObjectStorages(): bool { + $objectStore = $this->config->getSystemValue('objectstore', []); + return isset($objectStore['default']); + } + + /** + * @return ?array<string, ObjectStoreConfig|string> + * @throws InvalidObjectStoreConfigurationException + */ + public function getObjectStoreConfigs(): ?array { + $objectStore = $this->config->getSystemValue('objectstore', null); + $objectStoreMultiBucket = $this->config->getSystemValue('objectstore_multibucket', null); + + // new-style multibucket config uses the same 'objectstore' key but sets `'multibucket' => true`, transparently upgrade older style config + if ($objectStoreMultiBucket) { + $objectStoreMultiBucket['arguments']['multibucket'] = true; + return [ + 'default' => 'server1', + 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), + 'root' => 'server1', + ]; + } elseif ($objectStore) { + if (!isset($objectStore['default'])) { + $objectStore = [ + 'default' => 'server1', + 'root' => 'server1', + 'server1' => $objectStore, + ]; + } + if (!isset($objectStore['root'])) { + $objectStore['root'] = 'default'; + } + + if (!is_string($objectStore['default'])) { + throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); + } + return array_map($this->validateObjectStoreConfig(...), $objectStore); + } else { + return null; + } + } + + /** + * @param array|string $config + * @return string|ObjectStoreConfig + */ + private function validateObjectStoreConfig(array|string $config): array|string { + if (is_string($config)) { + return $config; + } + if (!isset($config['class'])) { + throw new InvalidObjectStoreConfigurationException('No class configured for object store'); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $class = $config['class']; + $arguments = $config['arguments']; + if (!is_array($arguments)) { + throw new InvalidObjectStoreConfigurationException('Configured object store arguments are not an array'); + } + if (!isset($arguments['multibucket'])) { + $arguments['multibucket'] = false; + } + if (!is_bool($arguments['multibucket'])) { + throw new InvalidObjectStoreConfigurationException('arguments.multibucket must be a boolean in object store configuration'); + } + + if (!is_string($class)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not a string'); + } + + if (str_starts_with($class, 'OCA\\') && substr_count($class, '\\') >= 2) { + [$appId] = explode('\\', $class); + $this->appManager->loadApp(strtolower($appId)); + } + + if (!is_a($class, IObjectStore::class, true)) { + throw new InvalidObjectStoreConfigurationException('Configured class for object store is not an object store'); + } + return [ + 'class' => $class, + 'arguments' => $arguments, + ]; + } + + public function getBucketForUser(IUser $user, array $config): string { + $bucket = $this->getSetBucketForUser($user); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new Mapper($user, $this->config); + $numBuckets = $config['arguments']['num_buckets'] ?? 64; + $bucket = $config['arguments']['bucket'] . $mapper->getBucket($numBuckets); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $bucket); + } + + return $bucket; + } + + public function getSetBucketForUser(IUser $user): ?string { + return $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + } + + public function getObjectStoreForUser(IUser $user): string { + if ($this->hasMultipleObjectStorages()) { + $value = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'objectstore', null); + if ($value === null) { + $value = $this->resolveAlias('default'); + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'objectstore', $value); + } + return $value; + } else { + return 'default'; + } + } +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index b1cd89388ae..72e1751e23d 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -1,38 +1,23 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ + namespace OC\Files\ObjectStore; use Aws\Result; use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore, IObjectStoreMultiPartUpload { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; - public function __construct($parameters) { + public function __construct(array $parameters) { $parameters['primary_storage'] = true; $this->parseParams($parameters); } @@ -79,7 +64,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { 'Key' => $urn, 'UploadId' => $uploadId, 'MaxParts' => 1000, - 'PartNumberMarker' => $partNumberMarker + 'PartNumberMarker' => $partNumberMarker, ] + $this->getSSECParameters()); $parts = array_merge($parts, $result->get('Parts') ?? []); $isTruncated = $result->get('IsTruncated'); @@ -107,7 +92,51 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { $this->getConnection()->abortMultipartUpload([ 'Bucket' => $this->bucket, 'Key' => $urn, - 'UploadId' => $uploadId + 'UploadId' => $uploadId, ]); } + + private function parseS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + if (str_starts_with($key, 'x-amz-meta-')) { + $result[substr($key, strlen('x-amz-meta-'))] = $value; + } + } + return $result; + } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ] + $this->parseS3Metadata($object['Metadata'] ?? []); + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } } diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php new file mode 100644 index 00000000000..5b086db8f77 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -0,0 +1,41 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +namespace OC\Files\ObjectStore; + +/** + * Shared configuration between ConnectionTrait and ObjectTrait to ensure both to be in sync + */ +trait S3ConfigTrait { + protected array $params; + + protected string $bucket; + + /** Maximum number of concurrent multipart uploads */ + protected int $concurrency; + + /** Timeout, in seconds, for the connection to S3 server, not for the + * request. */ + protected float $connectTimeout; + + protected int $timeout; + + protected string|false $proxy; + + protected string $storageClass; + + /** @var int Part size in bytes (float is added for 32bit support) */ + protected int|float $uploadPartSize; + + /** @var int Limit on PUT in bytes (float is added for 32bit support) */ + private int|float $putSizeLimit; + + /** @var int Limit on COPY in bytes (float is added for 32bit support) */ + private int|float $copySizeLimit; + + private bool $useMultipartCopy = true; +} diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index a1edfa1eb99..67b82a44ab7 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -1,35 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Florent <florent@coppint.com> - * @author James Letendre <James.Letendre@gmail.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com> - * @author Stephen Cuppett <steve@cuppett.com> - * @author Jasper Weyne <jasperweyne@gmail.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\Files\ObjectStore; use Aws\ClientResolver; @@ -38,56 +12,35 @@ use Aws\Credentials\Credentials; use Aws\Exception\CredentialsException; use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; -use GuzzleHttp\Promise; +use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; +use OCP\Files\StorageNotAvailableException; use OCP\ICertificateManager; +use OCP\Server; use Psr\Log\LoggerInterface; trait S3ConnectionTrait { - /** @var array */ - protected $params; - - /** @var S3Client */ - protected $connection; - - /** @var string */ - protected $id; - - /** @var string */ - protected $bucket; - - /** @var int */ - protected $timeout; + use S3ConfigTrait; - /** @var string */ - protected $proxy; + protected string $id; - /** @var string */ - protected $storageClass; + protected bool $test; - /** @var int */ - protected $uploadPartSize; - - /** @var int */ - private $putSizeLimit; - - /** @var int */ - private $copySizeLimit; - - private bool $useMultipartCopy = true; - - protected $test; + protected ?S3Client $connection = null; protected function parseParams($params) { if (empty($params['bucket'])) { - throw new \Exception("Bucket has to be configured."); + throw new \Exception('Bucket has to be configured.'); } $this->id = 'amazon::' . $params['bucket']; $this->test = isset($params['test']); $this->bucket = $params['bucket']; + // Default to 5 like the S3 SDK does + $this->concurrency = $params['concurrency'] ?? 5; $this->proxy = $params['proxy'] ?? false; + $this->connectTimeout = $params['connect_timeout'] ?? 5; $this->timeout = $params['timeout'] ?? 15; $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000; @@ -96,10 +49,16 @@ trait S3ConnectionTrait { $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true); $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; + $params['s3-accelerate'] = $params['hostname'] === 's3-accelerate.amazonaws.com' || $params['hostname'] === 's3-accelerate.dualstack.amazonaws.com'; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true; + + if ($params['s3-accelerate']) { + $params['verify_bucket_exists'] = false; + } + $this->params = $params; } @@ -118,7 +77,7 @@ trait S3ConnectionTrait { * @throws \Exception if connection could not be made */ public function getConnection() { - if (!is_null($this->connection)) { + if ($this->connection !== null) { return $this->connection; } @@ -143,9 +102,23 @@ trait S3ConnectionTrait { 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()), 'csm' => false, 'use_arn_region' => false, - 'http' => ['verify' => $this->getCertificateBundlePath()], + 'http' => [ + 'verify' => $this->getCertificateBundlePath(), + 'connect_timeout' => $this->connectTimeout, + ], 'use_aws_shared_config_files' => false, + 'retries' => [ + 'mode' => 'standard', + 'max_attempts' => 5, + ], ]; + + if ($this->params['s3-accelerate']) { + $options['use_accelerate_endpoint'] = true; + } else { + $options['endpoint'] = $base_url; + } + if ($this->getProxy()) { $options['http']['proxy'] = $this->getProxy(); } @@ -154,33 +127,38 @@ trait S3ConnectionTrait { } $this->connection = new S3Client($options); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - $logger = \OC::$server->get(LoggerInterface::class); - $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', - ['app' => 'objectstore']); - } + try { + $logger = Server::get(LoggerInterface::class); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', + ['app' => 'objectstore']); + } - if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { - $logger = \OC::$server->get(LoggerInterface::class); - try { - $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); + if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { + try { + $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket); + } + $this->connection->createBucket(['Bucket' => $this->bucket]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->debug('Invalid remote storage.', [ + 'exception' => $e, + 'app' => 'objectstore', + ]); + if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') { + throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); + } } - $this->connection->createBucket(['Bucket' => $this->bucket]); - $this->testTimeout(); - } catch (S3Exception $e) { - $logger->debug('Invalid remote storage.', [ - 'exception' => $e, - 'app' => 'objectstore', - ]); - throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); } - } - // google cloud's s3 compatibility doesn't like the EncodingType parameter - if (strpos($base_url, 'storage.googleapis.com')) { - $this->connection->getHandlerList()->remove('s3.auto_encode'); + // google cloud's s3 compatibility doesn't like the EncodingType parameter + if (strpos($base_url, 'storage.googleapis.com')) { + $this->connection->getHandlerList()->remove('s3.auto_encode'); + } + } catch (S3Exception $e) { + throw new StorageNotAvailableException('S3 service is unable to handle request: ' . $e->getMessage()); } return $this->connection; @@ -212,10 +190,12 @@ trait S3ConnectionTrait { return function () { $key = empty($this->params['key']) ? null : $this->params['key']; $secret = empty($this->params['secret']) ? null : $this->params['secret']; + $sessionToken = empty($this->params['session_token']) ? null : $this->params['session_token']; if ($key && $secret) { - return Promise\promise_for( - new Credentials($key, $secret) + return Create::promiseFor( + // a null sessionToken match the default signature of the constructor + new Credentials($key, $secret, $sessionToken) ); } @@ -225,11 +205,11 @@ trait S3ConnectionTrait { } protected function getCertificateBundlePath(): ?string { - if ((int)($this->params['use_nextcloud_bundle'] ?? "0")) { + if ((int)($this->params['use_nextcloud_bundle'] ?? '0')) { // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage if (!isset($this->params['primary_storage'])) { /** @var ICertificateManager $certManager */ - $certManager = \OC::$server->get(ICertificateManager::class); + $certManager = Server::get(ICertificateManager::class); return $certManager->getAbsoluteBundlePath(); } else { return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt'; @@ -240,7 +220,7 @@ trait S3ConnectionTrait { } protected function getSSECKey(): ?string { - if (isset($this->params['sse_c_key'])) { + if (isset($this->params['sse_c_key']) && !empty($this->params['sse_c_key'])) { return $this->params['sse_c_key']; } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 2ef9614ac85..89405de2e8e 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -1,31 +1,13 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Florent <florent@coppint.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; +use Aws\Command; +use Aws\Exception\MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; @@ -36,6 +18,8 @@ use OC\Files\Stream\SeekableHttpStream; use Psr\Http\Message\StreamInterface; trait S3ObjectTrait { + use S3ConfigTrait; + /** * Returns the connection * @@ -95,24 +79,42 @@ trait S3ObjectTrait { return $fh; } + private function buildS3Metadata(array $metadata): array { + $result = []; + foreach ($metadata as $key => $value) { + $result['x-amz-meta-' . $key] = $value; + } + return $result; + } /** * Single object put helper * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeSingle(string $urn, StreamInterface $stream, string $mimetype = null): void { - $this->getConnection()->putObject([ + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $args = [ 'Bucket' => $this->bucket, 'Key' => $urn, 'Body' => $stream, 'ACL' => 'private', 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters()); + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); } @@ -121,56 +123,116 @@ trait S3ObjectTrait { * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void { - $uploader = new MultipartUploader($this->getConnection(), $stream, [ - 'bucket' => $this->bucket, - 'key' => $urn, - 'part_size' => $this->uploadPartSize, - 'params' => [ - 'ContentType' => $mimetype, - 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters(), - ]); + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { + $mimetype = $metaData['mimetype'] ?? null; + unset($metaData['mimetype']); + unset($metaData['size']); + + $attempts = 0; + $uploaded = false; + $concurrency = $this->concurrency; + $exception = null; + $state = null; + $size = $stream->getSize(); + $totalWritten = 0; + + // retry multipart upload once with concurrency at half on failure + while (!$uploaded && $attempts <= 1) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'concurrency' => $concurrency, + 'key' => $urn, + 'part_size' => $this->uploadPartSize, + 'state' => $state, + 'params' => [ + 'ContentType' => $mimetype, + 'Metadata' => $this->buildS3Metadata($metaData), + 'StorageClass' => $this->storageClass, + ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, + ]); + + try { + $uploader->upload(); + $uploaded = true; + } catch (S3MultipartUploadException $e) { + $exception = $e; + $attempts++; + + if ($concurrency > 1) { + $concurrency = round($concurrency / 2); + } + + if ($stream->isSeekable()) { + $stream->rewind(); + } + } catch (MultipartUploadException $e) { + $exception = $e; + break; + } + } - try { - $uploader->upload(); - } catch (S3MultipartUploadException $e) { + if (!$uploaded) { // if anything goes wrong with multipart, make sure that you don´t poison and // slow down s3 bucket with orphaned fragments - $uploadInfo = $e->getState()->getId(); - if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { + $uploadInfo = $exception->getState()->getId(); + if ($exception->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) { $this->getConnection()->abortMultipartUpload($uploadInfo); } - throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway("Error while uploading to S3 bucket", 0, $e); + + throw new \OCA\DAV\Connector\Sabre\Exception\BadGateway('Error while uploading to S3 bucket', 0, $exception); } } + public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ - public function writeObject($urn, $stream, string $mimetype = null) { - $psrStream = Utils::streamFor($stream); - - // ($psrStream->isSeekable() && $psrStream->getSize() !== null) evaluates to true for a On-Seekable stream - // so the optimisation does not apply - $buffer = new Psr7\Stream(fopen("php://memory", 'rwb+')); - Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); - $buffer->seek(0); - if ($buffer->getSize() < $this->putSizeLimit) { - // buffer is fully seekable, so use it directly for the small upload - $this->writeSingle($urn, $buffer, $mimetype); + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { + $canSeek = fseek($stream, 0, SEEK_CUR) === 0; + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); + + + $size = $psrStream->getSize(); + if ($size === null || !$canSeek) { + // The s3 single-part upload requires the size to be known for the stream. + // So for input streams that don't have a known size, we need to copy (part of) + // the input into a temporary stream so the size can be determined + $buffer = new Psr7\Stream(fopen('php://temp', 'rw+')); + Utils::copyToStream($psrStream, $buffer, $this->putSizeLimit); + $buffer->seek(0); + if ($buffer->getSize() < $this->putSizeLimit) { + // buffer is fully seekable, so use it directly for the small upload + $this->writeSingle($urn, $buffer, $metaData); + } else { + $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); + $this->writeMultiPart($urn, $loadStream, $metaData); + } } else { - $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); - $this->writeMultiPart($urn, $loadStream, $mimetype); + if ($size < $this->putSizeLimit) { + $this->writeSingle($urn, $psrStream, $metaData); + } else { + $this->writeMultiPart($urn, $psrStream, $metaData); + } } + $psrStream->close(); } /** @@ -200,19 +262,20 @@ trait S3ObjectTrait { if ($this->useMultipartCopy && $size > $this->copySizeLimit) { $copy = new MultipartCopy($this->getConnection(), [ - "source_bucket" => $this->getBucket(), - "source_key" => $from + 'source_bucket' => $this->getBucket(), + 'source_key' => $from ], array_merge([ - "bucket" => $this->getBucket(), - "key" => $to, - "acl" => "private", - "params" => $this->getSSECParameters() + $this->getSSECParameters(true), - "source_metadata" => $sourceMetadata + 'bucket' => $this->getBucket(), + 'key' => $to, + 'acl' => 'private', + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'source_metadata' => $sourceMetadata ], $options)); $copy->copy(); } else { $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', array_merge([ - 'params' => $this->getSSECParameters() + $this->getSSECParameters(true) + 'params' => $this->getSSECParameters() + $this->getSSECParameters(true), + 'mup_threshold' => PHP_INT_MAX, ], $options)); } } diff --git a/lib/private/Files/ObjectStore/S3Signature.php b/lib/private/Files/ObjectStore/S3Signature.php index cf3d29c4185..b80382ff67d 100644 --- a/lib/private/Files/ObjectStore/S3Signature.php +++ b/lib/private/Files/ObjectStore/S3Signature.php @@ -1,26 +1,8 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; @@ -60,7 +42,7 @@ class S3Signature implements SignatureInterface { public function signRequest( RequestInterface $request, - CredentialsInterface $credentials + CredentialsInterface $credentials, ) { $request = $this->prepareRequest($request, $credentials); $stringToSign = $this->createCanonicalizedString($request); @@ -75,7 +57,7 @@ class S3Signature implements SignatureInterface { RequestInterface $request, CredentialsInterface $credentials, $expires, - array $options = [] + array $options = [], ) { $query = []; // URL encoding already occurs in the URI template expansion. Undo that @@ -112,20 +94,20 @@ class S3Signature implements SignatureInterface { } } - $queryString = http_build_query($query, null, '&', PHP_QUERY_RFC3986); + $queryString = http_build_query($query, '', '&', PHP_QUERY_RFC3986); return $request->withUri($request->getUri()->withQuery($queryString)); } /** - * @param RequestInterface $request + * @param RequestInterface $request * @param CredentialsInterface $creds * * @return RequestInterface */ private function prepareRequest( RequestInterface $request, - CredentialsInterface $creds + CredentialsInterface $creds, ) { $modify = [ 'remove_headers' => ['X-Amz-Date'], @@ -148,7 +130,7 @@ class S3Signature implements SignatureInterface { private function createCanonicalizedString( RequestInterface $request, - $expires = null + $expires = null, ) { $buffer = $request->getMethod() . "\n"; diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php index 85926be897e..888602a62e4 100644 --- a/lib/private/Files/ObjectStore/StorageObjectStore.php +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -1,25 +1,8 @@ <?php + /** - * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; @@ -45,8 +28,8 @@ class StorageObjectStore implements IObjectStore { * @return string the container or bucket name where objects are stored * @since 7.0.0 */ - public function getStorageId() { - $this->storage->getId(); + public function getStorageId(): string { + return $this->storage->getId(); } /** @@ -64,7 +47,7 @@ class StorageObjectStore implements IObjectStore { throw new \Exception(); } - public function writeObject($urn, $stream, string $mimetype = null) { + public function writeObject($urn, $stream, ?string $mimetype = null) { $handle = $this->storage->fopen($urn, 'w'); if ($handle) { stream_copy_to_stream($stream, $handle); diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index b463cb9d44d..aa8b3bb34ec 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Adrian Brzezinski <adrian.brzezinski@eo.pl> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\Files\ObjectStore; @@ -45,7 +27,7 @@ class Swift implements IObjectStore { /** @var SwiftFactory */ private $swiftFactory; - public function __construct($params, SwiftFactory $connectionFactory = null) { + public function __construct($params, ?SwiftFactory $connectionFactory = null) { $this->swiftFactory = $connectionFactory ?: new SwiftFactory( \OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params, @@ -74,7 +56,7 @@ class Swift implements IObjectStore { return $this->params['container']; } - public function writeObject($urn, $stream, string $mimetype = null) { + public function writeObject($urn, $stream, ?string $mimetype = null) { $tmpFile = \OC::$server->getTempManager()->getTemporaryFile('swiftwrite'); file_put_contents($tmpFile, $stream); $handle = fopen($tmpFile, 'rb'); diff --git a/lib/private/Files/ObjectStore/SwiftFactory.php b/lib/private/Files/ObjectStore/SwiftFactory.php index bd75ccada2e..118724159e5 100644 --- a/lib/private/Files/ObjectStore/SwiftFactory.php +++ b/lib/private/Files/ObjectStore/SwiftFactory.php @@ -3,32 +3,8 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Adrian Brzezinski <adrian.brzezinski@eo.pl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Julien Lutran <julien.lutran@corp.ovh.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Volker <skydiablo@gmx.net> - * @author William Pain <pain.william@gmail.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; @@ -194,7 +170,7 @@ class SwiftFactory { try { /** @var \OpenStack\Identity\v2\Models\Token $token */ $token = $authService->model(\OpenStack\Identity\v2\Models\Token::class, $cachedToken['token']); - $now = new \DateTimeImmutable("now"); + $now = new \DateTimeImmutable('now'); if ($token->expires > $now) { $hasValidCachedToken = true; $this->params['v2cachedToken'] = $token; @@ -218,13 +194,13 @@ class SwiftFactory { } catch (ClientException $e) { $statusCode = $e->getResponse()->getStatusCode(); if ($statusCode === 404) { - throw new StorageAuthException('Keystone not found, verify the keystone url', $e); + throw new StorageAuthException('Keystone not found while connecting to object storage, verify the keystone url', $e); } elseif ($statusCode === 412) { - throw new StorageAuthException('Precondition failed, verify the keystone url', $e); + throw new StorageAuthException('Precondition failed while connecting to object storage, verify the keystone url', $e); } elseif ($statusCode === 401) { - throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e); + throw new StorageAuthException('Authentication failed while connecting to object storage, verify the username, password and possibly tenant', $e); } else { - throw new StorageAuthException('Unknown error', $e); + throw new StorageAuthException('Unknown error while connecting to object storage', $e); } } catch (RequestException $e) { throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e); diff --git a/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php index b1478762550..266781af142 100644 --- a/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php +++ b/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php @@ -3,33 +3,19 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\Files\ObjectStore; +use OpenStack\Common\Auth\Token; use OpenStack\Identity\v2\Service; class SwiftV2CachingAuthService extends Service { public function authenticate(array $options = []): array { - if (!empty($options['v2cachedToken'])) { + if (isset($options['v2cachedToken'], $options['v2serviceUrl']) + && $options['v2cachedToken'] instanceof Token + && is_string($options['v2serviceUrl'])) { return [$options['v2cachedToken'], $options['v2serviceUrl']]; } else { return parent::authenticate($options); |