diff options
Diffstat (limited to 'lib/private/Files/ObjectStore/ObjectStoreStorage.php')
| -rw-r--r-- | lib/private/Files/ObjectStore/ObjectStoreStorage.php | 416 |
1 files changed, 245 insertions, 171 deletions
diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index eb8aaffe1e0..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -1,33 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bjoern Schiessle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Marcel Klehr <mklehr@gmx.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tigran Mkrtchyan <tigran.mkrtchyan@desy.de> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; use Aws\S3\Exception\S3Exception; @@ -40,62 +17,57 @@ use OC\Files\Cache\CacheEntry; use OC\Files\Storage\PolyFill\CopyDirectory; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; use OCP\Files\FileInfo; use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; - /** - * @var \OCP\Files\ObjectStore\IObjectStore $objectStore - */ - protected $objectStore; - /** - * @var string $id - */ - protected $id; - /** - * @var \OC\User\User $user - */ - protected $user; - - private $objectPrefix = 'urn:oid:'; + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; - private $logger; + private LoggerInterface $logger; private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + private bool $preserveCacheItemsOnDelete = false; - /** @var bool */ - protected $validateWrites = true; - - public function __construct($params) { - if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { - $this->objectStore = $params['objectstore']; + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) { + $this->objectStore = $parameters['objectstore']; } else { throw new \Exception('missing IObjectStore instance'); } - if (isset($params['storageid'])) { - $this->id = 'object::store:' . $params['storageid']; + if (isset($parameters['storageid'])) { + $this->id = 'object::store:' . $parameters['storageid']; } else { $this->id = 'object::store:' . $this->objectStore->getStorageId(); } - if (isset($params['objectPrefix'])) { - $this->objectPrefix = $params['objectPrefix']; + if (isset($parameters['objectPrefix'])) { + $this->objectPrefix = $parameters['objectPrefix']; } - if (isset($params['validateWrites'])) { - $this->validateWrites = (bool)$params['validateWrites']; + if (isset($parameters['validateWrites'])) { + $this->validateWrites = (bool)$parameters['validateWrites']; } - $this->handleCopiesAsOwned = (bool)($params['handleCopiesAsOwned'] ?? false); + $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false); - $this->logger = \OC::$server->getLogger(); + $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir($path, bool $force = false) { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); if (!$force && $this->file_exists($path)) { $this->logger->warning("Tried to create an object store folder that already exists: $path"); @@ -105,7 +77,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -140,11 +112,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - /** - * @param string $path - * @return string - */ - private function normalizePath($path) { + private function normalizePath(string $path): string { $path = trim($path, '/'); //FIXME why do we sometimes get a path like 'files//username'? $path = str_replace('//', '/', $path); @@ -160,26 +128,23 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil /** * Object Stores use a NoopScanner because metadata is directly stored in * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. - * - * @param string $path - * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner - * @return \OC\Files\ObjectStore\ObjectStoreScanner */ - public function getScanner($path = '', $storage = null) { + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { if (!$storage) { $storage = $this; } if (!isset($this->scanner)) { $this->scanner = new ObjectStoreScanner($storage); } + /** @var \OC\Files\ObjectStore\ObjectStoreScanner */ return $this->scanner; } - public function getId() { + public function getId(): string { return $this->id; } - public function rmdir($path) { + public function rmdir(string $path): bool { $path = $this->normalizePath($path); $entry = $this->getCache()->get($path); @@ -204,12 +169,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - $this->getCache()->remove($entry->getPath()); + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } return true; } - public function unlink($path) { + public function unlink(string $path): bool { $path = $this->normalizePath($path); $entry = $this->getCache()->get($path); @@ -228,19 +195,24 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $this->objectStore->deleteObject($this->getURN($entry->getId())); } catch (\Exception $ex) { if ($ex->getCode() !== 404) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), - ]); + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); return false; } //removing from cache is ok as it does not exist in the objectstore anyway } - $this->getCache()->remove($entry->getPath()); + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } return true; } - public function stat($path) { + public function stat(string $path): array|false { $path = $this->normalizePath($path); $cacheEntry = $this->getCache()->get($path); if ($cacheEntry instanceof CacheEntry) { @@ -257,7 +229,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - public function getPermissions($path) { + public function getPermissions(string $path): int { $stat = $this->stat($path); if (is_array($stat) && isset($stat['permissions'])) { @@ -272,17 +244,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. * You may need a mapping table to store your URN if it cannot be generated from the fileid. * - * @param int $fileId the fileid - * @return null|string the unified resource name used to identify the object + * @return string the unified resource name used to identify the object */ - public function getURN($fileId) { - if (is_numeric($fileId)) { - return $this->objectPrefix . $fileId; - } - return null; + public function getURN(int $fileId): string { + return $this->objectPrefix . $fileId; } - public function opendir($path) { + public function opendir(string $path) { $path = $this->normalizePath($path); try { @@ -294,12 +262,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return IteratorDirectory::wrap($files); } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); return false; } } - public function filetype($path) { + public function filetype(string $path): string|false { $path = $this->normalizePath($path); $stat = $this->stat($path); if ($stat) { @@ -312,7 +280,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - public function fopen($path, $mode) { + public function fopen(string $path, string $mode) { $path = $this->normalizePath($path); if (strrpos($path, '.') !== false) { @@ -344,16 +312,22 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } return $handle; } catch (NotFoundException $e) { - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw $e; - } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); return false; } } else { @@ -398,12 +372,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return false; } - public function file_exists($path) { + public function file_exists(string $path): bool { $path = $this->normalizePath($path); return (bool)$this->stat($path); } - public function rename($source, $target) { + public function rename(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); $this->remove($target); @@ -412,12 +386,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return true; } - public function getMimeType($path) { + public function getMimeType(string $path): string|false { $path = $this->normalizePath($path); return parent::getMimeType($path); } - public function touch($path, $mtime = null) { + public function touch(string $path, ?int $mtime = null): bool { if (is_null($mtime)) { $mtime = time(); } @@ -439,58 +413,48 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object for ' . $path, - ]); + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); throw $ex; } } return true; } - public function writeBack($tmpFile, $path) { + public function writeBack(string $tmpFile, string $path) { $size = filesize($tmpFile); $this->writeStream($path, fopen($tmpFile, 'r'), $size); } - /** - * external changes are not supported, exclusive access to the object storage is assumed - * - * @param string $path - * @param int $time - * @return false - */ - public function hasUpdated($path, $time) { + public function hasUpdated(string $path, int $time): bool { return false; } - public function needsPartFile() { + public function needsPartFile(): bool { return false; } - public function file_put_contents($path, $data) { - $handle = $this->fopen($path, 'w+'); - if (!$handle) { - return false; - } - $result = fwrite($handle, $data); - fclose($handle); - return $result; + public function file_put_contents(string $path, mixed $data): int { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $data); + rewind($fh); + return $this->writeStream($path, $fh, strlen($data)); } - public function writeStream(string $path, $stream, int $size = null): int { + public function writeStream(string $path, $stream, ?int $size = null): int { + if ($size === null) { + $stats = fstat($stream); + if (is_array($stats) && isset($stats['size'])) { + $size = $stats['size']; + } + } + $stat = $this->stat($path); if (empty($stat)) { // create new file @@ -506,6 +470,14 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -517,30 +489,37 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($exists) { $fileId = $stat['fileid']; } else { + $parent = $this->normalizePath(dirname($path)); + if (!$this->is_dir($parent)) { + throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)"); + } $fileId = $this->getCache()->put($uploadPath, $stat); } $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream, $mimetype); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream, $mimetype); - if (is_resource($stream)) { - fclose($stream); - } + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -548,17 +527,23 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil * Else people lose access to existing files */ $this->getCache()->remove($uploadPath); - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } else { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not update object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { @@ -574,7 +559,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { @@ -583,10 +568,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil public function copyFromStorage( IStorage $sourceStorage, - $sourceInternalPath, - $targetInternalPath, - $preserveMtime = false - ) { + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime = false, + ): bool { if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { /** @var ObjectStoreStorage $sourceStorage */ if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { @@ -607,7 +592,90 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); } - public function copy($source, $target) { + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { + $sourceCache = $sourceStorage->getCache(); + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + // Do not import any data when source and target bucket are identical. + return true; + } + if (!$sourceCacheEntry) { + $sourceCacheEntry = $sourceCache->get($sourceInternalPath); + } + + $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry); + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $sourceStorage->rmdir($sourceInternalPath); + } else { + $sourceStorage->unlink($sourceInternalPath); + } + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + + return true; + } + + /** + * Copy the object(s) of a file or folder into this storage, without touching the cache + */ + private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) { + $copiedFiles = []; + try { + foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) { + $sourceStream = $sourceStorage->fopen($file->getPath(), 'r'); + if (!$sourceStream) { + throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})"); + } + $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType()); + if (is_resource($sourceStream)) { + fclose($sourceStream); + } + $copiedFiles[] = $file->getId(); + } + } catch (\Exception $e) { + foreach ($copiedFiles as $fileId) { + try { + $this->objectStore->deleteObject($this->getURN($fileId)); + } catch (\Exception $e) { + // ignore + } + } + throw $e; + } + } + + /** + * @return \Iterator<ICacheEntry> + */ + private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + foreach ($cache->getFolderContentsById($entry->getId()) as $child) { + yield from $this->getAllChildObjects($cache, $child); + } + } else { + yield $entry; + } + } + + public function copy(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); @@ -629,7 +697,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); @@ -645,7 +713,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $sourceUrn = $this->getURN($sourceEntry->getId()); if (!$cache instanceof Cache) { - throw new \Exception("Invalid source cache for object store copy"); + throw new \Exception('Invalid source cache for object store copy'); } $targetId = $cache->copyFromCache($cache, $sourceEntry, $to); @@ -675,7 +743,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } /** - * * @throws GenericFileException */ public function putChunkedWritePart( @@ -683,7 +750,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil string $writeToken, string $chunkId, $data, - $size = null + $size = null, ): ?array { if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { throw new GenericFileException('Object store does not support multipart upload'); @@ -721,10 +788,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil } } catch (S3MultipartUploadException|S3Exception $e) { $this->objectStore->abortMultipartUpload($urn, $writeToken); - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, - ]); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw new GenericFileException('Could not write chunked file'); } return $size; @@ -738,4 +808,8 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil $urn = $this->getURN($cacheEntry->getId()); $this->objectStore->abortMultipartUpload($urn, $writeToken); } + + public function setPreserveCacheOnDelete(bool $preserve) { + $this->preserveCacheItemsOnDelete = $preserve; + } } |