diff options
Diffstat (limited to 'lib/private/Files/ObjectStore/ObjectStoreStorage.php')
| -rw-r--r-- | lib/private/Files/ObjectStore/ObjectStoreStorage.php | 618 |
1 files changed, 423 insertions, 195 deletions
diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 3378f00c4dd..9ab11f8a3df 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -1,98 +1,83 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bjoern Schiessle <bjoern@schiessle.org> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Marcel Klehr <mklehr@gmx.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Tigran Mkrtchyan <tigran.mkrtchyan@desy.de> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\Files\ObjectStore; +use Aws\S3\Exception\S3Exception; +use Aws\S3\Exception\S3MultipartUploadException; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\CountWrapper; use Icewind\Streams\IteratorDirectory; +use OC\Files\Cache\Cache; use OC\Files\Cache\CacheEntry; use OC\Files\Storage\PolyFill\CopyDirectory; +use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; +use OCP\Files\Cache\IScanner; use OCP\Files\FileInfo; +use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; +use OCP\Files\Storage\IChunkedFileWrite; +use OCP\Files\Storage\IStorage; +use Psr\Log\LoggerInterface; -class ObjectStoreStorage extends \OC\Files\Storage\Common { +class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; - /** - * @var \OCP\Files\ObjectStore\IObjectStore $objectStore - */ - protected $objectStore; - /** - * @var string $id - */ - protected $id; - /** - * @var \OC\User\User $user - */ - protected $user; + protected IObjectStore $objectStore; + protected string $id; + private string $objectPrefix = 'urn:oid:'; - private $objectPrefix = 'urn:oid:'; + private LoggerInterface $logger; - private $logger; + private bool $handleCopiesAsOwned; + protected bool $validateWrites = true; + private bool $preserveCacheItemsOnDelete = false; - public function __construct($params) { - if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { - $this->objectStore = $params['objectstore']; + /** + * @param array $parameters + * @throws \Exception + */ + public function __construct(array $parameters) { + if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) { + $this->objectStore = $parameters['objectstore']; } else { throw new \Exception('missing IObjectStore instance'); } - if (isset($params['storageid'])) { - $this->id = 'object::store:' . $params['storageid']; + if (isset($parameters['storageid'])) { + $this->id = 'object::store:' . $parameters['storageid']; } else { $this->id = 'object::store:' . $this->objectStore->getStorageId(); } - if (isset($params['objectPrefix'])) { - $this->objectPrefix = $params['objectPrefix']; + if (isset($parameters['objectPrefix'])) { + $this->objectPrefix = $parameters['objectPrefix']; } - //initialize cache with root directory in cache - if (!$this->is_dir('/')) { - $this->mkdir('/'); + if (isset($parameters['validateWrites'])) { + $this->validateWrites = (bool)$parameters['validateWrites']; } + $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false); - $this->logger = \OC::$server->getLogger(); + $this->logger = \OCP\Server::get(LoggerInterface::class); } - public function mkdir($path) { + public function mkdir(string $path, bool $force = false, array $metadata = []): bool { $path = $this->normalizePath($path); - - if ($this->file_exists($path)) { + if (!$force && $this->file_exists($path)) { + $this->logger->warning("Tried to create an object store folder that already exists: $path"); return false; } $mTime = time(); $data = [ 'mimetype' => 'httpd/unix-directory', - 'size' => 0, + 'size' => $metadata['size'] ?? 0, 'mtime' => $mTime, 'storage_mtime' => $mTime, 'permissions' => \OCP\Constants::PERMISSION_ALL, @@ -109,10 +94,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { if ($parentType === false) { if (!$this->mkdir($parent)) { // something went wrong + $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created"); return false; } } elseif ($parentType === 'file') { // parent is a file + $this->logger->warning("Parent ($parent) is a file"); return false; } // finally create the new dir @@ -125,11 +112,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - /** - * @param string $path - * @return string - */ - private function normalizePath($path) { + private function normalizePath(string $path): string { $path = trim($path, '/'); //FIXME why do we sometimes get a path like 'files//username'? $path = str_replace('//', '/', $path); @@ -145,95 +128,108 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { /** * Object Stores use a NoopScanner because metadata is directly stored in * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere. - * - * @param string $path - * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner - * @return \OC\Files\ObjectStore\NoopScanner */ - public function getScanner($path = '', $storage = null) { + public function getScanner(string $path = '', ?IStorage $storage = null): IScanner { if (!$storage) { $storage = $this; } if (!isset($this->scanner)) { - $this->scanner = new NoopScanner($storage); + $this->scanner = new ObjectStoreScanner($storage); } + /** @var \OC\Files\ObjectStore\ObjectStoreScanner */ return $this->scanner; } - public function getId() { + public function getId(): string { return $this->id; } - public function rmdir($path) { + public function rmdir(string $path): bool { $path = $this->normalizePath($path); + $entry = $this->getCache()->get($path); - if (!$this->is_dir($path)) { + if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) { return false; } - if (!$this->rmObjects($path)) { - return false; - } - - $this->getCache()->remove($path); - - return true; + return $this->rmObjects($entry); } - private function rmObjects($path) { - $children = $this->getCache()->getFolderContents($path); + private function rmObjects(ICacheEntry $entry): bool { + $children = $this->getCache()->getFolderContentsById($entry->getId()); foreach ($children as $child) { - if ($child['mimetype'] === 'httpd/unix-directory') { - if (!$this->rmObjects($child['path'])) { + if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + if (!$this->rmObjects($child)) { return false; } } else { - if (!$this->unlink($child['path'])) { + if (!$this->rmObject($child)) { return false; } } } + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + return true; } - public function unlink($path) { + public function unlink(string $path): bool { $path = $this->normalizePath($path); - $stat = $this->stat($path); + $entry = $this->getCache()->get($path); - if ($stat && isset($stat['fileid'])) { - if ($stat['mimetype'] === 'httpd/unix-directory') { - return $this->rmdir($path); + if ($entry instanceof ICacheEntry) { + if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + return $this->rmObjects($entry); + } else { + return $this->rmObject($entry); } - try { - $this->objectStore->deleteObject($this->getURN($stat['fileid'])); - } catch (\Exception $ex) { - if ($ex->getCode() !== 404) { - $this->logger->logException($ex, [ + } + return false; + } + + public function rmObject(ICacheEntry $entry): bool { + try { + $this->objectStore->deleteObject($this->getURN($entry->getId())); + } catch (\Exception $ex) { + if ($ex->getCode() !== 404) { + $this->logger->error( + 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(), + [ 'app' => 'objectstore', - 'message' => 'Could not delete object ' . $this->getURN($stat['fileid']) . ' for ' . $path, - ]); - return false; - } - //removing from cache is ok as it does not exist in the objectstore anyway + 'exception' => $ex, + ] + ); + return false; } - $this->getCache()->remove($path); - return true; + //removing from cache is ok as it does not exist in the objectstore anyway } - return false; + if (!$this->preserveCacheItemsOnDelete) { + $this->getCache()->remove($entry->getPath()); + } + return true; } - public function stat($path) { + public function stat(string $path): array|false { $path = $this->normalizePath($path); $cacheEntry = $this->getCache()->get($path); if ($cacheEntry instanceof CacheEntry) { return $cacheEntry->getData(); } else { + if ($path === '') { + $this->mkdir('', true); + $cacheEntry = $this->getCache()->get($path); + if ($cacheEntry instanceof CacheEntry) { + return $cacheEntry->getData(); + } + } return false; } } - public function getPermissions($path) { + public function getPermissions(string $path): int { $stat = $this->stat($path); if (is_array($stat) && isset($stat['permissions'])) { @@ -248,17 +244,13 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users. * You may need a mapping table to store your URN if it cannot be generated from the fileid. * - * @param int $fileId the fileid - * @return null|string the unified resource name used to identify the object + * @return string the unified resource name used to identify the object */ - public function getURN($fileId) { - if (is_numeric($fileId)) { - return $this->objectPrefix . $fileId; - } - return null; + public function getURN(int $fileId): string { + return $this->objectPrefix . $fileId; } - public function opendir($path) { + public function opendir(string $path) { $path = $this->normalizePath($path); try { @@ -270,12 +262,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return IteratorDirectory::wrap($files); } catch (\Exception $e) { - $this->logger->logException($e); + $this->logger->error($e->getMessage(), ['exception' => $e]); return false; } } - public function filetype($path) { + public function filetype(string $path): string|false { $path = $this->normalizePath($path); $stat = $this->stat($path); if ($stat) { @@ -288,7 +280,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - public function fopen($path, $mode) { + public function fopen(string $path, string $mode) { $path = $this->normalizePath($path); if (strrpos($path, '.') !== false) { @@ -302,38 +294,61 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { case 'rb': $stat = $this->stat($path); if (is_array($stat)) { + $filesize = $stat['size'] ?? 0; // Reading 0 sized files is a waste of time - if (isset($stat['size']) && $stat['size'] === 0) { + if ($filesize === 0) { return fopen('php://memory', $mode); } try { - return $this->objectStore->readObject($this->getURN($stat['fileid'])); + $handle = $this->objectStore->readObject($this->getURN($stat['fileid'])); + if ($handle === false) { + return false; // keep backward compatibility + } + $streamStat = fstat($handle); + $actualSize = $streamStat['size'] ?? -1; + if ($actualSize > -1 && $actualSize !== $filesize) { + $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]); + } + return $handle; } catch (NotFoundException $e) { - $this->logger->logException($e, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); throw $e; - } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, - ]); + } catch (\Exception $e) { + $this->logger->error( + 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); return false; } } else { return false; } - // no break + // no break case 'w': case 'wb': case 'w+': case 'wb+': + $dirName = dirname($path); + $parentExists = $this->is_dir($dirName); + if (!$parentExists) { + return false; + } + $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext); $handle = fopen($tmpFile, $mode); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); + unlink($tmpFile); }); case 'a': case 'ab': @@ -351,17 +366,18 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $handle = fopen($tmpFile, $mode); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); + unlink($tmpFile); }); } return false; } - public function file_exists($path) { + public function file_exists(string $path): bool { $path = $this->normalizePath($path); return (bool)$this->stat($path); } - public function rename($source, $target) { + public function rename(string $source, string $target): bool { $source = $this->normalizePath($source); $target = $this->normalizePath($target); $this->remove($target); @@ -370,12 +386,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { return true; } - public function getMimeType($path) { + public function getMimeType(string $path): string|false { $path = $this->normalizePath($path); return parent::getMimeType($path); } - public function touch($path, $mtime = null) { + public function touch(string $path, ?int $mtime = null): bool { if (is_null($mtime)) { $mtime = time(); } @@ -397,55 +413,48 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { //create a empty file, need to have at least on char to make it // work with all object storage implementations $this->file_put_contents($path, ' '); - $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $stat = [ - 'etag' => $this->getETag($path), - 'mimetype' => $mimeType, - 'size' => 0, - 'mtime' => $mtime, - 'storage_mtime' => $mtime, - 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE, - ]; - $this->getCache()->put($path, $stat); } catch (\Exception $ex) { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object for ' . $path, - ]); + $this->logger->error( + 'Could not create object for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); throw $ex; } } return true; } - public function writeBack($tmpFile, $path) { + public function writeBack(string $tmpFile, string $path) { $size = filesize($tmpFile); $this->writeStream($path, fopen($tmpFile, 'r'), $size); } - /** - * external changes are not supported, exclusive access to the object storage is assumed - * - * @param string $path - * @param int $time - * @return false - */ - public function hasUpdated($path, $time) { + public function hasUpdated(string $path, int $time): bool { return false; } - public function needsPartFile() { + public function needsPartFile(): bool { return false; } - public function file_put_contents($path, $data) { - $handle = $this->fopen($path, 'w+'); - $result = fwrite($handle, $data); - fclose($handle); - return $result; + public function file_put_contents(string $path, mixed $data): int { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $data); + rewind($fh); + return $this->writeStream($path, $fh, strlen($data)); } - public function writeStream(string $path, $stream, int $size = null): int { + public function writeStream(string $path, $stream, ?int $size = null): int { + if ($size === null) { + $stats = fstat($stream); + if (is_array($stats) && isset($stats['size'])) { + $size = $stats['size']; + } + } + $stat = $this->stat($path); if (empty($stat)) { // create new file @@ -461,9 +470,18 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + 'original-storage' => $this->getId(), + 'original-path' => $path, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); + $stat['checksum'] = ''; $exists = $this->getCache()->inCache($path); $uploadPath = $exists ? $path : $path . '.part'; @@ -471,27 +489,37 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { if ($exists) { $fileId = $stat['fileid']; } else { + $parent = $this->normalizePath(dirname($path)); + if (!$this->is_dir($parent)) { + throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)"); + } $fileId = $this->getCache()->put($uploadPath, $stat); } $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream); + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); } + if (is_resource($countStream)) { + fclose($countStream); + } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -499,23 +527,31 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { * Else people lose access to existing files */ $this->getCache()->remove($uploadPath); - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not create object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not create object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } else { - $this->logger->logException($ex, [ - 'app' => 'objectstore', - 'message' => 'Could not update object ' . $urn . ' for ' . $path, - ]); + $this->logger->error( + 'Could not update object ' . $urn . ' for ' . $path, + [ + 'app' => 'objectstore', + 'exception' => $ex, + ] + ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { + // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways + $stat['unencrypted_size'] = $stat['size']; $this->getCache()->update($fileId, $stat); } else { - if ($this->objectStore->objectExists($urn)) { + if (!$this->validateWrites || $this->objectStore->objectExists($urn)) { $this->getCache()->move($uploadPath, $path); } else { $this->getCache()->remove($uploadPath); @@ -523,39 +559,148 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { return $this->objectStore; } - public function copy($path1, $path2) { - $path1 = $this->normalizePath($path1); - $path2 = $this->normalizePath($path2); + public function copyFromStorage( + IStorage $sourceStorage, + string $sourceInternalPath, + string $targetInternalPath, + bool $preserveMtime = false, + ): bool { + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) { + /** @var CacheEntry $sourceEntry */ + $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath); + $sourceEntryData = $sourceEntry->getData(); + // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current + // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the + // unjailed storage. + if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) { + $sourceEntry['permissions'] = $sourceEntryData['scan_permissions']; + } + $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath); + return true; + } + } + + return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath); + } + + public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool { + $sourceCache = $sourceStorage->getCache(); + if ( + $sourceStorage->instanceOfStorage(ObjectStoreStorage::class) + && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId() + ) { + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + // Do not import any data when source and target bucket are identical. + return true; + } + if (!$sourceCacheEntry) { + $sourceCacheEntry = $sourceCache->get($sourceInternalPath); + } + + $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry); + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(true); + } + if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) { + $sourceStorage->rmdir($sourceInternalPath); + } else { + $sourceStorage->unlink($sourceInternalPath); + } + if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) { + /** @var ObjectStoreStorage $sourceStorage */ + $sourceStorage->setPreserveCacheOnDelete(false); + } + if ($this->getCache()->get($targetInternalPath)) { + $this->unlink($targetInternalPath); + $this->getCache()->remove($targetInternalPath); + } + $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath); + + return true; + } + + /** + * Copy the object(s) of a file or folder into this storage, without touching the cache + */ + private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) { + $copiedFiles = []; + try { + foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) { + $sourceStream = $sourceStorage->fopen($file->getPath(), 'r'); + if (!$sourceStream) { + throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})"); + } + $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType()); + if (is_resource($sourceStream)) { + fclose($sourceStream); + } + $copiedFiles[] = $file->getId(); + } + } catch (\Exception $e) { + foreach ($copiedFiles as $fileId) { + try { + $this->objectStore->deleteObject($this->getURN($fileId)); + } catch (\Exception $e) { + // ignore + } + } + throw $e; + } + } + + /** + * @return \Iterator<ICacheEntry> + */ + private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + foreach ($cache->getFolderContentsById($entry->getId()) as $child) { + yield from $this->getAllChildObjects($cache, $child); + } + } else { + yield $entry; + } + } + + public function copy(string $source, string $target): bool { + $source = $this->normalizePath($source); + $target = $this->normalizePath($target); $cache = $this->getCache(); - $sourceEntry = $cache->get($path1); + $sourceEntry = $cache->get($source); if (!$sourceEntry) { throw new NotFoundException('Source object not found'); } - $this->copyInner($sourceEntry, $path2); + $this->copyInner($cache, $sourceEntry, $target); return true; } - private function copyInner(ICacheEntry $sourceEntry, string $to) { + private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) { $cache = $this->getCache(); if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { if ($cache->inCache($to)) { $cache->remove($to); } - $this->mkdir($to); + $this->mkdir($to, false, ['size' => $sourceEntry->getSize()]); - foreach ($cache->getFolderContentsById($sourceEntry->getId()) as $child) { - $this->copyInner($child, $to . '/' . $child->getName()); + foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) { + $this->copyInner($sourceCache, $child, $to . '/' . $child->getName()); } } else { $this->copyFile($sourceEntry, $to); @@ -567,21 +712,104 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { $sourceUrn = $this->getURN($sourceEntry->getId()); - $cache->copyFromCache($cache, $sourceEntry, $to); - $targetEntry = $cache->get($to); - - if (!$targetEntry) { - throw new \Exception('Target not in cache after copy'); + if (!$cache instanceof Cache) { + throw new \Exception('Invalid source cache for object store copy'); } - $targetUrn = $this->getURN($targetEntry->getId()); + $targetId = $cache->copyFromCache($cache, $sourceEntry, $to); + + $targetUrn = $this->getURN($targetId); try { $this->objectStore->copyObject($sourceUrn, $targetUrn); + if ($this->handleCopiesAsOwned) { + // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage ! + $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]); + } } catch (\Exception $e) { $cache->remove($to); throw $e; } } + + public function startChunkedWrite(string $targetPath): string { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + return $this->objectStore->initiateMultipartUpload($urn); + } + + /** + * @throws GenericFileException + */ + public function putChunkedWritePart( + string $targetPath, + string $writeToken, + string $chunkId, + $data, + $size = null, + ): ?array { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + + $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size); + + $parts[$chunkId] = [ + 'PartNumber' => $chunkId, + 'ETag' => trim($result->get('ETag'), '"'), + ]; + return $parts[$chunkId]; + } + + public function completeChunkedWrite(string $targetPath, string $writeToken): int { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $parts = $this->objectStore->getMultipartUploads($urn, $writeToken); + $sortedParts = array_values($parts); + sort($sortedParts); + try { + $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts); + $stat = $this->stat($targetPath); + $mtime = time(); + if (is_array($stat)) { + $stat['size'] = $size; + $stat['mtime'] = $mtime; + $stat['mimetype'] = $this->getMimeType($targetPath); + $this->getCache()->update($stat['fileid'], $stat); + } + } catch (S3MultipartUploadException|S3Exception $e) { + $this->objectStore->abortMultipartUpload($urn, $writeToken); + $this->logger->error( + 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken, + [ + 'app' => 'objectstore', + 'exception' => $e, + ] + ); + throw new GenericFileException('Could not write chunked file'); + } + return $size; + } + + public function cancelChunkedWrite(string $targetPath, string $writeToken): void { + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $this->objectStore->abortMultipartUpload($urn, $writeToken); + } + + public function setPreserveCacheOnDelete(bool $preserve) { + $this->preserveCacheItemsOnDelete = $preserve; + } } |