*/ private CappedMemoryCache $objectCache; /** @var CappedMemoryCache */ private CappedMemoryCache $directoryCache; /** @var CappedMemoryCache */ private CappedMemoryCache $filesCache; private IMimeTypeDetector $mimeDetector; private ?bool $versioningEnabled = null; private ICache $memCache; public function __construct($parameters) { parent::__construct($parameters); $this->parseParams($parameters); $this->id = 'amazon::external::' . md5($this->params['hostname'] . ':' . $this->params['bucket'] . ':' . $this->params['key']); $this->objectCache = new CappedMemoryCache(); $this->directoryCache = new CappedMemoryCache(); $this->filesCache = new CappedMemoryCache(); $this->mimeDetector = Server::get(IMimeTypeDetector::class); /** @var ICacheFactory $cacheFactory */ $cacheFactory = Server::get(ICacheFactory::class); $this->memCache = $cacheFactory->createLocal('s3-external'); $this->logger = Server::get(LoggerInterface::class); } /** * @param string $path * @return string correctly encoded path */ private function normalizePath($path) { $path = trim($path, '/'); if (!$path) { $path = '.'; } return $path; } private function isRoot($path) { return $path === '.'; } private function cleanKey($path) { if ($this->isRoot($path)) { return '/'; } return $path; } private function clearCache() { $this->objectCache = new CappedMemoryCache(); $this->directoryCache = new CappedMemoryCache(); $this->filesCache = new CappedMemoryCache(); } private function invalidateCache($key) { unset($this->objectCache[$key]); $keys = array_keys($this->objectCache->getData()); $keyLength = strlen($key); foreach ($keys as $existingKey) { if (substr($existingKey, 0, $keyLength) === $key) { unset($this->objectCache[$existingKey]); } } unset($this->filesCache[$key]); $keys = array_keys($this->directoryCache->getData()); $keyLength = strlen($key); foreach ($keys as $existingKey) { if (substr($existingKey, 0, $keyLength) === $key) { unset($this->directoryCache[$existingKey]); } } unset($this->directoryCache[$key]); } /** * @return array|false */ private function headObject(string $key) { if (!isset($this->objectCache[$key])) { try { $this->objectCache[$key] = $this->getConnection()->headObject([ 'Bucket' => $this->bucket, 'Key' => $key ])->toArray(); } catch (S3Exception $e) { if ($e->getStatusCode() >= 500) { throw $e; } $this->objectCache[$key] = false; } } if (is_array($this->objectCache[$key]) && !isset($this->objectCache[$key]["Key"])) { /** @psalm-suppress InvalidArgument Psalm doesn't understand nested arrays well */ $this->objectCache[$key]["Key"] = $key; } return $this->objectCache[$key]; } /** * Return true if directory exists * * There are no folders in s3. A folder like structure could be archived * by prefixing files with the folder name. * * Implementation from flysystem-aws-s3-v3: * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694 * * @param $path * @return bool * @throws \Exception */ private function doesDirectoryExist($path) { if ($path === '.' || $path === '') { return true; } $path = rtrim($path, '/') . '/'; if (isset($this->directoryCache[$path])) { return $this->directoryCache[$path]; } try { // Maybe this isn't an actual key, but a prefix. // Do a prefix listing of objects to determine. $result = $this->getConnection()->listObjectsV2([ 'Bucket' => $this->bucket, 'Prefix' => $path, 'MaxKeys' => 1, ]); if (isset($result['Contents'])) { $this->directoryCache[$path] = true; return true; } // empty directories have their own object $object = $this->headObject($path); if ($object) { $this->directoryCache[$path] = true; return true; } } catch (S3Exception $e) { if ($e->getStatusCode() >= 400 && $e->getStatusCode() < 500) { $this->directoryCache[$path] = false; } throw $e; } $this->directoryCache[$path] = false; return false; } /** * Remove a file or folder * * @param string $path * @return bool */ protected function remove($path) { // remember fileType to reduce http calls $fileType = $this->filetype($path); if ($fileType === 'dir') { return $this->rmdir($path); } elseif ($fileType === 'file') { return $this->unlink($path); } else { return false; } } public function mkdir($path) { $path = $this->normalizePath($path); if ($this->is_dir($path)) { return false; } try { $this->getConnection()->putObject([ 'Bucket' => $this->bucket, 'Key' => $path . '/', 'Body' => '', 'ContentType' => FileInfo::MIMETYPE_FOLDER ]); $this->testTimeout(); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } $this->invalidateCache($path); return true; } public function file_exists($path) { return $this->filetype($path) !== false; } public function rmdir($path) { $path = $this->normalizePath($path); if ($this->isRoot($path)) { return $this->clearBucket(); } if (!$this->file_exists($path)) { return false; } $this->invalidateCache($path); return $this->batchDelete($path); } protected function clearBucket() { $this->clearCache(); return $this->batchDelete(); } private function batchDelete($path = null) { // TODO explore using https://docs.aws.amazon.com/aws-sdk-php/v3/api/class-Aws.S3.BatchDelete.html $params = [ 'Bucket' => $this->bucket ]; if ($path !== null) { $params['Prefix'] = $path . '/'; } try { $connection = $this->getConnection(); // Since there are no real directories on S3, we need // to delete all objects prefixed with the path. do { // instead of the iterator, manually loop over the list ... $objects = $connection->listObjects($params); // ... so we can delete the files in batches if (isset($objects['Contents'])) { $connection->deleteObjects([ 'Bucket' => $this->bucket, 'Delete' => [ 'Objects' => $objects['Contents'] ] ]); $this->testTimeout(); } // we reached the end when the list is no longer truncated } while ($objects['IsTruncated']); if ($path !== '' && $path !== null) { $this->deleteObject($path); } } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } return true; } public function opendir($path) { try { $content = iterator_to_array($this->getDirectoryContent($path)); return IteratorDirectory::wrap(array_map(function (array $item) { return $item['name']; }, $content)); } catch (S3Exception $e) { return false; } } public function stat($path) { $path = $this->normalizePath($path); if ($this->is_dir($path)) { $stat = $this->getDirectoryMetaData($path); } else { $object = $this->headObject($path); if ($object === false) { return false; } $stat = $this->objectToMetaData($object); } $stat['atime'] = time(); return $stat; } /** * Return content length for object * * When the information is already present (e.g. opendir has been called before) * this value is return. Otherwise a headObject is emitted. * * @param $path * @return int|mixed */ private function getContentLength($path) { if (isset($this->filesCache[$path])) { return (int)$this->filesCache[$path]['ContentLength']; } $result = $this->headObject($path); if (isset($result['ContentLength'])) { return (int)$result['ContentLength']; } return 0; } /** * Return last modified for object * * When the information is already present (e.g. opendir has been called before) * this value is return. Otherwise a headObject is emitted. * * @param $path * @return mixed|string */ private function getLastModified($path) { if (isset($this->filesCache[$path])) { return $this->filesCache[$path]['LastModified']; } $result = $this->headObject($path); if (isset($result['LastModified'])) { return $result['LastModified']; } return 'now'; } public function is_dir($path) { $path = $this->normalizePath($path); if (isset($this->filesCache[$path])) { return false; } try { return $this->doesDirectoryExist($path); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } } public function filetype($path) { $path = $this->normalizePath($path); if ($this->isRoot($path)) { return 'dir'; } try { if (isset($this->directoryCache[$path]) && $this->directoryCache[$path]) { return 'dir'; } if (isset($this->filesCache[$path]) || $this->headObject($path)) { return 'file'; } if ($this->doesDirectoryExist($path)) { return 'dir'; } } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } return false; } public function getPermissions($path) { $type = $this->filetype($path); if (!$type) { return 0; } return $type === 'dir' ? Constants::PERMISSION_ALL : Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE; } public function unlink($path) { $path = $this->normalizePath($path); if ($this->is_dir($path)) { return $this->rmdir($path); } try { $this->deleteObject($path); $this->invalidateCache($path); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } return true; } public function fopen($path, $mode) { $path = $this->normalizePath($path); switch ($mode) { case 'r': case 'rb': // Don't try to fetch empty files $stat = $this->stat($path); if (is_array($stat) && isset($stat['size']) && $stat['size'] === 0) { return fopen('php://memory', $mode); } try { return $this->readObject($path); } catch (\Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } case 'w': case 'wb': $tmpFile = \OC::$server->getTempManager()->getTemporaryFile(); $handle = fopen($tmpFile, 'w'); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); }); case 'a': case 'ab': case 'r+': case 'w+': case 'wb+': case 'a+': case 'x': case 'x+': case 'c': case 'c+': if (strrpos($path, '.') !== false) { $ext = substr($path, strrpos($path, '.')); } else { $ext = ''; } $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext); if ($this->file_exists($path)) { $source = $this->readObject($path); file_put_contents($tmpFile, $source); } $handle = fopen($tmpFile, $mode); return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); }); } return false; } public function touch($path, $mtime = null) { if (is_null($mtime)) { $mtime = time(); } $metadata = [ 'lastmodified' => gmdate(\DateTime::RFC1123, $mtime) ]; try { if ($this->file_exists($path)) { return false; } $mimeType = $this->mimeDetector->detectPath($path); $this->getConnection()->putObject([ 'Bucket' => $this->bucket, 'Key' => $this->cleanKey($path), 'Metadata' => $metadata, 'Body' => '', 'ContentType' => $mimeType, 'MetadataDirective' => 'REPLACE', ]); $this->testTimeout(); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } $this->invalidateCache($path); return true; } public function copy($source, $target, $isFile = null) { $source = $this->normalizePath($source); $target = $this->normalizePath($target); if ($isFile === true || $this->is_file($source)) { try { $this->copyObject($source, $target, [ 'StorageClass' => $this->storageClass, ]); $this->testTimeout(); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } } else { $this->remove($target); try { $this->mkdir($target); $this->testTimeout(); } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } foreach ($this->getDirectoryContent($source) as $item) { $childSource = $source . '/' . $item['name']; $childTarget = $target . '/' . $item['name']; $this->copy($childSource, $childTarget, $item['mimetype'] !== FileInfo::MIMETYPE_FOLDER); } } $this->invalidateCache($target); return true; } public function rename($source, $target) { $source = $this->normalizePath($source); $target = $this->normalizePath($target); if ($this->is_file($source)) { if ($this->copy($source, $target) === false) { return false; } if ($this->unlink($source) === false) { $this->unlink($target); return false; } } else { if ($this->copy($source, $target) === false) { return false; } if ($this->rmdir($source) === false) { $this->rmdir($target); return false; } } return true; } public function test() { $this->getConnection()->headBucket([ 'Bucket' => $this->bucket ]); return true; } public function getId() { return $this->id; } public function writeBack($tmpFile, $path) { try { $source = fopen($tmpFile, 'r'); $this->writeObject($path, $source, $this->mimeDetector->detectPath($path)); $this->invalidateCache($path); unlink($tmpFile); return true; } catch (S3Exception $e) { $this->logger->error($e->getMessage(), [ 'app' => 'files_external', 'exception' => $e, ]); return false; } } /** * check if curl is installed */ public static function checkDependencies() { return true; } public function getDirectoryContent($directory): \Traversable { $path = $this->normalizePath($directory); if ($this->isRoot($path)) { $path = ''; } else { $path .= '/'; } $results = $this->getConnection()->getPaginator('ListObjectsV2', [ 'Bucket' => $this->bucket, 'Delimiter' => '/', 'Prefix' => $path, ]); foreach ($results as $result) { // sub folders if (is_array($result['CommonPrefixes'])) { foreach ($result['CommonPrefixes'] as $prefix) { $dir = $this->getDirectoryMetaData($prefix['Prefix']); if ($dir) { yield $dir; } } } if (is_array($result['Contents'])) { foreach ($result['Contents'] as $object) { $this->objectCache[$object['Key']] = $object; if ($object['Key'] !== $path) { yield $this->objectToMetaData($object); } } } } } private function objectToMetaData(array $object): array { return [ 'name' => basename($object['Key']), 'mimetype' => $this->mimeDetector->detectPath($object['Key']), 'mtime' => strtotime($object['LastModified']), 'storage_mtime' => strtotime($object['LastModified']), 'etag' => trim($object['ETag'], '"'), 'permissions' => Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE, 'size' => (int)($object['Size'] ?? $object['ContentLength']), ]; } private function getDirectoryMetaData(string $path): ?array { $path = trim($path, '/'); // when versioning is enabled, delete markers are returned as part of CommonPrefixes // resulting in "ghost" folders, verify that each folder actually exists if ($this->versioningEnabled() && !$this->doesDirectoryExist($path)) { return null; } $cacheEntry = $this->getCache()->get($path); if ($cacheEntry instanceof CacheEntry) { return $cacheEntry->getData(); } else { return [ 'name' => basename($path), 'mimetype' => FileInfo::MIMETYPE_FOLDER, 'mtime' => time(), 'storage_mtime' => time(), 'etag' => uniqid(), 'permissions' => Constants::PERMISSION_ALL, 'size' => -1, ]; } } public function versioningEnabled(): bool { if ($this->versioningEnabled === null) { $cached = $this->memCache->get('versioning-enabled::' . $this->getBucket()); if ($cached === null) { $this->versioningEnabled = $this->getVersioningStatusFromBucket(); $this->memCache->set('versioning-enabled::' . $this->getBucket(), $this->versioningEnabled, 60); } else { $this->versioningEnabled = $cached; } } return $this->versioningEnabled; } protected function getVersioningStatusFromBucket(): bool { try { $result = $this->getConnection()->getBucketVersioning(['Bucket' => $this->getBucket()]); return $result->get('Status') === 'Enabled'; } catch (S3Exception $s3Exception) { // This is needed for compatibility with Storj gateway which does not support versioning yet if ($s3Exception->getAwsErrorCode() === 'NotImplemented' || $s3Exception->getAwsErrorCode() === 'AccessDenied') { return false; } throw $s3Exception; } } public function hasUpdated($path, $time) { // for files we can get the proper mtime if ($path !== '' && $object = $this->headObject($path)) { $stat = $this->objectToMetaData($object); return $stat['mtime'] > $time; } else { // for directories, the only real option we have is to do a prefix listing and iterate over all objects // however, since this is just as expensive as just re-scanning the directory, we can simply return true // and have the scanner figure out if anything has actually changed return true; } } public function writeStream(string $path, $stream, ?int $size = null): int { if ($size === null) { $size = 0; // track the number of bytes read from the input stream to return as the number of written bytes. $stream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size) { $size = $writtenSize; }); } if (!is_resource($stream)) { throw new \InvalidArgumentException("Invalid stream provided"); } $path = $this->normalizePath($path); $this->writeObject($path, $stream, $this->mimeDetector->detectPath($path)); $this->invalidateCache($path); return $size; } } ef='#n419'>419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579