From dad18baec8274995ff384789ccc2577aabc6dde8 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 20 Apr 2017 17:08:22 +0200 Subject: update aws sdk and move it to 3rdparty Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3.php | 38 ++++++---------------- .../Files/ObjectStore/S3ConnectionTrait.php | 24 +++++++------- 2 files changed, 21 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 6f37492271c..17caf1cb126 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -22,11 +22,7 @@ namespace OC\Files\ObjectStore; use OCP\Files\ObjectStore\IObjectStore; - -// TODO: proper composer -set_include_path(get_include_path() . PATH_SEPARATOR . - \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php'); -require_once 'aws-autoloader.php'; +use Psr\Http\Message\StreamInterface; class S3 implements IObjectStore { use S3ConnectionTrait; @@ -49,32 +45,18 @@ class S3 implements IObjectStore { * @throws \Exception when something goes wrong, message will be logged * @since 7.0.0 */ - public function readObject($urn) { - // Create the command and serialize the request - $request = $this->getConnection()->getCommand('GetObject', [ + function readObject($urn) { + $client = $this->getConnection(); + $command = $client->getCommand('GetObject', [ 'Bucket' => $this->bucket, 'Key' => $urn - ])->prepare(); - - $request->dispatch('request.before_send', array( - 'request' => $request - )); - - $headers = $request->getHeaderLines(); - $headers[] = 'Connection: close'; - - $opts = [ - 'http' => [ - 'method' => "GET", - 'header' => $headers - ], - 'ssl' => [ - 'verify_peer' => true - ] - ]; + ]); + $command['@http']['stream'] = true; + $result = $client->execute($command); + /** @var StreamInterface $body */ + $body = $result['Body']; - $context = stream_context_create($opts); - return fopen($request->getUrl(), 'r', false, $context); + return $body->detach(); } /** diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index a8b57cb18d3..f665bc2e7ce 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -53,7 +53,7 @@ trait S3ConnectionTrait { $this->bucket = $params['bucket']; $this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout']; $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; - $params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname']; + $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } @@ -76,20 +76,23 @@ trait S3ConnectionTrait { $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; $options = [ - 'key' => $this->params['key'], - 'secret' => $this->params['secret'], - 'base_url' => $base_url, + 'version' => isset($this->params['version']) ? $this->params['version'] : 'latest', + 'credentials' => [ + 'key' => $this->params['key'], + 'secret' => $this->params['secret'], + ], + 'endpoint' => $base_url, 'region' => $this->params['region'], - S3Client::COMMAND_PARAMS => [ + 'command.params' => [ 'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, ] ]; if (isset($this->params['proxy'])) { - $options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']]; + $options['request.options'] = ['proxy' => $this->params['proxy']]; } - $this->connection = S3Client::factory($options); + $this->connection = new S3Client($options); - if (!$this->connection->isValidBucketName($this->bucket)) { + if (!S3Client::isBucketDnsCompatible($this->bucket)) { throw new \Exception("The configured bucket name is invalid."); } @@ -98,11 +101,6 @@ trait S3ConnectionTrait { $this->connection->createBucket(array( 'Bucket' => $this->bucket )); - $this->connection->waitUntilBucketExists(array( - 'Bucket' => $this->bucket, - 'waiter.interval' => 1, - 'waiter.max_attempts' => 15 - )); $this->testTimeout(); } catch (S3Exception $e) { \OCP\Util::logException('files_external', $e); -- cgit v1.2.3 From d70607104e7000ba008b4eb20bcf58a47b88fe35 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 7 Jun 2017 16:29:13 +0200 Subject: reuse object read/write/delete logic in s3 implementations Signed-off-by: Robin Appelman --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 49 ++++---------- lib/private/Files/ObjectStore/S3.php | 50 +-------------- lib/private/Files/ObjectStore/S3ObjectTrait.php | 82 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 84 deletions(-) create mode 100644 lib/private/Files/ObjectStore/S3ObjectTrait.php (limited to 'lib') diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index edcc084ae44..f26dcc3de02 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -41,14 +41,11 @@ use Aws\S3\Exception\S3Exception; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\IteratorDirectory; use OC\Files\ObjectStore\S3ConnectionTrait; +use OC\Files\ObjectStore\S3ObjectTrait; class AmazonS3 extends \OC\Files\Storage\Common { use S3ConnectionTrait; - - /** - * @var array - */ - private static $tmpFiles = array(); + use S3ObjectTrait; /** * @var int in seconds @@ -91,7 +88,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { * * @param array $params */ - public function updateLegacyId (array $params) { + public function updateLegacyId(array $params) { $oldId = 'amazon::' . $params['key'] . md5($params['secret']); // find by old id or bucket @@ -187,7 +184,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { return false; } - private function batchDelete ($path = null) { + private function batchDelete($path = null) { $params = array( 'Bucket' => $this->bucket ); @@ -283,7 +280,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { $stat['atime'] = time(); return $stat; - } catch(S3Exception $e) { + } catch (S3Exception $e) { \OCP\Util::logException('files_external', $e); return false; } @@ -300,7 +297,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { if ($this->getConnection()->doesObjectExist($this->bucket, $path)) { return 'file'; } - if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) { + if ($this->getConnection()->doesObjectExist($this->bucket, $path . '/')) { return 'dir'; } } catch (S3Exception $e) { @@ -319,11 +316,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { } try { - $this->getConnection()->deleteObject(array( - 'Bucket' => $this->bucket, - 'Key' => $path - )); - $this->testTimeout(); + $this->deleteObject($path); } catch (S3Exception $e) { \OCP\Util::logException('files_external', $e); return false; @@ -338,21 +331,12 @@ class AmazonS3 extends \OC\Files\Storage\Common { switch ($mode) { case 'r': case 'rb': - $tmpFile = \OCP\Files::tmpFile(); - self::$tmpFiles[$tmpFile] = $path; - try { - $this->getConnection()->getObject(array( - 'Bucket' => $this->bucket, - 'Key' => $path, - 'SaveAs' => $tmpFile - )); + return $this->readObject($path); } catch (S3Exception $e) { \OCP\Util::logException('files_external', $e); return false; } - - return fopen($tmpFile, 'r'); case 'w': case 'wb': case 'a': @@ -372,12 +356,12 @@ class AmazonS3 extends \OC\Files\Storage\Common { } $tmpFile = \OCP\Files::tmpFile($ext); if ($this->file_exists($path)) { - $source = $this->fopen($path, 'r'); + $source = $this->readObject($path); file_put_contents($tmpFile, $source); } $handle = fopen($tmpFile, $mode); - return CallbackWrapper::wrap($handle, null, null, function() use ($path, $tmpFile) { + return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) { $this->writeBack($tmpFile, $path); }); } @@ -398,7 +382,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { $fileType = $this->filetype($path); try { if ($fileType !== false) { - if ($fileType === 'dir' && ! $this->isRoot($path)) { + if ($fileType === 'dir' && !$this->isRoot($path)) { $path .= '/'; } $this->getConnection()->copyObject([ @@ -522,14 +506,9 @@ class AmazonS3 extends \OC\Files\Storage\Common { public function writeBack($tmpFile, $path) { try { - $this->getConnection()->putObject(array( - 'Bucket' => $this->bucket, - 'Key' => $this->cleanKey($path), - 'SourceFile' => $tmpFile, - 'ContentType' => \OC::$server->getMimeTypeDetector()->detect($tmpFile), - 'ContentLength' => filesize($tmpFile) - )); - $this->testTimeout(); + $source = $this->fopen($tmpFile, 'r'); + $this->writeObject($path, $source); + fclose($source); unlink($tmpFile); } catch (S3Exception $e) { diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 17caf1cb126..e4a7b068b65 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -22,10 +22,10 @@ namespace OC\Files\ObjectStore; use OCP\Files\ObjectStore\IObjectStore; -use Psr\Http\Message\StreamInterface; class S3 implements IObjectStore { use S3ConnectionTrait; + use S3ObjectTrait; public function __construct($parameters) { $this->parseParams($parameters); @@ -38,52 +38,4 @@ class S3 implements IObjectStore { public function getStorageId() { return $this->id; } - - /** - * @param string $urn the unified resource name used to identify the object - * @return resource stream with the read data - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ - function readObject($urn) { - $client = $this->getConnection(); - $command = $client->getCommand('GetObject', [ - 'Bucket' => $this->bucket, - 'Key' => $urn - ]); - $command['@http']['stream'] = true; - $result = $client->execute($command); - /** @var StreamInterface $body */ - $body = $result['Body']; - - return $body->detach(); - } - - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ - public function writeObject($urn, $stream) { - $this->getConnection()->putObject([ - 'Bucket' => $this->bucket, - 'Key' => $urn, - 'Body' => $stream - ]); - } - - /** - * @param string $urn the unified resource name used to identify the object - * @return void - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ - public function deleteObject($urn) { - $this->getConnection()->deleteObject([ - 'Bucket' => $this->bucket, - 'Key' => $urn - ]); - } - } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php new file mode 100644 index 00000000000..3ba4da92b98 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -0,0 +1,82 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +use Aws\S3\S3Client; +use Psr\Http\Message\StreamInterface; + +trait S3ObjectTrait { + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ + abstract protected function getConnection(); + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + $client = $this->getConnection(); + $command = $client->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn + ]); + $command['@http']['stream'] = true; + $result = $client->execute($command); + /** @var StreamInterface $body */ + $body = $result['Body']; + + return $body->detach(); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $this->getConnection()->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream + ]); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->getConnection()->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ]); + } +} \ No newline at end of file -- cgit v1.2.3 From d526969a686b550248da90257729326b463b167a Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 31 Aug 2017 15:28:40 +0200 Subject: fix path style Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3ConnectionTrait.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index f665bc2e7ce..fdda19ff700 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -83,9 +83,7 @@ trait S3ConnectionTrait { ], 'endpoint' => $base_url, 'region' => $this->params['region'], - 'command.params' => [ - 'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false, - ] + 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false ]; if (isset($this->params['proxy'])) { $options['request.options'] = ['proxy' => $this->params['proxy']]; -- cgit v1.2.3 From e7f3c12b203c7afdf22423bb4d2bf756b2819cd1 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 18 Sep 2017 14:18:09 +0200 Subject: update autoloader Signed-off-by: Robin Appelman --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + 2 files changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 425eeb6ce7b..e88471f9461 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -608,6 +608,7 @@ return array( 'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\S3' => $baseDir . '/lib/private/Files/ObjectStore/S3.php', 'OC\\Files\\ObjectStore\\S3ConnectionTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php', + 'OC\\Files\\ObjectStore\\S3ObjectTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ObjectTrait.php', 'OC\\Files\\ObjectStore\\StorageObjectStore' => $baseDir . '/lib/private/Files/ObjectStore/StorageObjectStore.php', 'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php', 'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 68262fafb0d..17c9c8129da 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -638,6 +638,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\S3' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3.php', 'OC\\Files\\ObjectStore\\S3ConnectionTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php', + 'OC\\Files\\ObjectStore\\S3ObjectTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ObjectTrait.php', 'OC\\Files\\ObjectStore\\StorageObjectStore' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/StorageObjectStore.php', 'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php', 'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php', -- cgit v1.2.3