From eefd0597166b26b85690fba784d843486badefef Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 8 Aug 2016 15:50:15 +0200 Subject: add amazon s3 objectstore backend Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3.php | 116 +++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 lib/private/Files/ObjectStore/S3.php (limited to 'lib/private/Files/ObjectStore/S3.php') diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php new file mode 100644 index 00000000000..641c1722b58 --- /dev/null +++ b/lib/private/Files/ObjectStore/S3.php @@ -0,0 +1,116 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +use Guzzle\Http\EntityBody; +use Guzzle\Http\Message\RequestInterface; +use Guzzle\Service\Command\CommandInterface; +use Guzzle\Stream\PhpStreamRequestFactory; +use Icewind\Streams\CallbackWrapper; +use OCP\Files\ObjectStore\IObjectStore; + +// TODO: proper composer +set_include_path(get_include_path() . PATH_SEPARATOR . + \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php'); +require_once 'aws-autoloader.php'; + +class S3 implements IObjectStore { + use S3ConnectionTrait; + + public function __construct($parameters) { + $this->parseParams($parameters); + } + + /** + * @return string the container or bucket name where objects are stored + * @since 7.0.0 + */ + function getStorageId() { + return $this->id; + } + + /** + * Serialize and sign a command, returning a request object + * + * @param CommandInterface $command Command to sign + * + * @return RequestInterface + */ + protected function getSignedRequest($command) { + $request = $command->prepare(); + $request->dispatch('request.before_send', array('request' => $request)); + + return $request; + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return resource stream with the read data + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function readObject($urn) { + // Create the command and serialize the request + $request = $this->getSignedRequest($this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn + ])); + // Create a stream that uses the EntityBody object + $factory = new PhpStreamRequestFactory(); + /** @var EntityBody $body */ + $body = $factory->fromRequest($request, array(), array('stream_class' => 'Guzzle\Http\EntityBody')); + $stream = $body->getStream(); + + // we need to keep the guzzle request in scope untill the stream is closed + return CallbackWrapper::wrap($stream, null, null, function () use ($body) { + $body->close(); + }); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function writeObject($urn, $stream) { + $this->getConnection()->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream + ]); + } + + /** + * @param string $urn the unified resource name used to identify the object + * @return void + * @throws \Exception when something goes wrong, message will be logged + * @since 7.0.0 + */ + function deleteObject($urn) { + $this->getConnection()->deleteObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ]); + } + +} -- cgit v1.2.3 From 3ab160dd5a9071bb993fe95361d29f1f8d74ab93 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 28 Oct 2016 16:35:39 +0200 Subject: fopen s3 objects directly to work around unexplainable guzzle bug For some reason when a text file started with a valid hex character ([0-9a-f]) it would eat the text untill the first newline The new code does basically the same thing as guzzle/s3-sdk did only without wrapping everything in a guzzle stream Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3.php | 51 +++++++++++++++--------------------- 1 file changed, 21 insertions(+), 30 deletions(-) (limited to 'lib/private/Files/ObjectStore/S3.php') diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 641c1722b58..5251b473bdf 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -21,11 +21,6 @@ namespace OC\Files\ObjectStore; -use Guzzle\Http\EntityBody; -use Guzzle\Http\Message\RequestInterface; -use Guzzle\Service\Command\CommandInterface; -use Guzzle\Stream\PhpStreamRequestFactory; -use Icewind\Streams\CallbackWrapper; use OCP\Files\ObjectStore\IObjectStore; // TODO: proper composer @@ -48,20 +43,6 @@ class S3 implements IObjectStore { return $this->id; } - /** - * Serialize and sign a command, returning a request object - * - * @param CommandInterface $command Command to sign - * - * @return RequestInterface - */ - protected function getSignedRequest($command) { - $request = $command->prepare(); - $request->dispatch('request.before_send', array('request' => $request)); - - return $request; - } - /** * @param string $urn the unified resource name used to identify the object * @return resource stream with the read data @@ -70,20 +51,30 @@ class S3 implements IObjectStore { */ function readObject($urn) { // Create the command and serialize the request - $request = $this->getSignedRequest($this->getConnection()->getCommand('GetObject', [ + $request = $this->getConnection()->getCommand('GetObject', [ 'Bucket' => $this->bucket, 'Key' => $urn - ])); - // Create a stream that uses the EntityBody object - $factory = new PhpStreamRequestFactory(); - /** @var EntityBody $body */ - $body = $factory->fromRequest($request, array(), array('stream_class' => 'Guzzle\Http\EntityBody')); - $stream = $body->getStream(); + ])->prepare(); + + $request->dispatch('request.before_send', array( + 'request' => $request + )); + + $headers = $request->getHeaderLines(); + $headers[] = 'Connection: close'; + + $opts = [ + 'http' => [ + 'method' => "GET", + 'header' => $headers + ], + 'ssl' => [ + 'verify_peer' => true + ] + ]; - // we need to keep the guzzle request in scope untill the stream is closed - return CallbackWrapper::wrap($stream, null, null, function () use ($body) { - $body->close(); - }); + $context = stream_context_create($opts); + return fopen($request->getUrl(), 'r', false, $context); } /** -- cgit v1.2.3