diff options
| -rw-r--r-- | .drone.yml | 3 | ||||
| m--------- | 3rdparty | 0 | ||||
| -rw-r--r-- | apps/files_external/lib/Lib/Storage/Swift.php | 247 | ||||
| -rw-r--r-- | apps/files_external/tests/Storage/SwiftTest.php | 16 | ||||
| -rw-r--r-- | lib/composer/composer/autoload_classmap.php | 1 | ||||
| -rw-r--r-- | lib/composer/composer/autoload_static.php | 1 | ||||
| -rw-r--r-- | lib/private/Files/ObjectStore/Swift.php | 243 | ||||
| -rw-r--r-- | lib/private/Files/ObjectStore/SwiftFactory.php | 174 | ||||
| -rw-r--r-- | tests/Settings/Controller/CheckSetupControllerTest.php | 6 | ||||
| -rwxr-xr-x | tests/drone-wait-objectstore.sh | 57 |
10 files changed, 362 insertions, 386 deletions
diff --git a/.drone.yml b/.drone.yml index 1c7eba5f0f2..9a0e7b25484 100644 --- a/.drone.yml +++ b/.drone.yml @@ -851,9 +851,8 @@ services: matrix: OBJECT_STORE: s3 dockswift: - image: icewind1991/dockswift + image: icewind1991/dockswift:nextcloud-ci environment: - - INITIALIZE=yes - IPADDRESS=dockswift when: matrix: diff --git a/3rdparty b/3rdparty -Subproject ffb6a3a10985ae6dca121200760a78b5a75aa5d +Subproject 600b0a7ab4c8f3c5e0fb48c68e171190f1c9006 diff --git a/apps/files_external/lib/Lib/Storage/Swift.php b/apps/files_external/lib/Lib/Storage/Swift.php index 88a1cf4c386..051c65350d5 100644 --- a/apps/files_external/lib/Lib/Storage/Swift.php +++ b/apps/files_external/lib/Lib/Storage/Swift.php @@ -1,4 +1,5 @@ <?php +declare(strict_types=1); /** * @copyright Copyright (c) 2016, ownCloud, Inc. * @@ -39,32 +40,22 @@ namespace OCA\Files_External\Lib\Storage; -use Guzzle\Http\Url; -use Guzzle\Http\Exception\ClientErrorResponseException; +use GuzzleHttp\Psr7\Uri; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\IteratorDirectory; -use Icewind\Streams\RetryWrapper; -use OpenCloud; -use OpenCloud\Common\Exceptions; -use OpenCloud\OpenStack; -use OpenCloud\Rackspace; -use OpenCloud\ObjectStore\Resource\DataObject; +use OC\Files\ObjectStore\SwiftFactory; +use OCP\Files\StorageBadConfigException; +use OpenStack\Common\Error\BadResponseError; +use OpenStack\ObjectStore\v1\Models\StorageObject; class Swift extends \OC\Files\Storage\Common { - - /** - * @var \OpenCloud\ObjectStore\Service - */ - private $connection; + /** @var SwiftFactory */ + private $connectionFactory; /** - * @var \OpenCloud\ObjectStore\Resource\Container + * @var \OpenStack\ObjectStore\v1\Models\Container */ private $container; /** - * @var \OpenCloud\OpenStack - */ - private $anchor; - /** * @var string */ private $bucket; @@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common { */ private $params; - /** @var string */ + /** @var string */ private $id; + /** @var \OC\Files\ObjectStore\Swift */ + private $objectStore; + /** * Key value cache mapping path to data object. Maps path to * \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing * paths and path to false for not existing paths. + * * @var \OCP\ICache */ private $objectCache; /** * @param string $path + * @return mixed|string */ - private function normalizePath($path) { + private function normalizePath(string $path) { $path = trim($path, '/'); if (!$path) { @@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common { * that one will be returned. * * @param string $path - * @return \OpenCloud\ObjectStore\Resource\DataObject|bool object + * @return StorageObject|bool object * or false if the object did not exist + * @throws \OCP\Files\StorageAuthException + * @throws \OCP\Files\StorageNotAvailableException */ - private function fetchObject($path) { + private function fetchObject(string $path) { if ($this->objectCache->hasKey($path)) { // might be "false" if object did not exist from last check return $this->objectCache->get($path); } try { - $object = $this->getContainer()->getPartialObject($path); + $object = $this->getContainer()->getObject($path); + $object->retrieve(); $this->objectCache->set($path, $object); return $object; - } catch (ClientErrorResponseException $e) { - // this exception happens when the object does not exist, which - // is expected in most cases - $this->objectCache->set($path, false); - return false; - } catch (ClientErrorResponseException $e) { + } catch (BadResponseError $e) { // Expected response is "404 Not Found", so only log if it isn't if ($e->getResponse()->getStatusCode() !== 404) { \OC::$server->getLogger()->logException($e, [ @@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common { 'app' => 'files_external', ]); } + $this->objectCache->set($path, false); return false; } } @@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common { * @param string $path * * @return bool true if the object exist, false otherwise + * @throws \OCP\Files\StorageAuthException + * @throws \OCP\Files\StorageNotAvailableException */ private function doesObjectExist($path) { return $this->fetchObject($path) !== false; @@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common { or empty($params['user']) or empty($params['bucket']) or empty($params['region']) ) { - throw new \Exception("API Key or password, Username, Bucket and Region have to be configured."); + throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured."); } $this->id = 'swift::' . $params['user'] . md5($params['bucket']); - $bucketUrl = Url::factory($params['bucket']); - if ($bucketUrl->isAbsolute()) { - $this->bucket = end($bucketUrl->getPathSegments()); - $params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath(); - } else { - $this->bucket = $params['bucket']; + $bucketUrl = new Uri($params['bucket']); + if ($bucketUrl->getHost()) { + $params['bucket'] = basename($bucketUrl->getPath()); + $params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath())); } if (empty($params['url'])) { @@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common { $params['service_name'] = 'cloudFiles'; } + $params['autocreate'] = true; + $this->params = $params; // FIXME: private class... $this->objectCache = new \OC\Cache\CappedMemoryCache(); + $this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params); + $this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory); + $this->bucket = $params['bucket']; } public function mkdir($path) { @@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common { } try { - $customHeaders = array('content-type' => 'httpd/unix-directory'); - $metadataHeaders = DataObject::stockHeaders(array()); - $allHeaders = $customHeaders + $metadataHeaders; - $this->getContainer()->uploadObject($path, '', $allHeaders); + $this->getContainer()->createObject([ + 'name' => $path, + 'content' => '', + 'headers' => ['content-type' => 'httpd/unix-directory'] + ]); // invalidate so that the next access gets the real object // with all properties $this->objectCache->remove($path); - } catch (Exceptions\CreateUpdateError $e) { + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common { } try { - $this->getContainer()->dataObject()->setName($path . '/')->delete(); + $this->objectStore->deleteObject($path . '/'); $this->objectCache->remove($path . '/'); - } catch (Exceptions\DeleteError $e) { + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common { $path .= '/'; } - $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of # +// $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of # try { - $files = array(); - /** @var OpenCloud\Common\Collection $objects */ - $objects = $this->getContainer()->objectList(array( + $files = []; + $objects = $this->getContainer()->listObjects([ 'prefix' => $path, 'delimiter' => '/' - )); + ]); - /** @var OpenCloud\ObjectStore\Resource\DataObject $object */ + /** @var StorageObject $object */ foreach ($objects as $object) { - $file = basename($object->getName()); + $file = basename($object->name); if ($file !== basename($path) && $file !== '.') { $files[] = $file; } @@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common { } try { - /** @var DataObject $object */ $object = $this->fetchObject($path); if (!$object) { return false; } - } catch (ClientErrorResponseException $e) { + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common { return false; } - $dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified()); - if ($dateTime !== false) { - $mtime = $dateTime->getTimestamp(); - } else { - $mtime = null; - } + $dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false; + $mtime = $dateTime ? $dateTime->getTimestamp() : null; $objectMetadata = $object->getMetadata(); - $metaTimestamp = $objectMetadata->getProperty('timestamp'); - if (isset($metaTimestamp)) { - $mtime = $metaTimestamp; + if (isset($objectMetadata['timestamp'])) { + $mtime = $objectMetadata['timestamp']; } if (!empty($mtime)) { @@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common { } $stat = array(); - $stat['size'] = (int)$object->getContentLength(); + $stat['size'] = (int)$object->contentLength; $stat['mtime'] = $mtime; $stat['atime'] = time(); return $stat; @@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common { } try { - $this->getContainer()->dataObject()->setName($path)->delete(); + $this->objectStore->deleteObject($path); $this->objectCache->remove($path); $this->objectCache->remove($path . '/'); - } catch (ClientErrorResponseException $e) { + } catch (BadResponseError $e) { if ($e->getResponse()->getStatusCode() !== 404) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', ]); + throw $e; } - return false; } return true; @@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common { case 'r': case 'rb': try { - $c = $this->getContainer(); - $streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory(); - /** @var \OpenCloud\Common\Http\Client $client */ - $client = $c->getClient(); - $streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path))); - $streamInterface->rewind(); - $stream = $streamInterface->getStream(); - stream_context_set_option($stream, 'swift','content', $streamInterface); - if(!strrpos($streamInterface - ->getMetaData('wrapper_data')[0], '404 Not Found')) { - return RetryWrapper::wrap($stream); - } - return false; - } catch (\Guzzle\Http\Exception\BadResponseException $e) { + return $this->objectStore->readObject($path); + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common { if (is_null($mtime)) { $mtime = time(); } - $metadata = array('timestamp' => $mtime); + $metadata = ['timestamp' => $mtime]; if ($this->file_exists($path)) { if ($this->is_dir($path) && $path !== '.') { $path .= '/'; } $object = $this->fetchObject($path); - if ($object->saveMetadata($metadata)) { + if ($object->mergeMetadata($metadata)) { // invalidate target object to force repopulation on fetch $this->objectCache->remove($path); } return true; } else { $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); - $customHeaders = array('content-type' => $mimeType); - $metadataHeaders = DataObject::stockHeaders($metadata); - $allHeaders = $customHeaders + $metadataHeaders; - $this->getContainer()->uploadObject($path, '', $allHeaders); + $this->getContainer()->createObject([ + 'name' => $path, + 'content' => '', + 'headers' => ['content-type' => 'httpd/unix-directory'] + ]); // invalidate target object to force repopulation on fetch $this->objectCache->remove($path); return true; @@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common { $path2 = $this->normalizePath($path2); $fileType = $this->filetype($path1); - if ($fileType === 'file') { - + if ($fileType) { // make way $this->unlink($path2); + } + if ($fileType === 'file') { try { $source = $this->fetchObject($path1); - $source->copy($this->bucket . '/' . $path2); + $source->copy([ + 'destination' => $this->bucket . '/' . $path2 + ]); // invalidate target object to force repopulation on fetch $this->objectCache->remove($path2); $this->objectCache->remove($path2 . '/'); - } catch (ClientErrorResponseException $e) { + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common { } } else if ($fileType === 'dir') { - - // make way - $this->unlink($path2); - try { $source = $this->fetchObject($path1 . '/'); - $source->copy($this->bucket . '/' . $path2 . '/'); + $source->copy([ + 'destination' => $this->bucket . '/' . $path2 . '/' + ]); // invalidate target object to force repopulation on fetch $this->objectCache->remove($path2); $this->objectCache->remove($path2 . '/'); - } catch (ClientErrorResponseException $e) { + } catch (BadResponseError $e) { \OC::$server->getLogger()->logException($e, [ 'level' => \OCP\Util::ERROR, 'app' => 'files_external', @@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common { // cleanup if ($this->unlink($path1) === false) { + throw new \Exception('failed to remove original'); $this->unlink($path2); return false; } @@ -568,80 +553,26 @@ class Swift extends \OC\Files\Storage\Common { } /** - * Returns the connection - * - * @return OpenCloud\ObjectStore\Service connected client - * @throws \Exception if connection could not be made - */ - public function getConnection() { - if (!is_null($this->connection)) { - return $this->connection; - } - - $settings = array( - 'username' => $this->params['user'], - ); - - if (!empty($this->params['password'])) { - $settings['password'] = $this->params['password']; - } else if (!empty($this->params['key'])) { - $settings['apiKey'] = $this->params['key']; - } - - if (!empty($this->params['tenant'])) { - $settings['tenantName'] = $this->params['tenant']; - } - - if (!empty($this->params['timeout'])) { - $settings['timeout'] = $this->params['timeout']; - } - - if (isset($settings['apiKey'])) { - $this->anchor = new Rackspace($this->params['url'], $settings); - } else { - $this->anchor = new OpenStack($this->params['url'], $settings); - } - - $connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']); - - if (!empty($this->params['endpoint_url'])) { - $endpoint = $connection->getEndpoint(); - $endpoint->setPublicUrl($this->params['endpoint_url']); - $endpoint->setPrivateUrl($this->params['endpoint_url']); - $connection->setEndpoint($endpoint); - } - - $this->connection = $connection; - - return $this->connection; - } - - /** * Returns the initialized object store container. * - * @return OpenCloud\ObjectStore\Resource\Container + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws \OCP\Files\StorageAuthException + * @throws \OCP\Files\StorageNotAvailableException */ public function getContainer() { - if (!is_null($this->container)) { - return $this->container; - } + if (is_null($this->container)) { + $this->container = $this->connectionFactory->getContainer(); - try { - $this->container = $this->getConnection()->getContainer($this->bucket); - } catch (ClientErrorResponseException $e) { - $this->container = $this->getConnection()->createContainer($this->bucket); - } - - if (!$this->file_exists('.')) { - $this->mkdir('.'); + if (!$this->file_exists('.')) { + $this->mkdir('.'); + } } - return $this->container; } public function writeBack($tmpFile, $path) { $fileData = fopen($tmpFile, 'r'); - $this->getContainer()->uploadObject($path, $fileData); + $this->objectStore->writeObject($path, $fileData); // invalidate target object to force repopulation on fetch $this->objectCache->remove($path); unlink($tmpFile); diff --git a/apps/files_external/tests/Storage/SwiftTest.php b/apps/files_external/tests/Storage/SwiftTest.php index 19f185ed033..fdda8baace8 100644 --- a/apps/files_external/tests/Storage/SwiftTest.php +++ b/apps/files_external/tests/Storage/SwiftTest.php @@ -27,6 +27,7 @@ namespace OCA\Files_External\Tests\Storage; +use GuzzleHttp\Exception\ClientException; use \OCA\Files_External\Lib\Storage\Swift; /** @@ -40,6 +41,11 @@ class SwiftTest extends \Test\Files\Storage\Storage { private $config; + /** + * @var Swift instance + */ + protected $instance; + protected function setUp() { parent::setUp(); @@ -53,17 +59,15 @@ class SwiftTest extends \Test\Files\Storage\Storage { protected function tearDown() { if ($this->instance) { try { - $connection = $this->instance->getConnection(); - $container = $connection->getContainer($this->config['bucket']); + $container = $this->instance->getContainer(); - $objects = $container->objectList(); - while($object = $objects->next()) { - $object->setName(str_replace('#','%23',$object->getName())); + $objects = $container->listObjects(); + foreach ($objects as $object) { $object->delete(); } $container->delete(); - } catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) { + } catch (ClientException $e) { // container didn't exist, so we don't need to delete it } } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 06879c5179a..b3158ac7e04 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -653,6 +653,7 @@ return array( 'OC\\Files\\ObjectStore\\S3Signature' => $baseDir . '/lib/private/Files/ObjectStore/S3Signature.php', 'OC\\Files\\ObjectStore\\StorageObjectStore' => $baseDir . '/lib/private/Files/ObjectStore/StorageObjectStore.php', 'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php', + 'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php', 'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php', 'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php', 'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 118094c30ef..65b36cc489e 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -683,6 +683,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\ObjectStore\\S3Signature' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3Signature.php', 'OC\\Files\\ObjectStore\\StorageObjectStore' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/StorageObjectStore.php', 'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php', + 'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php', 'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php', 'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php', 'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php', diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index a3cba488f5f..4451fbcc750 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -25,223 +25,37 @@ namespace OC\Files\ObjectStore; -use Guzzle\Http\Exception\ClientErrorResponseException; -use Guzzle\Http\Exception\CurlException; +use function GuzzleHttp\Psr7\stream_for; use Icewind\Streams\RetryWrapper; use OCP\Files\ObjectStore\IObjectStore; use OCP\Files\StorageAuthException; -use OCP\Files\StorageNotAvailableException; -use OpenCloud\Common\Service\Catalog; -use OpenCloud\Common\Service\CatalogItem; -use OpenCloud\Identity\Resource\Token; -use OpenCloud\ObjectStore\Service; -use OpenCloud\OpenStack; -use OpenCloud\Rackspace; class Swift implements IObjectStore { - - /** - * @var \OpenCloud\OpenStack - */ - private $client; - /** * @var array */ private $params; /** - * @var \OpenCloud\ObjectStore\Service - */ - private $objectStoreService; - - /** - * @var \OpenCloud\ObjectStore\Resource\Container + * @var \OpenStack\ObjectStore\v1\Models\Container|null */ - private $container; - - private $memcache; - - public function __construct($params) { - if (isset($params['bucket'])) { - $params['container'] = $params['bucket']; - } - if (!isset($params['container'])) { - $params['container'] = 'owncloud'; - } - if (!isset($params['autocreate'])) { - // should only be true for tests - $params['autocreate'] = false; - } - - if (isset($params['apiKey'])) { - $this->client = new Rackspace($params['url'], $params); - $cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket']; - } else { - $this->client = new OpenStack($params['url'], $params); - $cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket']; - } + private $container = null; - $cacheFactory = \OC::$server->getMemCacheFactory(); - $this->memcache = $cacheFactory->createDistributed('swift::' . $cacheKey); + /** @var SwiftFactory */ + private $swiftFactory; + public function __construct($params, SwiftFactory $connectionFactory = null) { + $this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params); $this->params = $params; } /** - * @suppress PhanNonClassMethodCall - */ - protected function init() { - if ($this->container) { - return; - } - - $this->importToken(); - - /** @var Token $token */ - $token = $this->client->getTokenObject(); - - if (!$token || $token->hasExpired()) { - try { - $this->client->authenticate(); - $this->exportToken(); - } catch (ClientErrorResponseException $e) { - $statusCode = $e->getResponse()->getStatusCode(); - if ($statusCode == 412) { - throw new StorageAuthException('Precondition failed, verify the keystone url', $e); - } else if ($statusCode === 401) { - throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e); - } else { - throw new StorageAuthException('Unknown error', $e); - } - } - } - - - /** @var Catalog $catalog */ - $catalog = $this->client->getCatalog(); - - if (count($catalog->getItems()) === 0) { - throw new StorageAuthException('Keystone did not provide a valid catalog, verify the credentials'); - } - - if (isset($this->params['serviceName'])) { - $serviceName = $this->params['serviceName']; - } else { - $serviceName = Service::DEFAULT_NAME; - } - - if (isset($this->params['urlType'])) { - $urlType = $this->params['urlType']; - if ($urlType !== 'internalURL' && $urlType !== 'publicURL') { - throw new StorageNotAvailableException('Invalid url type'); - } - } else { - $urlType = Service::DEFAULT_URL_TYPE; - } - - $catalogItem = $this->getCatalogForService($catalog, $serviceName); - if (!$catalogItem) { - $available = implode(', ', $this->getAvailableServiceNames($catalog)); - throw new StorageNotAvailableException( - "Service $serviceName not found in service catalog, available services: $available" - ); - } else if (isset($this->params['region'])) { - $this->validateRegion($catalogItem, $this->params['region']); - } - - $this->objectStoreService = $this->client->objectStoreService($serviceName, $this->params['region'], $urlType); - - try { - $this->container = $this->objectStoreService->getContainer($this->params['container']); - } catch (ClientErrorResponseException $ex) { - // if the container does not exist and autocreate is true try to create the container on the fly - if (isset($this->params['autocreate']) && $this->params['autocreate'] === true) { - $this->container = $this->objectStoreService->createContainer($this->params['container']); - } else { - throw $ex; - } - } catch (CurlException $e) { - if ($e->getErrorNo() === 7) { - $host = $e->getCurlHandle()->getUrl()->getHost() . ':' . $e->getCurlHandle()->getUrl()->getPort(); - \OC::$server->getLogger()->error("Can't connect to object storage server at $host"); - throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e); - } - throw $e; - } - } - - private function exportToken() { - $export = $this->client->exportCredentials(); - $export['catalog'] = array_map(function (CatalogItem $item) { - return [ - 'name' => $item->getName(), - 'endpoints' => $item->getEndpoints(), - 'type' => $item->getType() - ]; - }, $export['catalog']->getItems()); - $this->memcache->set('token', json_encode($export)); - } - - private function importToken() { - $cachedTokenString = $this->memcache->get('token'); - if ($cachedTokenString) { - $cachedToken = json_decode($cachedTokenString, true); - $cachedToken['catalog'] = array_map(function (array $item) { - $itemClass = new \stdClass(); - $itemClass->name = $item['name']; - $itemClass->endpoints = array_map(function (array $endpoint) { - return (object)$endpoint; - }, $item['endpoints']); - $itemClass->type = $item['type']; - - return $itemClass; - }, $cachedToken['catalog']); - try { - $this->client->importCredentials($cachedToken); - } catch (\Exception $e) { - $this->client->setTokenObject(new Token()); - } - } - } - - /** - * @param Catalog $catalog - * @param $name - * @return null|CatalogItem + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws \OCP\Files\StorageNotAvailableException */ - private function getCatalogForService(Catalog $catalog, $name) { - foreach ($catalog->getItems() as $item) { - /** @var CatalogItem $item */ - if ($item->hasType(Service::DEFAULT_TYPE) && $item->hasName($name)) { - return $item; - } - } - - return null; - } - - private function validateRegion(CatalogItem $item, $region) { - $endPoints = $item->getEndpoints(); - foreach ($endPoints as $endPoint) { - if ($endPoint->region === $region) { - return; - } - } - - $availableRegions = implode(', ', array_map(function ($endpoint) { - return $endpoint->region; - }, $endPoints)); - - throw new StorageNotAvailableException("Invalid region '$region', available regions: $availableRegions"); - } - - private function getAvailableServiceNames(Catalog $catalog) { - return array_map(function (CatalogItem $item) { - return $item->getName(); - }, array_filter($catalog->getItems(), function (CatalogItem $item) { - return $item->hasType(Service::DEFAULT_TYPE); - })); + private function getContainer() { + return $this->swiftFactory->getContainer(); } /** @@ -254,29 +68,29 @@ class Swift implements IObjectStore { /** * @param string $urn the unified resource name used to identify the object * @param resource $stream stream with the data to write - * @throws Exception from openstack lib when something goes wrong + * @throws \Exception from openstack lib when something goes wrong */ public function writeObject($urn, $stream) { - $this->init(); - $this->container->uploadObject($urn, $stream); + $this->getContainer()->createObject([ + 'name' => $urn, + 'stream' => stream_for($stream) + ]); } /** * @param string $urn the unified resource name used to identify the object * @return resource stream with the read data - * @throws Exception from openstack lib when something goes wrong + * @throws \Exception from openstack lib when something goes wrong */ public function readObject($urn) { - $this->init(); - $object = $this->container->getObject($urn); + $object = $this->getContainer()->getObject($urn); // we need to keep a reference to objectContent or // the stream will be closed before we can do anything with it - /** @var $objectContent \Guzzle\Http\EntityBody * */ - $objectContent = $object->getContent(); + $objectContent = $object->download(); $objectContent->rewind(); - $stream = $objectContent->getStream(); + $stream = $objectContent->detach(); // save the object content in the context of the stream to prevent it being gc'd until the stream is closed stream_context_set_option($stream, 'swift', 'content', $objectContent); @@ -286,17 +100,18 @@ class Swift implements IObjectStore { /** * @param string $urn Unified Resource Name * @return void - * @throws Exception from openstack lib when something goes wrong + * @throws \Exception from openstack lib when something goes wrong */ public function deleteObject($urn) { - $this->init(); - // see https://github.com/rackspace/php-opencloud/issues/243#issuecomment-30032242 - $this->container->dataObject()->setName($urn)->delete(); + $this->getContainer()->getObject($urn)->delete(); } - public function deleteContainer($recursive = false) { - $this->init(); - $this->container->delete($recursive); + /** + * @return void + * @throws \Exception from openstack lib when something goes wrong + */ + public function deleteContainer() { + $this->getContainer()->delete(); } } diff --git a/lib/private/Files/ObjectStore/SwiftFactory.php b/lib/private/Files/ObjectStore/SwiftFactory.php new file mode 100644 index 00000000000..0df6fb6efcd --- /dev/null +++ b/lib/private/Files/ObjectStore/SwiftFactory.php @@ -0,0 +1,174 @@ +<?php +declare(strict_types=1); +/** + * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OC\Files\ObjectStore; + +use GuzzleHttp\Client; +use GuzzleHttp\Exception\ClientException; +use GuzzleHttp\Exception\ConnectException; +use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\HandlerStack; +use OCP\Files\StorageAuthException; +use OCP\Files\StorageNotAvailableException; +use OCP\ICache; +use OpenStack\Common\Error\BadResponseError; +use OpenStack\Identity\v2\Models\Token; +use OpenStack\Identity\v2\Service; +use OpenStack\OpenStack; +use OpenStack\Common\Transport\Utils as TransportUtils; +use Psr\Http\Message\RequestInterface; +use OpenStack\ObjectStore\v1\Models\Container; + +class SwiftFactory { + private $cache; + private $params; + /** @var Container|null */ + private $container = null; + + public function __construct(ICache $cache, array $params) { + $this->cache = $cache; + $this->params = $params; + } + + private function getCachedToken(string $cacheKey) { + $cachedTokenString = $this->cache->get($cacheKey . '/token'); + if ($cachedTokenString) { + return json_decode($cachedTokenString); + } else { + return null; + } + } + + private function cacheToken(Token $token, string $cacheKey) { + $this->cache->set($cacheKey . '/token', json_encode($token)); + } + + /** + * @return OpenStack + * @throws StorageAuthException + */ + private function getClient() { + if (isset($this->params['bucket'])) { + $this->params['container'] = $this->params['bucket']; + } + if (!isset($this->params['container'])) { + $this->params['container'] = 'owncloud'; + } + if (!isset($this->params['autocreate'])) { + // should only be true for tests + $this->params['autocreate'] = false; + } + if (!isset($this->params['username']) && isset($this->params['user'])) { + $this->params['username'] = $this->params['user']; + } + if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) { + $this->params['tenantName'] = $this->params['tenant']; + } + + $cacheKey = $this->params['username'] . '@' . $this->params['url'] . '/' . $this->params['bucket']; + $token = $this->getCachedToken($cacheKey); + $hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now')); + if ($hasToken) { + $this->params['cachedToken'] = $token; + } + $httpClient = new Client([ + 'base_uri' => TransportUtils::normalizeUrl($this->params['url']), + 'handler' => HandlerStack::create() + ]); + + $authService = Service::factory($httpClient); + $this->params['identityService'] = $authService; + $this->params['authUrl'] = $this->params['url']; + $client = new OpenStack($this->params); + + if (!$hasToken) { + try { + $token = $authService->generateToken($this->params); + $this->cacheToken($token, $cacheKey); + } catch (ConnectException $e) { + throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e); + } catch (ClientException $e) { + $statusCode = $e->getResponse()->getStatusCode(); + if ($statusCode === 404) { + throw new StorageAuthException('Keystone not found, verify the keystone url', $e); + } else if ($statusCode === 412) { + throw new StorageAuthException('Precondition failed, verify the keystone url', $e); + } else if ($statusCode === 401) { + throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e); + } else { + throw new StorageAuthException('Unknown error', $e); + } + } catch (RequestException $e) { + throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e); + } + } + + return $client; + } + + /** + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws StorageNotAvailableException + */ + public function getContainer() { + if (is_null($this->container)) { + $this->container = $this->createContainer(); + } + + return $this->container; + } + + /** + * @return \OpenStack\ObjectStore\v1\Models\Container + * @throws StorageAuthException + * @throws StorageNotAvailableException + */ + private function createContainer() { + $client = $this->getClient(); + $objectStoreService = $client->objectStoreV1(); + + $autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true; + try { + $container = $objectStoreService->getContainer($this->params['container']); + if ($autoCreate) { + $container->getMetadata(); + } + return $container; + } catch (BadResponseError $ex) { + // if the container does not exist and autocreate is true try to create the container on the fly + if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) { + return $objectStoreService->createContainer([ + 'name' => $this->params['container'] + ]); + } else { + throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $e); + } + } catch (ConnectException $e) { + /** @var RequestInterface $request */ + $request = $e->getRequest(); + $host = $request->getUri()->getHost() . ':' . $request->getUri()->getPort(); + \OC::$server->getLogger()->error("Can't connect to object storage server at $host"); + throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e); + } + } +} diff --git a/tests/Settings/Controller/CheckSetupControllerTest.php b/tests/Settings/Controller/CheckSetupControllerTest.php index a0ee7f6cae6..a616cc3e70b 100644 --- a/tests/Settings/Controller/CheckSetupControllerTest.php +++ b/tests/Settings/Controller/CheckSetupControllerTest.php @@ -21,7 +21,6 @@ namespace Tests\Settings\Controller; -use Guzzle\Http\Message\Response; use OC\Settings\Controller\CheckSetupController; use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataDisplayResponse; @@ -34,6 +33,7 @@ use OCP\ILogger; use OCP\IRequest; use OCP\IURLGenerator; use OC_Util; +use Psr\Http\Message\ResponseInterface; use Test\TestCase; use OC\IntegrityCheck\Checker; @@ -461,7 +461,7 @@ class CheckSetupControllerTest extends TestCase { ->disableOriginalConstructor()->getMock(); $exception = $this->getMockBuilder('\GuzzleHttp\Exception\ClientException') ->disableOriginalConstructor()->getMock(); - $response = $this->getMockBuilder(Response::class) + $response = $this->getMockBuilder(ResponseInterface::class) ->disableOriginalConstructor()->getMock(); $response->expects($this->once()) ->method('getStatusCode') @@ -495,7 +495,7 @@ class CheckSetupControllerTest extends TestCase { ->disableOriginalConstructor()->getMock(); $exception = $this->getMockBuilder('\GuzzleHttp\Exception\ClientException') ->disableOriginalConstructor()->getMock(); - $response = $this->getMockBuilder(Response::class) + $response = $this->getMockBuilder(ResponseInterface::class) ->disableOriginalConstructor()->getMock(); $response->expects($this->once()) ->method('getStatusCode') diff --git a/tests/drone-wait-objectstore.sh b/tests/drone-wait-objectstore.sh index 14d0b6f1f67..228accc3da9 100755 --- a/tests/drone-wait-objectstore.sh +++ b/tests/drone-wait-objectstore.sh @@ -1,10 +1,61 @@ #!/bin/bash +function get_swift_token() { + KEYSTONE_OUT=$(curl -s 'http://dockswift:5000/v2.0/tokens' -H 'Content-Type: application/json' -d '{"auth":{"passwordCredentials":{"username":"swift","password":"swift"},"tenantName":"service"}}') + if (echo "$KEYSTONE_OUT" | grep -q 'object-store') + then + SWIFT_ENDPOINT=$(echo "$KEYSTONE_OUT" | php -r "echo array_values(array_filter(json_decode(file_get_contents('php://stdin'),true)['access']['serviceCatalog'], function(\$endpoint){return \$endpoint['type']==='object-store';}))[0]['endpoints'][0]['publicURL'];") + SWIFT_TOKEN=$(echo "$KEYSTONE_OUT" | php -r "echo json_decode(file_get_contents('php://stdin'),true)['access']['token']['id'];") + return 0 + else + return -1 + fi +} + if [ "$OBJECT_STORE" == "swift" ]; then - echo "waiting for swift" - until curl -I http://dockswift:5000/v3 + echo "waiting for keystone" + until get_swift_token + do + sleep 2 + done + + echo "waiting for object store at $SWIFT_ENDPOINT" + + until curl -s -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT" + do + sleep 2 + done + + echo "creating container" + + sleep 2 + + while [ 1 ] do sleep 2 + + respCode=$(curl -s -o /dev/null -w "%{http_code}" -X PUT -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT/nextcloud") + + if [ "$respCode" == "201" ] + then + break + fi done - sleep 60 + + echo "creating test file" + + while [ 1 ] + do + sleep 2 + + respCode=$(curl -s -o /dev/null -w "%{http_code}" -X PUT -H "X-Auth-Token: $SWIFT_TOKEN" -H "Content-Type: text/html; charset=UTF-8" -d "Hello world" "$SWIFT_ENDPOINT/nextcloud/helloworld.txt") + + if [ "$respCode" == "201" ] + then + break + fi + done + + echo "deleting test file" + curl -s -o /dev/null -w "%{http_code}\n" -X DELETE -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT/nextcloud/helloworld.txt" fi |