Merge pull request #8314 from nextcloud/openstack

update openstack sdk

4 files changed, 205 insertions, 214 deletions

diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 06879c5179a..b3158ac7e04 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -653,6 +653,7 @@ return array(
     'OC\\Files\\ObjectStore\\S3Signature' => $baseDir . '/lib/private/Files/ObjectStore/S3Signature.php',
     'OC\\Files\\ObjectStore\\StorageObjectStore' => $baseDir . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
     'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
+    'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
     'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
     'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php',
     'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 118094c30ef..65b36cc489e 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -683,6 +683,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Files\\ObjectStore\\S3Signature' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3Signature.php',
         'OC\\Files\\ObjectStore\\StorageObjectStore' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
         'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
+        'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
         'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
         'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php',
         'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php',
diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php
index a3cba488f5f..4451fbcc750 100644
--- a/lib/private/Files/ObjectStore/Swift.php
+++ b/lib/private/Files/ObjectStore/Swift.php
@@ -25,223 +25,37 @@
 
 namespace OC\Files\ObjectStore;
 
-use Guzzle\Http\Exception\ClientErrorResponseException;
-use Guzzle\Http\Exception\CurlException;
+use function GuzzleHttp\Psr7\stream_for;
 use Icewind\Streams\RetryWrapper;
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\Files\StorageAuthException;
-use OCP\Files\StorageNotAvailableException;
-use OpenCloud\Common\Service\Catalog;
-use OpenCloud\Common\Service\CatalogItem;
-use OpenCloud\Identity\Resource\Token;
-use OpenCloud\ObjectStore\Service;
-use OpenCloud\OpenStack;
-use OpenCloud\Rackspace;
 
 class Swift implements IObjectStore {
-
-	/**
-	 * @var \OpenCloud\OpenStack
-	 */
-	private $client;
-
 	/**
 	 * @var array
 	 */
 	private $params;
 
 	/**
-	 * @var \OpenCloud\ObjectStore\Service
-	 */
-	private $objectStoreService;
-
-	/**
-	 * @var \OpenCloud\ObjectStore\Resource\Container
+	 * @var \OpenStack\ObjectStore\v1\Models\Container|null
 	 */
-	private $container;
-
-	private $memcache;
-
-	public function __construct($params) {
-		if (isset($params['bucket'])) {
-			$params['container'] = $params['bucket'];
-		}
-		if (!isset($params['container'])) {
-			$params['container'] = 'owncloud';
-		}
-		if (!isset($params['autocreate'])) {
-			// should only be true for tests
-			$params['autocreate'] = false;
-		}
-
-		if (isset($params['apiKey'])) {
-			$this->client = new Rackspace($params['url'], $params);
-			$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
-		} else {
-			$this->client = new OpenStack($params['url'], $params);
-			$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
-		}
+	private $container = null;
 
-		$cacheFactory = \OC::$server->getMemCacheFactory();
-		$this->memcache = $cacheFactory->createDistributed('swift::' . $cacheKey);
+	/** @var SwiftFactory */
+	private $swiftFactory;
 
+	public function __construct($params, SwiftFactory $connectionFactory = null) {
+		$this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params);
 		$this->params = $params;
 	}
 
 	/**
-	 * @suppress PhanNonClassMethodCall
-	 */
-	protected function init() {
-		if ($this->container) {
-			return;
-		}
-
-		$this->importToken();
-
-		/** @var Token $token */
-		$token = $this->client->getTokenObject();
-
-		if (!$token || $token->hasExpired()) {
-			try {
-				$this->client->authenticate();
-				$this->exportToken();
-			} catch (ClientErrorResponseException $e) {
-				$statusCode = $e->getResponse()->getStatusCode();
-				if ($statusCode == 412) {
-					throw new StorageAuthException('Precondition failed, verify the keystone url', $e);
-				} else if ($statusCode === 401) {
-					throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e);
-				} else {
-					throw new StorageAuthException('Unknown error', $e);
-				}
-			}
-		}
-
-
-		/** @var Catalog $catalog */
-		$catalog = $this->client->getCatalog();
-
-		if (count($catalog->getItems()) === 0) {
-			throw new StorageAuthException('Keystone did not provide a valid catalog, verify the credentials');
-		}
-
-		if (isset($this->params['serviceName'])) {
-			$serviceName = $this->params['serviceName'];
-		} else {
-			$serviceName = Service::DEFAULT_NAME;
-		}
-
-		if (isset($this->params['urlType'])) {
-			$urlType = $this->params['urlType'];
-			if ($urlType !== 'internalURL' && $urlType !== 'publicURL') {
-				throw new StorageNotAvailableException('Invalid url type');
-			}
-		} else {
-			$urlType = Service::DEFAULT_URL_TYPE;
-		}
-
-		$catalogItem = $this->getCatalogForService($catalog, $serviceName);
-		if (!$catalogItem) {
-			$available = implode(', ', $this->getAvailableServiceNames($catalog));
-			throw new StorageNotAvailableException(
-				"Service $serviceName not found in service catalog, available services: $available"
-			);
-		} else if (isset($this->params['region'])) {
-			$this->validateRegion($catalogItem, $this->params['region']);
-		}
-
-		$this->objectStoreService = $this->client->objectStoreService($serviceName, $this->params['region'], $urlType);
-
-		try {
-			$this->container = $this->objectStoreService->getContainer($this->params['container']);
-		} catch (ClientErrorResponseException $ex) {
-			// if the container does not exist and autocreate is true try to create the container on the fly
-			if (isset($this->params['autocreate']) && $this->params['autocreate'] === true) {
-				$this->container = $this->objectStoreService->createContainer($this->params['container']);
-			} else {
-				throw $ex;
-			}
-		} catch (CurlException $e) {
-			if ($e->getErrorNo() === 7) {
-				$host = $e->getCurlHandle()->getUrl()->getHost() . ':' . $e->getCurlHandle()->getUrl()->getPort();
-				\OC::$server->getLogger()->error("Can't connect to object storage server at $host");
-				throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e);
-			}
-			throw $e;
-		}
-	}
-
-	private function exportToken() {
-		$export = $this->client->exportCredentials();
-		$export['catalog'] = array_map(function (CatalogItem $item) {
-			return [
-				'name' => $item->getName(),
-				'endpoints' => $item->getEndpoints(),
-				'type' => $item->getType()
-			];
-		}, $export['catalog']->getItems());
-		$this->memcache->set('token', json_encode($export));
-	}
-
-	private function importToken() {
-		$cachedTokenString = $this->memcache->get('token');
-		if ($cachedTokenString) {
-			$cachedToken = json_decode($cachedTokenString, true);
-			$cachedToken['catalog'] = array_map(function (array $item) {
-				$itemClass = new \stdClass();
-				$itemClass->name = $item['name'];
-				$itemClass->endpoints = array_map(function (array $endpoint) {
-					return (object)$endpoint;
-				}, $item['endpoints']);
-				$itemClass->type = $item['type'];
-
-				return $itemClass;
-			}, $cachedToken['catalog']);
-			try {
-				$this->client->importCredentials($cachedToken);
-			} catch (\Exception $e) {
-				$this->client->setTokenObject(new Token());
-			}
-		}
-	}
-
-	/**
-	 * @param Catalog $catalog
-	 * @param $name
-	 * @return null|CatalogItem
+	 * @return \OpenStack\ObjectStore\v1\Models\Container
+	 * @throws StorageAuthException
+	 * @throws \OCP\Files\StorageNotAvailableException
 	 */
-	private function getCatalogForService(Catalog $catalog, $name) {
-		foreach ($catalog->getItems() as $item) {
-			/** @var CatalogItem $item */
-			if ($item->hasType(Service::DEFAULT_TYPE) && $item->hasName($name)) {
-				return $item;
-			}
-		}
-
-		return null;
-	}
-
-	private function validateRegion(CatalogItem $item, $region) {
-		$endPoints = $item->getEndpoints();
-		foreach ($endPoints as $endPoint) {
-			if ($endPoint->region === $region) {
-				return;
-			}
-		}
-
-		$availableRegions = implode(', ', array_map(function ($endpoint) {
-			return $endpoint->region;
-		}, $endPoints));
-
-		throw new StorageNotAvailableException("Invalid region '$region', available regions: $availableRegions");
-	}
-
-	private function getAvailableServiceNames(Catalog $catalog) {
-		return array_map(function (CatalogItem $item) {
-			return $item->getName();
-		}, array_filter($catalog->getItems(), function (CatalogItem $item) {
-			return $item->hasType(Service::DEFAULT_TYPE);
-		}));
+	private function getContainer() {
+		return $this->swiftFactory->getContainer();
 	}
 
 	/**
@@ -254,29 +68,29 @@ class Swift implements IObjectStore {
 	/**
 	 * @param string $urn the unified resource name used to identify the object
 	 * @param resource $stream stream with the data to write
-	 * @throws Exception from openstack lib when something goes wrong
+	 * @throws \Exception from openstack lib when something goes wrong
 	 */
 	public function writeObject($urn, $stream) {
-		$this->init();
-		$this->container->uploadObject($urn, $stream);
+		$this->getContainer()->createObject([
+			'name' => $urn,
+			'stream' => stream_for($stream)
+		]);
 	}
 
 	/**
 	 * @param string $urn the unified resource name used to identify the object
 	 * @return resource stream with the read data
-	 * @throws Exception from openstack lib when something goes wrong
+	 * @throws \Exception from openstack lib when something goes wrong
 	 */
 	public function readObject($urn) {
-		$this->init();
-		$object = $this->container->getObject($urn);
+		$object = $this->getContainer()->getObject($urn);
 
 		// we need to keep a reference to objectContent or
 		// the stream will be closed before we can do anything with it
-		/** @var $objectContent \Guzzle\Http\EntityBody * */
-		$objectContent = $object->getContent();
+		$objectContent = $object->download();
 		$objectContent->rewind();
 
-		$stream = $objectContent->getStream();
+		$stream = $objectContent->detach();
 		// save the object content in the context of the stream to prevent it being gc'd until the stream is closed
 		stream_context_set_option($stream, 'swift', 'content', $objectContent);
 
@@ -286,17 +100,18 @@ class Swift implements IObjectStore {
 	/**
 	 * @param string $urn Unified Resource Name
 	 * @return void
-	 * @throws Exception from openstack lib when something goes wrong
+	 * @throws \Exception from openstack lib when something goes wrong
 	 */
 	public function deleteObject($urn) {
-		$this->init();
-		// see https://github.com/rackspace/php-opencloud/issues/243#issuecomment-30032242
-		$this->container->dataObject()->setName($urn)->delete();
+		$this->getContainer()->getObject($urn)->delete();
 	}
 
-	public function deleteContainer($recursive = false) {
-		$this->init();
-		$this->container->delete($recursive);
+	/**
+	 * @return void
+	 * @throws \Exception from openstack lib when something goes wrong
+	 */
+	public function deleteContainer() {
+		$this->getContainer()->delete();
 	}
 
 }
diff --git a/lib/private/Files/ObjectStore/SwiftFactory.php b/lib/private/Files/ObjectStore/SwiftFactory.php
new file mode 100644
index 00000000000..0df6fb6efcd
--- /dev/null
+++ b/lib/private/Files/ObjectStore/SwiftFactory.php
@@ -0,0 +1,174 @@
+<?php
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\ClientException;
+use GuzzleHttp\Exception\ConnectException;
+use GuzzleHttp\Exception\RequestException;
+use GuzzleHttp\HandlerStack;
+use OCP\Files\StorageAuthException;
+use OCP\Files\StorageNotAvailableException;
+use OCP\ICache;
+use OpenStack\Common\Error\BadResponseError;
+use OpenStack\Identity\v2\Models\Token;
+use OpenStack\Identity\v2\Service;
+use OpenStack\OpenStack;
+use OpenStack\Common\Transport\Utils as TransportUtils;
+use Psr\Http\Message\RequestInterface;
+use OpenStack\ObjectStore\v1\Models\Container;
+
+class SwiftFactory {
+	private $cache;
+	private $params;
+	/** @var Container|null */
+	private $container = null;
+
+	public function __construct(ICache $cache, array $params) {
+		$this->cache = $cache;
+		$this->params = $params;
+	}
+
+	private function getCachedToken(string $cacheKey) {
+		$cachedTokenString = $this->cache->get($cacheKey . '/token');
+		if ($cachedTokenString) {
+			return json_decode($cachedTokenString);
+		} else {
+			return null;
+		}
+	}
+
+	private function cacheToken(Token $token, string $cacheKey) {
+		$this->cache->set($cacheKey . '/token', json_encode($token));
+	}
+
+	/**
+	 * @return OpenStack
+	 * @throws StorageAuthException
+	 */
+	private function getClient() {
+		if (isset($this->params['bucket'])) {
+			$this->params['container'] = $this->params['bucket'];
+		}
+		if (!isset($this->params['container'])) {
+			$this->params['container'] = 'owncloud';
+		}
+		if (!isset($this->params['autocreate'])) {
+			// should only be true for tests
+			$this->params['autocreate'] = false;
+		}
+		if (!isset($this->params['username']) && isset($this->params['user'])) {
+			$this->params['username'] = $this->params['user'];
+		}
+		if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) {
+			$this->params['tenantName'] = $this->params['tenant'];
+		}
+
+		$cacheKey = $this->params['username'] . '@' . $this->params['url'] . '/' . $this->params['bucket'];
+		$token = $this->getCachedToken($cacheKey);
+		$hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now'));
+		if ($hasToken) {
+			$this->params['cachedToken'] = $token;
+		}
+		$httpClient = new Client([
+			'base_uri' => TransportUtils::normalizeUrl($this->params['url']),
+			'handler' => HandlerStack::create()
+		]);
+
+		$authService = Service::factory($httpClient);
+		$this->params['identityService'] = $authService;
+		$this->params['authUrl'] = $this->params['url'];
+		$client = new OpenStack($this->params);
+
+		if (!$hasToken) {
+			try {
+				$token = $authService->generateToken($this->params);
+				$this->cacheToken($token, $cacheKey);
+			} catch (ConnectException $e) {
+				throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e);
+			} catch (ClientException $e) {
+				$statusCode = $e->getResponse()->getStatusCode();
+				if ($statusCode === 404) {
+					throw new StorageAuthException('Keystone not found, verify the keystone url', $e);
+				} else if ($statusCode === 412) {
+					throw new StorageAuthException('Precondition failed, verify the keystone url', $e);
+				} else if ($statusCode === 401) {
+					throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e);
+				} else {
+					throw new StorageAuthException('Unknown error', $e);
+				}
+			} catch (RequestException $e) {
+				throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e);
+			}
+		}
+
+		return $client;
+	}
+
+	/**
+	 * @return \OpenStack\ObjectStore\v1\Models\Container
+	 * @throws StorageAuthException
+	 * @throws StorageNotAvailableException
+	 */
+	public function getContainer() {
+		if (is_null($this->container)) {
+			$this->container = $this->createContainer();
+		}
+
+		return $this->container;
+	}
+
+	/**
+	 * @return \OpenStack\ObjectStore\v1\Models\Container
+	 * @throws StorageAuthException
+	 * @throws StorageNotAvailableException
+	 */
+	private function createContainer() {
+		$client = $this->getClient();
+		$objectStoreService = $client->objectStoreV1();
+
+		$autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true;
+		try {
+			$container = $objectStoreService->getContainer($this->params['container']);
+			if ($autoCreate) {
+				$container->getMetadata();
+			}
+			return $container;
+		} catch (BadResponseError $ex) {
+			// if the container does not exist and autocreate is true try to create the container on the fly
+			if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) {
+				return $objectStoreService->createContainer([
+					'name' => $this->params['container']
+				]);
+			} else {
+				throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $e);
+			}
+		} catch (ConnectException $e) {
+			/** @var RequestInterface $request */
+			$request = $e->getRequest();
+			$host = $request->getUri()->getHost() . ':' . $request->getUri()->getPort();
+			\OC::$server->getLogger()->error("Can't connect to object storage server at $host");
+			throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e);
+		}
+	}
+}