Merge pull request #8314 from nextcloud/openstack

update openstack sdk

2 files changed, 99 insertions, 164 deletions

diff --git a/apps/files_external/lib/Lib/Storage/Swift.php b/apps/files_external/lib/Lib/Storage/Swift.php
index 88a1cf4c386..051c65350d5 100644
--- a/apps/files_external/lib/Lib/Storage/Swift.php
+++ b/apps/files_external/lib/Lib/Storage/Swift.php
@@ -1,4 +1,5 @@
 <?php
+declare(strict_types=1);
 /**
  * @copyright Copyright (c) 2016, ownCloud, Inc.
  *
@@ -39,32 +40,22 @@
 
 namespace OCA\Files_External\Lib\Storage;
 
-use Guzzle\Http\Url;
-use Guzzle\Http\Exception\ClientErrorResponseException;
+use GuzzleHttp\Psr7\Uri;
 use Icewind\Streams\CallbackWrapper;
 use Icewind\Streams\IteratorDirectory;
-use Icewind\Streams\RetryWrapper;
-use OpenCloud;
-use OpenCloud\Common\Exceptions;
-use OpenCloud\OpenStack;
-use OpenCloud\Rackspace;
-use OpenCloud\ObjectStore\Resource\DataObject;
+use OC\Files\ObjectStore\SwiftFactory;
+use OCP\Files\StorageBadConfigException;
+use OpenStack\Common\Error\BadResponseError;
+use OpenStack\ObjectStore\v1\Models\StorageObject;
 
 class Swift extends \OC\Files\Storage\Common {
-
-	/**
-	 * @var \OpenCloud\ObjectStore\Service
-	 */
-	private $connection;
+	/** @var SwiftFactory */
+	private $connectionFactory;
 	/**
-	 * @var \OpenCloud\ObjectStore\Resource\Container
+	 * @var \OpenStack\ObjectStore\v1\Models\Container
 	 */
 	private $container;
 	/**
-	 * @var \OpenCloud\OpenStack
-	 */
-	private $anchor;
-	/**
 	 * @var string
 	 */
 	private $bucket;
@@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common {
 	 */
 	private $params;
 
-	/** @var string  */
+	/** @var string */
 	private $id;
 
+	/** @var \OC\Files\ObjectStore\Swift */
+	private $objectStore;
+
 	/**
 	 * Key value cache mapping path to data object. Maps path to
 	 * \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing
 	 * paths and path to false for not existing paths.
+	 *
 	 * @var \OCP\ICache
 	 */
 	private $objectCache;
 
 	/**
 	 * @param string $path
+	 * @return mixed|string
 	 */
-	private function normalizePath($path) {
+	private function normalizePath(string $path) {
 		$path = trim($path, '/');
 
 		if (!$path) {
@@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common {
 	 * that one will be returned.
 	 *
 	 * @param string $path
-	 * @return \OpenCloud\ObjectStore\Resource\DataObject|bool object
+	 * @return StorageObject|bool object
 	 * or false if the object did not exist
+	 * @throws \OCP\Files\StorageAuthException
+	 * @throws \OCP\Files\StorageNotAvailableException
 	 */
-	private function fetchObject($path) {
+	private function fetchObject(string $path) {
 		if ($this->objectCache->hasKey($path)) {
 			// might be "false" if object did not exist from last check
 			return $this->objectCache->get($path);
 		}
 		try {
-			$object = $this->getContainer()->getPartialObject($path);
+			$object = $this->getContainer()->getObject($path);
+			$object->retrieve();
 			$this->objectCache->set($path, $object);
 			return $object;
-		} catch (ClientErrorResponseException $e) {
-			// this exception happens when the object does not exist, which
-			// is expected in most cases
-			$this->objectCache->set($path, false);
-			return false;
-		} catch (ClientErrorResponseException $e) {
+		} catch (BadResponseError $e) {
 			// Expected response is "404 Not Found", so only log if it isn't
 			if ($e->getResponse()->getStatusCode() !== 404) {
 				\OC::$server->getLogger()->logException($e, [
@@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common {
 					'app' => 'files_external',
 				]);
 			}
+			$this->objectCache->set($path, false);
 			return false;
 		}
 	}
@@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common {
 	 * @param string $path
 	 *
 	 * @return bool true if the object exist, false otherwise
+	 * @throws \OCP\Files\StorageAuthException
+	 * @throws \OCP\Files\StorageNotAvailableException
 	 */
 	private function doesObjectExist($path) {
 		return $this->fetchObject($path) !== false;
@@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common {
 			or empty($params['user']) or empty($params['bucket'])
 			or empty($params['region'])
 		) {
-			throw new \Exception("API Key or password, Username, Bucket and Region have to be configured.");
+			throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured.");
 		}
 
 		$this->id = 'swift::' . $params['user'] . md5($params['bucket']);
 
-		$bucketUrl = Url::factory($params['bucket']);
-		if ($bucketUrl->isAbsolute()) {
-			$this->bucket = end($bucketUrl->getPathSegments());
-			$params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath();
-		} else {
-			$this->bucket = $params['bucket'];
+		$bucketUrl = new Uri($params['bucket']);
+		if ($bucketUrl->getHost()) {
+			$params['bucket'] = basename($bucketUrl->getPath());
+			$params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath()));
 		}
 
 		if (empty($params['url'])) {
@@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common {
 			$params['service_name'] = 'cloudFiles';
 		}
 
+		$params['autocreate'] = true;
+
 		$this->params = $params;
 		// FIXME: private class...
 		$this->objectCache = new \OC\Cache\CappedMemoryCache();
+		$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params);
+		$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory);
+		$this->bucket = $params['bucket'];
 	}
 
 	public function mkdir($path) {
@@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$customHeaders = array('content-type' => 'httpd/unix-directory');
-			$metadataHeaders = DataObject::stockHeaders(array());
-			$allHeaders = $customHeaders + $metadataHeaders;
-			$this->getContainer()->uploadObject($path, '', $allHeaders);
+			$this->getContainer()->createObject([
+				'name' => $path,
+				'content' => '',
+				'headers' => ['content-type' => 'httpd/unix-directory']
+			]);
 			// invalidate so that the next access gets the real object
 			// with all properties
 			$this->objectCache->remove($path);
-		} catch (Exceptions\CreateUpdateError $e) {
+		} catch (BadResponseError $e) {
 			\OC::$server->getLogger()->logException($e, [
 				'level' => \OCP\Util::ERROR,
 				'app' => 'files_external',
@@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->getContainer()->dataObject()->setName($path . '/')->delete();
+			$this->objectStore->deleteObject($path . '/');
 			$this->objectCache->remove($path . '/');
-		} catch (Exceptions\DeleteError $e) {
+		} catch (BadResponseError $e) {
 			\OC::$server->getLogger()->logException($e, [
 				'level' => \OCP\Util::ERROR,
 				'app' => 'files_external',
@@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common {
 			$path .= '/';
 		}
 
-		$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
+//		$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
 
 		try {
-			$files = array();
-			/** @var OpenCloud\Common\Collection $objects */
-			$objects = $this->getContainer()->objectList(array(
+			$files = [];
+			$objects = $this->getContainer()->listObjects([
 				'prefix' => $path,
 				'delimiter' => '/'
-			));
+			]);
 
-			/** @var OpenCloud\ObjectStore\Resource\DataObject $object */
+			/** @var StorageObject $object */
 			foreach ($objects as $object) {
-				$file = basename($object->getName());
+				$file = basename($object->name);
 				if ($file !== basename($path) && $file !== '.') {
 					$files[] = $file;
 				}
@@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			/** @var DataObject $object */
 			$object = $this->fetchObject($path);
 			if (!$object) {
 				return false;
 			}
-		} catch (ClientErrorResponseException $e) {
+		} catch (BadResponseError $e) {
 			\OC::$server->getLogger()->logException($e, [
 				'level' => \OCP\Util::ERROR,
 				'app' => 'files_external',
@@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common {
 			return false;
 		}
 
-		$dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified());
-		if ($dateTime !== false) {
-			$mtime = $dateTime->getTimestamp();
-		} else {
-			$mtime = null;
-		}
+		$dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false;
+		$mtime = $dateTime ? $dateTime->getTimestamp() : null;
 		$objectMetadata = $object->getMetadata();
-		$metaTimestamp = $objectMetadata->getProperty('timestamp');
-		if (isset($metaTimestamp)) {
-			$mtime = $metaTimestamp;
+		if (isset($objectMetadata['timestamp'])) {
+			$mtime = $objectMetadata['timestamp'];
 		}
 
 		if (!empty($mtime)) {
@@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		$stat = array();
-		$stat['size'] = (int)$object->getContentLength();
+		$stat['size'] = (int)$object->contentLength;
 		$stat['mtime'] = $mtime;
 		$stat['atime'] = time();
 		return $stat;
@@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->getContainer()->dataObject()->setName($path)->delete();
+			$this->objectStore->deleteObject($path);
 			$this->objectCache->remove($path);
 			$this->objectCache->remove($path . '/');
-		} catch (ClientErrorResponseException $e) {
+		} catch (BadResponseError $e) {
 			if ($e->getResponse()->getStatusCode() !== 404) {
 				\OC::$server->getLogger()->logException($e, [
 					'level' => \OCP\Util::ERROR,
 					'app' => 'files_external',
 				]);
+				throw $e;
 			}
-			return false;
 		}
 
 		return true;
@@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common {
 			case 'r':
 			case 'rb':
 				try {
-					$c = $this->getContainer();
-					$streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory();
-					/** @var \OpenCloud\Common\Http\Client $client */
-					$client = $c->getClient();
-					$streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path)));
-					$streamInterface->rewind();
-					$stream = $streamInterface->getStream();
-					stream_context_set_option($stream, 'swift','content', $streamInterface);
-					if(!strrpos($streamInterface
-						->getMetaData('wrapper_data')[0], '404 Not Found')) {
-						return RetryWrapper::wrap($stream);
-					}
-					return false;
-				} catch (\Guzzle\Http\Exception\BadResponseException $e) {
+					return $this->objectStore->readObject($path);
+				} catch (BadResponseError $e) {
 					\OC::$server->getLogger()->logException($e, [
 						'level' => \OCP\Util::ERROR,
 						'app' => 'files_external',
@@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common {
 		if (is_null($mtime)) {
 			$mtime = time();
 		}
-		$metadata = array('timestamp' => $mtime);
+		$metadata = ['timestamp' => $mtime];
 		if ($this->file_exists($path)) {
 			if ($this->is_dir($path) && $path !== '.') {
 				$path .= '/';
 			}
 
 			$object = $this->fetchObject($path);
-			if ($object->saveMetadata($metadata)) {
+			if ($object->mergeMetadata($metadata)) {
 				// invalidate target object to force repopulation on fetch
 				$this->objectCache->remove($path);
 			}
 			return true;
 		} else {
 			$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
-			$customHeaders = array('content-type' => $mimeType);
-			$metadataHeaders = DataObject::stockHeaders($metadata);
-			$allHeaders = $customHeaders + $metadataHeaders;
-			$this->getContainer()->uploadObject($path, '', $allHeaders);
+			$this->getContainer()->createObject([
+				'name' => $path,
+				'content' => '',
+				'headers' => ['content-type' => 'httpd/unix-directory']
+			]);
 			// invalidate target object to force repopulation on fetch
 			$this->objectCache->remove($path);
 			return true;
@@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common {
 		$path2 = $this->normalizePath($path2);
 
 		$fileType = $this->filetype($path1);
-		if ($fileType === 'file') {
-
+		if ($fileType) {
 			// make way
 			$this->unlink($path2);
+		}
 
+		if ($fileType === 'file') {
 			try {
 				$source = $this->fetchObject($path1);
-				$source->copy($this->bucket . '/' . $path2);
+				$source->copy([
+					'destination' => $this->bucket . '/' . $path2
+				]);
 				// invalidate target object to force repopulation on fetch
 				$this->objectCache->remove($path2);
 				$this->objectCache->remove($path2 . '/');
-			} catch (ClientErrorResponseException $e) {
+			} catch (BadResponseError $e) {
 				\OC::$server->getLogger()->logException($e, [
 					'level' => \OCP\Util::ERROR,
 					'app' => 'files_external',
@@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common {
 			}
 
 		} else if ($fileType === 'dir') {
-
-			// make way
-			$this->unlink($path2);
-
 			try {
 				$source = $this->fetchObject($path1 . '/');
-				$source->copy($this->bucket . '/' . $path2 . '/');
+				$source->copy([
+					'destination' => $this->bucket . '/' . $path2 . '/'
+				]);
 				// invalidate target object to force repopulation on fetch
 				$this->objectCache->remove($path2);
 				$this->objectCache->remove($path2 . '/');
-			} catch (ClientErrorResponseException $e) {
+			} catch (BadResponseError $e) {
 				\OC::$server->getLogger()->logException($e, [
 					'level' => \OCP\Util::ERROR,
 					'app' => 'files_external',
@@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common {
 
 			// cleanup
 			if ($this->unlink($path1) === false) {
+				throw new \Exception('failed to remove original');
 				$this->unlink($path2);
 				return false;
 			}
@@ -568,80 +553,26 @@ class Swift extends \OC\Files\Storage\Common {
 	}
 
 	/**
-	 * Returns the connection
-	 *
-	 * @return OpenCloud\ObjectStore\Service connected client
-	 * @throws \Exception if connection could not be made
-	 */
-	public function getConnection() {
-		if (!is_null($this->connection)) {
-			return $this->connection;
-		}
-
-		$settings = array(
-			'username' => $this->params['user'],
-		);
-
-		if (!empty($this->params['password'])) {
-			$settings['password'] = $this->params['password'];
-		} else if (!empty($this->params['key'])) {
-			$settings['apiKey'] = $this->params['key'];
-		}
-
-		if (!empty($this->params['tenant'])) {
-			$settings['tenantName'] = $this->params['tenant'];
-		}
-
-		if (!empty($this->params['timeout'])) {
-			$settings['timeout'] = $this->params['timeout'];
-		}
-
-		if (isset($settings['apiKey'])) {
-			$this->anchor = new Rackspace($this->params['url'], $settings);
-		} else {
-			$this->anchor = new OpenStack($this->params['url'], $settings);
-		}
-
-		$connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
-
-		if (!empty($this->params['endpoint_url'])) {
-			$endpoint = $connection->getEndpoint();
-			$endpoint->setPublicUrl($this->params['endpoint_url']);
-			$endpoint->setPrivateUrl($this->params['endpoint_url']);
-			$connection->setEndpoint($endpoint);
-		}
-
-		$this->connection = $connection;
-
-		return $this->connection;
-	}
-
-	/**
 	 * Returns the initialized object store container.
 	 *
-	 * @return OpenCloud\ObjectStore\Resource\Container
+	 * @return \OpenStack\ObjectStore\v1\Models\Container
+	 * @throws \OCP\Files\StorageAuthException
+	 * @throws \OCP\Files\StorageNotAvailableException
 	 */
 	public function getContainer() {
-		if (!is_null($this->container)) {
-			return $this->container;
-		}
+		if (is_null($this->container)) {
+			$this->container = $this->connectionFactory->getContainer();
 
-		try {
-			$this->container = $this->getConnection()->getContainer($this->bucket);
-		} catch (ClientErrorResponseException $e) {
-			$this->container = $this->getConnection()->createContainer($this->bucket);
-		}
-
-		if (!$this->file_exists('.')) {
-			$this->mkdir('.');
+			if (!$this->file_exists('.')) {
+				$this->mkdir('.');
+			}
 		}
-
 		return $this->container;
 	}
 
 	public function writeBack($tmpFile, $path) {
 		$fileData = fopen($tmpFile, 'r');
-		$this->getContainer()->uploadObject($path, $fileData);
+		$this->objectStore->writeObject($path, $fileData);
 		// invalidate target object to force repopulation on fetch
 		$this->objectCache->remove($path);
 		unlink($tmpFile);
diff --git a/apps/files_external/tests/Storage/SwiftTest.php b/apps/files_external/tests/Storage/SwiftTest.php
index 19f185ed033..fdda8baace8 100644
--- a/apps/files_external/tests/Storage/SwiftTest.php
+++ b/apps/files_external/tests/Storage/SwiftTest.php
@@ -27,6 +27,7 @@
 
 namespace OCA\Files_External\Tests\Storage;
 
+use GuzzleHttp\Exception\ClientException;
 use \OCA\Files_External\Lib\Storage\Swift;
 
 /**
@@ -40,6 +41,11 @@ class SwiftTest extends \Test\Files\Storage\Storage {
 
 	private $config;
 
+	/**
+	 * @var Swift instance
+	 */
+	protected $instance;
+
 	protected function setUp() {
 		parent::setUp();
 
@@ -53,17 +59,15 @@ class SwiftTest extends \Test\Files\Storage\Storage {
 	protected function tearDown() {
 		if ($this->instance) {
 			try {
-				$connection = $this->instance->getConnection();
-				$container = $connection->getContainer($this->config['bucket']);
+				$container = $this->instance->getContainer();
 
-				$objects = $container->objectList();
-				while($object = $objects->next()) {
-					$object->setName(str_replace('#','%23',$object->getName()));
+				$objects = $container->listObjects();
+				foreach ($objects as $object) {
 					$object->delete();
 				}
 
 				$container->delete();
-			} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
+			} catch (ClientException $e) {
 				// container didn't exist, so we don't need to delete it
 			}
 		}