mirrors
/
nextcloud
mirror of https://github.com/nextcloud/server.git
Watch 1
Star 0
Fork 0
Code Issues 0 Releases 1008 Wiki Activity
Browse Source

adjust swift external storage to new openstack sdk 

Signed-off-by: Robin Appelman <robin@icewind.nl>
tags/v14.0.0beta1
Robin Appelman 6 years ago
parent
3192efc415
commit
817f2f8f49
No account linked to committer's email address
4 changed files with 150 additions and 198 deletions
Split View Show Diff Stats
  1. 89
    158
      apps/files_external/lib/Lib/Storage/Swift.php
  2. 10
    6
      apps/files_external/tests/Storage/SwiftTest.php
  3. 5
    8
      lib/private/Files/ObjectStore/Swift.php
  4. 46
    26
      lib/private/Files/ObjectStore/SwiftFactory.php

+ 89
- 158
apps/files_external/lib/Lib/Storage/Swift.php View File

@@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
@@ -39,31 +40,21 @@

namespace OCA\Files_External\Lib\Storage;

use Guzzle\Http\Url;
use Guzzle\Http\Exception\ClientErrorResponseException;
use GuzzleHttp\Psr7\Uri;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use Icewind\Streams\RetryWrapper;
use OpenCloud;
use OpenCloud\Common\Exceptions;
use OpenCloud\OpenStack;
use OpenCloud\Rackspace;
use OpenCloud\ObjectStore\Resource\DataObject;
use OC\Files\ObjectStore\SwiftFactory;
use OCP\Files\StorageBadConfigException;
use OpenStack\Common\Error\BadResponseError;
use OpenStack\ObjectStore\v1\Models\StorageObject;

class Swift extends \OC\Files\Storage\Common {

/**
* @var \OpenCloud\ObjectStore\Service
*/
private $connection;
/** @var SwiftFactory */
private $connectionFactory;
/**
* @var \OpenCloud\ObjectStore\Resource\Container
* @var \OpenStack\ObjectStore\v1\Models\Container
*/
private $container;
/**
* @var \OpenCloud\OpenStack
*/
private $anchor;
/**
* @var string
*/
@@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common {
*/
private $params;

/** @var string */
/** @var string */
private $id;

/** @var \OC\Files\ObjectStore\Swift */
private $objectStore;

/**
* Key value cache mapping path to data object. Maps path to
* \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing
* paths and path to false for not existing paths.
*
* @var \OCP\ICache
*/
private $objectCache;

/**
* @param string $path
* @return mixed|string
*/
private function normalizePath($path) {
private function normalizePath(string $path) {
$path = trim($path, '/');

if (!$path) {
@@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common {
* that one will be returned.
*
* @param string $path
* @return \OpenCloud\ObjectStore\Resource\DataObject|bool object
* @return StorageObject|bool object
* or false if the object did not exist
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function fetchObject($path) {
private function fetchObject(string $path) {
if ($this->objectCache->hasKey($path)) {
// might be "false" if object did not exist from last check
return $this->objectCache->get($path);
}
try {
$object = $this->getContainer()->getPartialObject($path);
$object = $this->getContainer()->getObject($path);
$object->retrieve();
$this->objectCache->set($path, $object);
return $object;
} catch (ClientErrorResponseException $e) {
// this exception happens when the object does not exist, which
// is expected in most cases
$this->objectCache->set($path, false);
return false;
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
// Expected response is "404 Not Found", so only log if it isn't
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
@@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common {
'app' => 'files_external',
]);
}
$this->objectCache->set($path, false);
return false;
}
}
@@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common {
* @param string $path
*
* @return bool true if the object exist, false otherwise
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function doesObjectExist($path) {
return $this->fetchObject($path) !== false;
@@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common {
or empty($params['user']) or empty($params['bucket'])
or empty($params['region'])
) {
throw new \Exception("API Key or password, Username, Bucket and Region have to be configured.");
throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured.");
}

$this->id = 'swift::' . $params['user'] . md5($params['bucket']);

$bucketUrl = Url::factory($params['bucket']);
if ($bucketUrl->isAbsolute()) {
$this->bucket = end($bucketUrl->getPathSegments());
$params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath();
} else {
$this->bucket = $params['bucket'];
$bucketUrl = new Uri($params['bucket']);
if ($bucketUrl->getHost()) {
$params['bucket'] = basename($bucketUrl->getPath());
$params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath()));
}

if (empty($params['url'])) {
@@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common {
$params['service_name'] = 'cloudFiles';
}

$params['autocreate'] = true;

$this->params = $params;
// FIXME: private class...
$this->objectCache = new \OC\Cache\CappedMemoryCache();
$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params);
$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory);
$this->bucket = $params['bucket'];
}

public function mkdir($path) {
@@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common {
}

try {
$customHeaders = array('content-type' => 'httpd/unix-directory');
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate so that the next access gets the real object
// with all properties
$this->objectCache->remove($path);
} catch (Exceptions\CreateUpdateError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common {
}

try {
$this->getContainer()->dataObject()->setName($path . '/')->delete();
$this->objectStore->deleteObject($path . '/');
$this->objectCache->remove($path . '/');
} catch (Exceptions\DeleteError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common {
$path .= '/';
}

$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
// $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #

try {
$files = array();
/** @var OpenCloud\Common\Collection $objects */
$objects = $this->getContainer()->objectList(array(
$files = [];
$objects = $this->getContainer()->listObjects([
'prefix' => $path,
'delimiter' => '/'
));
]);

/** @var OpenCloud\ObjectStore\Resource\DataObject $object */
/** @var StorageObject $object */
foreach ($objects as $object) {
$file = basename($object->getName());
$file = basename($object->name);
if ($file !== basename($path) && $file !== '.') {
$files[] = $file;
}
@@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common {
}

try {
/** @var DataObject $object */
$object = $this->fetchObject($path);
if (!$object) {
return false;
}
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common {
return false;
}

$dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified());
if ($dateTime !== false) {
$mtime = $dateTime->getTimestamp();
} else {
$mtime = null;
}
$dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false;
$mtime = $dateTime ? $dateTime->getTimestamp() : null;
$objectMetadata = $object->getMetadata();
$metaTimestamp = $objectMetadata->getProperty('timestamp');
if (isset($metaTimestamp)) {
$mtime = $metaTimestamp;
if (isset($objectMetadata['timestamp'])) {
$mtime = $objectMetadata['timestamp'];
}

if (!empty($mtime)) {
@@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common {
}

$stat = array();
$stat['size'] = (int)$object->getContentLength();
$stat['size'] = (int)$object->contentLength;
$stat['mtime'] = $mtime;
$stat['atime'] = time();
return $stat;
@@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common {
}

try {
$this->getContainer()->dataObject()->setName($path)->delete();
$this->objectStore->deleteObject($path);
$this->objectCache->remove($path);
$this->objectCache->remove($path . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
]);
throw $e;
}
return false;
}

return true;
@@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common {
case 'r':
case 'rb':
try {
$c = $this->getContainer();
$streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory();
/** @var \OpenCloud\Common\Http\Client $client */
$client = $c->getClient();
$streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path)));
$streamInterface->rewind();
$stream = $streamInterface->getStream();
stream_context_set_option($stream, 'swift','content', $streamInterface);
if(!strrpos($streamInterface
->getMetaData('wrapper_data')[0], '404 Not Found')) {
return RetryWrapper::wrap($stream);
}
return false;
} catch (\Guzzle\Http\Exception\BadResponseException $e) {
return $this->objectStore->readObject($path);
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common {
if (is_null($mtime)) {
$mtime = time();
}
$metadata = array('timestamp' => $mtime);
$metadata = ['timestamp' => $mtime];
if ($this->file_exists($path)) {
if ($this->is_dir($path) && $path !== '.') {
$path .= '/';
}

$object = $this->fetchObject($path);
if ($object->saveMetadata($metadata)) {
if ($object->mergeMetadata($metadata)) {
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
}
return true;
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
return true;
@@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common {
$path2 = $this->normalizePath($path2);

$fileType = $this->filetype($path1);
if ($fileType === 'file') {

if ($fileType) {
// make way
$this->unlink($path2);
}

if ($fileType === 'file') {
try {
$source = $this->fetchObject($path1);
$source->copy($this->bucket . '/' . $path2);
$source->copy([
'destination' => $this->bucket . '/' . $path2
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common {
}

} else if ($fileType === 'dir') {

// make way
$this->unlink($path2);

try {
$source = $this->fetchObject($path1 . '/');
$source->copy($this->bucket . '/' . $path2 . '/');
$source->copy([
'destination' => $this->bucket . '/' . $path2 . '/'
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common {

// cleanup
if ($this->unlink($path1) === false) {
throw new \Exception('failed to remove original');
$this->unlink($path2);
return false;
}
@@ -567,81 +552,27 @@ class Swift extends \OC\Files\Storage\Common {
return $this->id;
}

/**
* Returns the connection
*
* @return OpenCloud\ObjectStore\Service connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}

$settings = array(
'username' => $this->params['user'],
);

if (!empty($this->params['password'])) {
$settings['password'] = $this->params['password'];
} else if (!empty($this->params['key'])) {
$settings['apiKey'] = $this->params['key'];
}

if (!empty($this->params['tenant'])) {
$settings['tenantName'] = $this->params['tenant'];
}

if (!empty($this->params['timeout'])) {
$settings['timeout'] = $this->params['timeout'];
}

if (isset($settings['apiKey'])) {
$this->anchor = new Rackspace($this->params['url'], $settings);
} else {
$this->anchor = new OpenStack($this->params['url'], $settings);
}

$connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);

if (!empty($this->params['endpoint_url'])) {
$endpoint = $connection->getEndpoint();
$endpoint->setPublicUrl($this->params['endpoint_url']);
$endpoint->setPrivateUrl($this->params['endpoint_url']);
$connection->setEndpoint($endpoint);
}

$this->connection = $connection;

return $this->connection;
}

/**
* Returns the initialized object store container.
*
* @return OpenCloud\ObjectStore\Resource\Container
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
public function getContainer() {
if (!is_null($this->container)) {
return $this->container;
}
if (is_null($this->container)) {
$this->container = $this->connectionFactory->getContainer();

try {
$this->container = $this->getConnection()->getContainer($this->bucket);
} catch (ClientErrorResponseException $e) {
$this->container = $this->getConnection()->createContainer($this->bucket);
}

if (!$this->file_exists('.')) {
$this->mkdir('.');
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
}

return $this->container;
}

public function writeBack($tmpFile, $path) {
$fileData = fopen($tmpFile, 'r');
$this->getContainer()->uploadObject($path, $fileData);
$this->objectStore->writeObject($path, $fileData);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
unlink($tmpFile);

+ 10
- 6
apps/files_external/tests/Storage/SwiftTest.php View File

@@ -27,6 +27,7 @@

namespace OCA\Files_External\Tests\Storage;

use GuzzleHttp\Exception\ClientException;
use \OCA\Files_External\Lib\Storage\Swift;

/**
@@ -40,6 +41,11 @@ class SwiftTest extends \Test\Files\Storage\Storage {

private $config;

/**
* @var Swift instance
*/
protected $instance;

protected function setUp() {
parent::setUp();

@@ -53,17 +59,15 @@ class SwiftTest extends \Test\Files\Storage\Storage {
protected function tearDown() {
if ($this->instance) {
try {
$connection = $this->instance->getConnection();
$container = $connection->getContainer($this->config['bucket']);
$container = $this->instance->getContainer();

$objects = $container->objectList();
while($object = $objects->next()) {
$object->setName(str_replace('#','%23',$object->getName()));
$objects = $container->listObjects();
foreach ($objects as $object) {
$object->delete();
}

$container->delete();
} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
} catch (ClientException $e) {
// container didn't exist, so we don't need to delete it
}
}

+ 5
- 8
lib/private/Files/ObjectStore/Swift.php View File

@@ -41,24 +41,21 @@ class Swift implements IObjectStore {
*/
private $container = null;

/** @var SwiftConnectionFactory */
/** @var SwiftFactory */
private $swiftFactory;

public function __construct($params) {
$this->swiftFactory = new SwiftConnectionFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'));
public function __construct($params, SwiftFactory $connectionFactory = null) {
$this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params);
$this->params = $params;
}

/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws \Exception
* @throws \OCP\Files\StorageNotAvailableException
*/
private function getContainer() {
if (is_null($this->container)) {
$this->container = $this->swiftFactory->getContainer($this->params);
}
return $this->container;
return $this->swiftFactory->getContainer();
}

/**

lib/private/Files/ObjectStore/SwiftConnectionFactory.php → lib/private/Files/ObjectStore/SwiftFactory.php View File

@@ -36,12 +36,17 @@ use OpenStack\Identity\v2\Service;
use OpenStack\OpenStack;
use OpenStack\Common\Transport\Utils as TransportUtils;
use Psr\Http\Message\RequestInterface;
use OpenStack\ObjectStore\v1\Models\Container;

class SwiftConnectionFactory {
class SwiftFactory {
private $cache;
private $params;
/** @var Container|null */
private $container = null;

public function __construct(ICache $cache) {
public function __construct(ICache $cache, array $params) {
$this->cache = $cache;
$this->params = $params;
}

private function getCachedToken(string $cacheKey) {
@@ -58,42 +63,46 @@ class SwiftConnectionFactory {
}

/**
* @param array $params
* @return OpenStack
* @throws StorageAuthException
* @throws \Exception
*/
private function getClient(array &$params) {
if (isset($params['bucket'])) {
$params['container'] = $params['bucket'];
private function getClient() {
if (isset($this->params['bucket'])) {
$this->params['container'] = $this->params['bucket'];
}
if (!isset($params['container'])) {
$params['container'] = 'owncloud';
if (!isset($this->params['container'])) {
$this->params['container'] = 'owncloud';
}
if (!isset($params['autocreate'])) {
if (!isset($this->params['autocreate'])) {
// should only be true for tests
$params['autocreate'] = false;
$this->params['autocreate'] = false;
}
if (!isset($this->params['username']) && isset($this->params['user'])) {
$this->params['username'] = $this->params['user'];
}
if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) {
$this->params['tenantName'] = $this->params['tenant'];
}

$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
$cacheKey = $this->params['username'] . '@' . $this->params['url'] . '/' . $this->params['bucket'];
$token = $this->getCachedToken($cacheKey);
$hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now'));
if ($hasToken) {
$params['cachedToken'] = $token;
$this->params['cachedToken'] = $token;
}
$httpClient = new Client([
'base_uri' => TransportUtils::normalizeUrl($params['url']),
'base_uri' => TransportUtils::normalizeUrl($this->params['url']),
'handler' => HandlerStack::create()
]);

$authService = Service::factory($httpClient);
$params['identityService'] = $authService;
$params['authUrl'] = $params['url'];
$client = new OpenStack($params);
$this->params['identityService'] = $authService;
$this->params['authUrl'] = $this->params['url'];
$client = new OpenStack($this->params);

if (!$hasToken) {
try {
$token = $authService->generateToken($params);
$token = $authService->generateToken($this->params);
$this->cacheToken($token, $cacheKey);
} catch (ConnectException $e) {
throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e);
@@ -117,19 +126,30 @@ class SwiftConnectionFactory {
}

/**
* @param array $params
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws \Exception
* @throws StorageNotAvailableException
*/
public function getContainer(array $params) {
public function getContainer() {
if (is_null($this->container)) {
$this->container = $this->createContainer();
}

return $this->container;
}

$client = $this->getClient($params);
/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws StorageNotAvailableException
*/
private function createContainer() {
$client = $this->getClient();
$objectStoreService = $client->objectStoreV1();

$autoCreate = isset($params['autocreate']) && $params['autocreate'] === true;
$autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true;
try {
$container = $objectStoreService->getContainer($params['container']);
$container = $objectStoreService->getContainer($this->params['container']);
if ($autoCreate) {
$container->getMetadata();
}
@@ -138,10 +158,10 @@ class SwiftConnectionFactory {
// if the container does not exist and autocreate is true try to create the container on the fly
if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) {
return $objectStoreService->createContainer([
'name' => $params['container']
'name' => $this->params['container']
]);
} else {
throw $ex;
throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $e);
}
} catch (ConnectException $e) {
/** @var RequestInterface $request */

Write Preview
Loading…
Cancel
Save