From df6e23c98ce3d5a50c417a085f2095f42b663efa Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 7 Feb 2018 17:16:54 +0100 Subject: add ci for swift primary storage Signed-off-by: Robin Appelman --- .drone.yml | 11 +++++++++++ autotest.sh | 3 +++ tests/drone-wait-objectstore.sh | 10 ++++++++++ tests/lib/Files/ObjectStore/SwiftTest.php | 3 +++ tests/preseed-config.php | 16 ++++++++++++++++ 5 files changed, 43 insertions(+) create mode 100755 tests/drone-wait-objectstore.sh diff --git a/.drone.yml b/.drone.yml index 385138d6c86..614c0dc40c2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -596,6 +596,7 @@ pipeline: image: nextcloudci/php7.0:php7.0-19 commands: - phpenmod xdebug + - ./tests/drone-wait-objectstore.sh - TEST_SELECTION=PRIMARY-${OBJECT_STORE} ./autotest.sh sqlite - wget https://codecov.io/bash -O codecov.sh - sh -c "if [ '$DRONE_BUILD_EVENT' = 'pull_request' ]; then bash codecov.sh -B $DRONE_BRANCH -C $DRONE_COMMIT -P $DRONE_PULL_REQUEST -t 117641e2-a9e8-4b7b-984b-ae872d9b05f5 -f tests/autotest-clover-sqlite.xml; fi" @@ -743,6 +744,8 @@ matrix: - TESTS: carddavtester-old-endpoint - TESTS: object-store OBJECT_STORE: s3 + - TESTS: object-store + OBJECT_STORE: swift - TESTS: sqlite-php7.0-samba-native - TESTS: sqlite-php7.0-samba-non-native - TEST: memcache-memcached @@ -838,6 +841,14 @@ services: when: matrix: OBJECT_STORE: s3 + dockswift: + image: icewind1991/dockswift + environment: + - INITIALIZE=yes + - IPADDRESS=dockswift + when: + matrix: + OBJECT_STORE: swift selenium: image: selenium/standalone-firefox:2.53.1-beryllium environment: diff --git a/autotest.sh b/autotest.sh index 7a91b077193..ff59386e1bc 100755 --- a/autotest.sh +++ b/autotest.sh @@ -369,6 +369,9 @@ function execute_tests { if [ "$TEST_SELECTION" == "PRIMARY-s3" ]; then GROUP='--group PRIMARY-s3' fi + if [ "$TEST_SELECTION" == "PRIMARY-swift" ]; then + GROUP='--group PRIMARY-swift' + fi COVER='' if [ -z "$NOCOVERAGE" ]; then diff --git a/tests/drone-wait-objectstore.sh b/tests/drone-wait-objectstore.sh new file mode 100755 index 00000000000..1e5a13ee55e --- /dev/null +++ b/tests/drone-wait-objectstore.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ "$OBJECT_STORE" == "swift" ]; then + echo "waiting for swift" + until curl -I http://dockswift:5000/v3 + do + sleep 2 + done + sleep 30 +fi diff --git a/tests/lib/Files/ObjectStore/SwiftTest.php b/tests/lib/Files/ObjectStore/SwiftTest.php index f2aeace769c..1ea55a84628 100644 --- a/tests/lib/Files/ObjectStore/SwiftTest.php +++ b/tests/lib/Files/ObjectStore/SwiftTest.php @@ -23,6 +23,9 @@ namespace Test\Files\ObjectStore; use OC\Files\ObjectStore\Swift; +/** + * @group PRIMARY-swift + */ class SwiftTest extends ObjectStoreTest { /** * @return \OCP\Files\ObjectStore\IObjectStore diff --git a/tests/preseed-config.php b/tests/preseed-config.php index 7c080038734..5fbdc565417 100644 --- a/tests/preseed-config.php +++ b/tests/preseed-config.php @@ -34,3 +34,19 @@ if (getenv('OBJECT_STORE') === 's3') { ) ]; } +if (getenv('OBJECT_STORE') === 'swift') { + $swiftHost = getenv('DRONE') === 'true' ? 'dockswift' : 'localhost'; + $CONFIG['objectstore'] = [ + 'class' => 'OC\\Files\\ObjectStore\\Swift', + 'arguments' => array( + 'autocreate' => true, + 'username' => 'swift', + 'tenantName' => 'service', + 'password' => 'swift', + 'serviceName' => 'swift', + 'region' => 'regionOne', + 'url' => "http://$swiftHost:5000/v2.0", + 'bucket' => 'nextcloud' + ) + ]; +} -- cgit v1.2.3 From 85d23dd6fcefe0acf07b152c9ae57824e02e0429 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 7 Feb 2018 17:26:42 +0100 Subject: better error message on catalog error Signed-off-by: Robin Appelman --- lib/private/Files/Mount/ObjectHomeMountProvider.php | 3 ++- lib/private/Files/ObjectStore/Swift.php | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 440a8bc4608..9c09c0737c8 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -84,10 +84,11 @@ class ObjectHomeMountProvider implements IHomeMountProvider { if (!isset($config['arguments'])) { $config['arguments'] = []; } - $config['arguments']['user'] = $user; // instantiate object store implementation $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + $config['arguments']['user'] = $user; + return $config; } diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index 70bc4ed8438..f9c53fd333f 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -117,6 +117,10 @@ class Swift implements IObjectStore { /** @var Catalog $catalog */ $catalog = $this->client->getCatalog(); + if (count($catalog->getItems()) === 0) { + throw new StorageAuthException('Keystone did not provide a valid catalog, verify the credentials'); + } + if (isset($this->params['serviceName'])) { $serviceName = $this->params['serviceName']; } else { -- cgit v1.2.3 From e34c20af7dcbd077a3ed028f75deea302cf7ecfd Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 7 Feb 2018 22:06:07 +0100 Subject: don't swallow exception from touch Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/ObjectStoreStorage.php | 2 +- lib/private/Files/ObjectStore/Swift.php | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 094a2915730..45c22a81a7b 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -369,7 +369,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { 'app' => 'objectstore', 'message' => 'Could not create object for ' . $path, ]); - return false; + throw $ex; } } return true; diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index f9c53fd333f..e07903a00bc 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -26,6 +26,7 @@ namespace OC\Files\ObjectStore; use Guzzle\Http\Exception\ClientErrorResponseException; +use Guzzle\Http\Exception\CurlException; use Icewind\Streams\RetryWrapper; use OCP\Files\ObjectStore\IObjectStore; use OCP\Files\StorageAuthException; @@ -157,6 +158,13 @@ class Swift implements IObjectStore { } else { throw $ex; } + } catch (CurlException $e) { + if ($e->getErrorNo() === 7) { + $host = $e->getCurlHandle()->getUrl()->getHost() . ':' . $e->getCurlHandle()->getUrl()->getPort(); + \OC::$server->getLogger()->error("Can't connect to object storage server at $host"); + throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e); + } + throw $e; } } @@ -180,7 +188,7 @@ class Swift implements IObjectStore { $itemClass = new \stdClass(); $itemClass->name = $item['name']; $itemClass->endpoints = array_map(function (array $endpoint) { - return (object) $endpoint; + return (object)$endpoint; }, $item['endpoints']); $itemClass->type = $item['type']; -- cgit v1.2.3 From 3e0ef7e3f217973cc05117dd9e65980e9587e5e9 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 8 Feb 2018 14:10:33 +0100 Subject: Try to silence phan Signed-off-by: Roeland Jago Douma --- lib/private/Files/ObjectStore/Swift.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index e07903a00bc..a3cba488f5f 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -88,6 +88,9 @@ class Swift implements IObjectStore { $this->params = $params; } + /** + * @suppress PhanNonClassMethodCall + */ protected function init() { if ($this->container) { return; -- cgit v1.2.3 From 2e3ed7db9058d948b8c0469701c7f6e2fdc57f08 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 8 Feb 2018 14:21:37 +0100 Subject: sleep more Signed-off-by: Robin Appelman --- tests/drone-wait-objectstore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/drone-wait-objectstore.sh b/tests/drone-wait-objectstore.sh index 1e5a13ee55e..14d0b6f1f67 100755 --- a/tests/drone-wait-objectstore.sh +++ b/tests/drone-wait-objectstore.sh @@ -6,5 +6,5 @@ if [ "$OBJECT_STORE" == "swift" ]; then do sleep 2 done - sleep 30 + sleep 60 fi -- cgit v1.2.3