From 09d80ac9fc723793ee6e81defe5bd667b4e009a4 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Wed, 22 Oct 2014 14:50:22 +0200 Subject: [PATCH] Lazy initialize external storages Backport of 075e8d8e8658913e1c5b8869f3e457fa6db2d847 from master --- apps/files_external/lib/amazons3.php | 100 ++++++++++++---------- apps/files_external/lib/dropbox.php | 1 + apps/files_external/lib/ftp.php | 2 +- apps/files_external/lib/google.php | 1 + apps/files_external/lib/sftp.php | 39 ++++++--- apps/files_external/lib/swift.php | 119 +++++++++++++++++---------- 6 files changed, 159 insertions(+), 103 deletions(-) diff --git a/apps/files_external/lib/amazons3.php b/apps/files_external/lib/amazons3.php index 06910e04f2a..c73860d99a8 100644 --- a/apps/files_external/lib/amazons3.php +++ b/apps/files_external/lib/amazons3.php @@ -111,34 +111,6 @@ class AmazonS3 extends \OC\Files\Storage\Common { $params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443; } $base_url = $scheme . '://' . $params['hostname'] . ':' . $params['port'] . '/'; - - $this->connection = S3Client::factory(array( - 'key' => $params['key'], - 'secret' => $params['secret'], - 'base_url' => $base_url, - 'region' => $params['region'] - )); - - if (!$this->connection->isValidBucketName($this->bucket)) { - throw new \Exception("The configured bucket name is invalid."); - } - - if (!$this->connection->doesBucketExist($this->bucket)) { - try { - $this->connection->createBucket(array( - 'Bucket' => $this->bucket - )); - $this->connection->waitUntilBucketExists(array( - 'Bucket' => $this->bucket, - 'waiter.interval' => 1, - 'waiter.max_attempts' => 15 - )); - $this->testTimeout(); - } catch (S3Exception $e) { - \OCP\Util::logException('files_external', $e); - throw new \Exception('Creation of bucket failed. '.$e->getMessage()); - } - } } /** @@ -181,7 +153,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { } try { - $this->connection->putObject(array( + $this->getConnection()->putObject(array( 'Bucket' => $this->bucket, 'Key' => $path . '/', 'ContentType' => 'httpd/unix-directory' @@ -216,7 +188,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { protected function clearBucket() { try { - $this->connection->clearBucket($this->bucket); + $this->getConnection()->clearBucket($this->bucket); return true; // clearBucket() is not working with Ceph, so if it fails we try the slower approach } catch (\Exception $e) { @@ -237,9 +209,9 @@ class AmazonS3 extends \OC\Files\Storage\Common { // to delete all objects prefixed with the path. do { // instead of the iterator, manually loop over the list ... - $objects = $this->connection->listObjects($params); + $objects = $this->getConnection()->listObjects($params); // ... so we can delete the files in batches - $this->connection->deleteObjects(array( + $this->getConnection()->deleteObjects(array( 'Bucket' => $this->bucket, 'Objects' => $objects['Contents'] )); @@ -264,7 +236,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { try { $files = array(); - $result = $this->connection->getIterator('ListObjects', array( + $result = $this->getConnection()->getIterator('ListObjects', array( 'Bucket' => $this->bucket, 'Delimiter' => '/', 'Prefix' => $path @@ -299,7 +271,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { $stat['size'] = -1; //unknown $stat['mtime'] = time() - $this->rescanDelay * 1000; } else { - $result = $this->connection->headObject(array( + $result = $this->getConnection()->headObject(array( 'Bucket' => $this->bucket, 'Key' => $path )); @@ -328,10 +300,10 @@ class AmazonS3 extends \OC\Files\Storage\Common { } try { - if ($this->connection->doesObjectExist($this->bucket, $path)) { + if ($this->getConnection()->doesObjectExist($this->bucket, $path)) { return 'file'; } - if ($this->connection->doesObjectExist($this->bucket, $path.'/')) { + if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) { return 'dir'; } } catch (S3Exception $e) { @@ -350,7 +322,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { } try { - $this->connection->deleteObject(array( + $this->getConnection()->deleteObject(array( 'Bucket' => $this->bucket, 'Key' => $path )); @@ -373,7 +345,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { self::$tmpFiles[$tmpFile] = $path; try { - $this->connection->getObject(array( + $this->getConnection()->getObject(array( 'Bucket' => $this->bucket, 'Key' => $path, 'SaveAs' => $tmpFile @@ -421,7 +393,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { return 'httpd/unix-directory'; } else if ($this->file_exists($path)) { try { - $result = $this->connection->headObject(array( + $result = $this->getConnection()->headObject(array( 'Bucket' => $this->bucket, 'Key' => $path )); @@ -449,7 +421,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { if ($fileType === 'dir' && ! $this->isRoot($path)) { $path .= '/'; } - $this->connection->copyObject(array( + $this->getConnection()->copyObject(array( 'Bucket' => $this->bucket, 'Key' => $this->cleanKey($path), 'Metadata' => $metadata, @@ -458,7 +430,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { $this->testTimeout(); } else { $mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path); - $this->connection->putObject(array( + $this->getConnection()->putObject(array( 'Bucket' => $this->bucket, 'Key' => $this->cleanKey($path), 'Metadata' => $metadata, @@ -480,7 +452,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { if ($this->is_file($path1)) { try { - $this->connection->copyObject(array( + $this->getConnection()->copyObject(array( 'Bucket' => $this->bucket, 'Key' => $this->cleanKey($path2), 'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1) @@ -494,7 +466,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { $this->remove($path2); try { - $this->connection->copyObject(array( + $this->getConnection()->copyObject(array( 'Bucket' => $this->bucket, 'Key' => $path2 . '/', 'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/') @@ -552,7 +524,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { } public function test() { - $test = $this->connection->getBucketAcl(array( + $test = $this->getConnection()->getBucketAcl(array( 'Bucket' => $this->bucket, )); if (isset($test) && !is_null($test->getPath('Owner/ID'))) { @@ -565,7 +537,45 @@ class AmazonS3 extends \OC\Files\Storage\Common { return $this->id; } + /** + * Returns the connection + * + * @return S3Client connected client + * @throws \Exception if connection could not be made + */ public function getConnection() { + if (!is_null($this->connection)) { + return $this->connection; + } + + $this->connection = S3Client::factory(array( + 'key' => $params['key'], + 'secret' => $params['secret'], + 'base_url' => $base_url, + 'region' => $params['region'] + )); + + if (!$this->connection->isValidBucketName($this->bucket)) { + throw new \Exception("The configured bucket name is invalid."); + } + + if (!$this->connection->doesBucketExist($this->bucket)) { + try { + $this->connection->createBucket(array( + 'Bucket' => $this->bucket + )); + $this->connection->waitUntilBucketExists(array( + 'Bucket' => $this->bucket, + 'waiter.interval' => 1, + 'waiter.max_attempts' => 15 + )); + $this->testTimeout(); + } catch (S3Exception $e) { + \OCP\Util::logException('files_external', $e); + throw new \Exception('Creation of bucket failed. '.$e->getMessage()); + } + } + return $this->connection; } @@ -575,7 +585,7 @@ class AmazonS3 extends \OC\Files\Storage\Common { } try { - $this->connection->putObject(array( + $this->getConnection()->putObject(array( 'Bucket' => $this->bucket, 'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]), 'SourceFile' => $tmpFile, diff --git a/apps/files_external/lib/dropbox.php b/apps/files_external/lib/dropbox.php index 9f297d22dcb..cc1e628f851 100755 --- a/apps/files_external/lib/dropbox.php +++ b/apps/files_external/lib/dropbox.php @@ -44,6 +44,7 @@ class Dropbox extends \OC\Files\Storage\Common { $this->id = 'dropbox::'.$params['app_key'] . $params['token']. '/' . $this->root; $oauth = new \Dropbox_OAuth_Curl($params['app_key'], $params['app_secret']); $oauth->setToken($params['token'], $params['token_secret']); + // note: Dropbox_API connection is lazy $this->dropbox = new \Dropbox_API($oauth, 'auto'); } else { throw new \Exception('Creating \OC\Files\Storage\Dropbox storage failed'); diff --git a/apps/files_external/lib/ftp.php b/apps/files_external/lib/ftp.php index 2650a94f85e..4a995d21157 100644 --- a/apps/files_external/lib/ftp.php +++ b/apps/files_external/lib/ftp.php @@ -39,7 +39,7 @@ class FTP extends \OC\Files\Storage\StreamWrapper{ $this->root .= '/'; } } else { - throw new \Exception(); + throw new \Exception('Creating \OC\Files\Storage\FTP storage failed'); } } diff --git a/apps/files_external/lib/google.php b/apps/files_external/lib/google.php index 56c0d451651..e71c7ab9621 100644 --- a/apps/files_external/lib/google.php +++ b/apps/files_external/lib/google.php @@ -52,6 +52,7 @@ class Google extends \OC\Files\Storage\Common { $client->setScopes(array('https://www.googleapis.com/auth/drive')); $client->setUseObjects(true); $client->setAccessToken($params['token']); + // note: API connection is lazy $this->service = new \Google_DriveService($client); $token = json_decode($params['token'], true); $this->id = 'google::'.substr($params['client_id'], 0, 30).$token['created']; diff --git a/apps/files_external/lib/sftp.php b/apps/files_external/lib/sftp.php index e0655cc8d3d..e8681296912 100644 --- a/apps/files_external/lib/sftp.php +++ b/apps/files_external/lib/sftp.php @@ -53,6 +53,18 @@ class SFTP extends \OC\Files\Storage\Common { if (substr($this->root, -1, 1) != '/') { $this->root .= '/'; } + } + + /** + * Returns the connection. + * + * @return \Net_SFTP connected client instance + * @throws \Exception when the connection failed + */ + public function getConnection() { + if (!is_null($this->client)) { + return $this->client; + } $hostKeys = $this->readHostKeys(); $this->client = new \Net_SFTP($this->host); @@ -71,6 +83,7 @@ class SFTP extends \OC\Files\Storage\Common { if (!$this->client->login($this->user, $this->password)) { throw new \Exception('Login failed'); } + return $this->client; } public function test() { @@ -81,7 +94,7 @@ class SFTP extends \OC\Files\Storage\Common { ) { return false; } - return $this->client->nlist() !== false; + return $this->getConnection()->nlist() !== false; } public function getId(){ @@ -149,7 +162,7 @@ class SFTP extends \OC\Files\Storage\Common { public function mkdir($path) { try { - return $this->client->mkdir($this->absPath($path)); + return $this->getConnection()->mkdir($this->absPath($path)); } catch (\Exception $e) { return false; } @@ -157,7 +170,7 @@ class SFTP extends \OC\Files\Storage\Common { public function rmdir($path) { try { - return $this->client->delete($this->absPath($path), true); + return $this->getConnection()->delete($this->absPath($path), true); } catch (\Exception $e) { return false; } @@ -165,7 +178,7 @@ class SFTP extends \OC\Files\Storage\Common { public function opendir($path) { try { - $list = $this->client->nlist($this->absPath($path)); + $list = $this->getConnection()->nlist($this->absPath($path)); $id = md5('sftp:' . $path); $dirStream = array(); @@ -183,7 +196,7 @@ class SFTP extends \OC\Files\Storage\Common { public function filetype($path) { try { - $stat = $this->client->stat($this->absPath($path)); + $stat = $this->getConnection()->stat($this->absPath($path)); if ($stat['type'] == NET_SFTP_TYPE_REGULAR) { return 'file'; } @@ -199,7 +212,7 @@ class SFTP extends \OC\Files\Storage\Common { public function file_exists($path) { try { - return $this->client->stat($this->absPath($path)) !== false; + return $this->getConnection()->stat($this->absPath($path)) !== false; } catch (\Exception $e) { return false; } @@ -207,7 +220,7 @@ class SFTP extends \OC\Files\Storage\Common { public function unlink($path) { try { - return $this->client->delete($this->absPath($path), true); + return $this->getConnection()->delete($this->absPath($path), true); } catch (\Exception $e) { return false; } @@ -234,7 +247,7 @@ class SFTP extends \OC\Files\Storage\Common { case 'x+': case 'c': case 'c+': - $context = stream_context_create(array('sftp' => array('session' => $this->client))); + $context = stream_context_create(array('sftp' => array('session' => $this->getConnection()))); return fopen($this->constructUrl($path), $mode, false, $context); } } catch (\Exception $e) { @@ -248,7 +261,7 @@ class SFTP extends \OC\Files\Storage\Common { return false; } if (!$this->file_exists($path)) { - $this->client->put($this->absPath($path), ''); + $this->getConnection()->put($this->absPath($path), ''); } else { return false; } @@ -259,11 +272,11 @@ class SFTP extends \OC\Files\Storage\Common { } public function getFile($path, $target) { - $this->client->get($path, $target); + $this->getConnection()->get($path, $target); } public function uploadFile($path, $target) { - $this->client->put($target, $path, NET_SFTP_LOCAL_FILE); + $this->getConnection()->put($target, $path, NET_SFTP_LOCAL_FILE); } public function rename($source, $target) { @@ -271,7 +284,7 @@ class SFTP extends \OC\Files\Storage\Common { if (!$this->is_dir($target) && $this->file_exists($target)) { $this->unlink($target); } - return $this->client->rename( + return $this->getConnection()->rename( $this->absPath($source), $this->absPath($target) ); @@ -282,7 +295,7 @@ class SFTP extends \OC\Files\Storage\Common { public function stat($path) { try { - $stat = $this->client->stat($this->absPath($path)); + $stat = $this->getConnection()->stat($this->absPath($path)); $mtime = $stat ? $stat['mtime'] : -1; $size = $stat ? $stat['size'] : 0; diff --git a/apps/files_external/lib/swift.php b/apps/files_external/lib/swift.php index 3b298df8c32..35f75d4cc83 100644 --- a/apps/files_external/lib/swift.php +++ b/apps/files_external/lib/swift.php @@ -47,6 +47,12 @@ class Swift extends \OC\Files\Storage\Common { * @var string */ private $bucket; + /** + * Connection parameters + * + * @var array + */ + private $params; /** * @var array */ @@ -84,7 +90,7 @@ class Swift extends \OC\Files\Storage\Common { */ private function doesObjectExist($path) { try { - $this->container->getPartialObject($path); + $this->getContainer()->getPartialObject($path); return true; } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); @@ -110,37 +116,7 @@ class Swift extends \OC\Files\Storage\Common { $params['service_name'] = 'cloudFiles'; } - $settings = array( - 'username' => $params['user'], - ); - - if (isset($params['password'])) { - $settings['password'] = $params['password']; - } else if (isset($params['key'])) { - $settings['apiKey'] = $params['key']; - } - - if (isset($params['tenant'])) { - $settings['tenantName'] = $params['tenant']; - } - - if (isset($params['timeout'])) { - $settings['timeout'] = $params['timeout']; - } - - $this->anchor = new OpenStack($params['url'], $settings); - - $this->connection = $this->anchor->objectStoreService($params['service_name'], $params['region']); - - try { - $this->container = $this->connection->getContainer($this->bucket); - } catch (ClientErrorResponseException $e) { - $this->container = $this->connection->createContainer($this->bucket); - } - - if (!$this->file_exists('.')) { - $this->mkdir('.'); - } + $this->params = $params; } public function mkdir($path) { @@ -158,7 +134,7 @@ class Swift extends \OC\Files\Storage\Common { $customHeaders = array('content-type' => 'httpd/unix-directory'); $metadataHeaders = DataObject::stockHeaders(array()); $allHeaders = $customHeaders + $metadataHeaders; - $this->container->uploadObject($path, '', $allHeaders); + $this->getContainer()->uploadObject($path, '', $allHeaders); } catch (Exceptions\CreateUpdateError $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); return false; @@ -198,7 +174,7 @@ class Swift extends \OC\Files\Storage\Common { } try { - $this->container->dataObject()->setName($path . '/')->delete(); + $this->getContainer()->dataObject()->setName($path . '/')->delete(); } catch (Exceptions\DeleteError $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); return false; @@ -221,7 +197,7 @@ class Swift extends \OC\Files\Storage\Common { try { $files = array(); /** @var OpenCloud\Common\Collection $objects */ - $objects = $this->container->objectList(array( + $objects = $this->getContainer()->objectList(array( 'prefix' => $path, 'delimiter' => '/' )); @@ -251,7 +227,7 @@ class Swift extends \OC\Files\Storage\Common { } try { - $object = $this->container->getPartialObject($path); + $object = $this->getContainer()->getPartialObject($path); } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); return false; @@ -304,7 +280,7 @@ class Swift extends \OC\Files\Storage\Common { } try { - $this->container->dataObject()->setName($path)->delete(); + $this->getContainer()->dataObject()->setName($path)->delete(); } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); return false; @@ -322,7 +298,7 @@ class Swift extends \OC\Files\Storage\Common { $tmpFile = \OC_Helper::tmpFile(); self::$tmpFiles[$tmpFile] = $path; try { - $object = $this->container->getObject($path); + $object = $this->getContainer()->getObject($path); } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); return false; @@ -375,7 +351,7 @@ class Swift extends \OC\Files\Storage\Common { if ($this->is_dir($path)) { return 'httpd/unix-directory'; } else if ($this->file_exists($path)) { - $object = $this->container->getPartialObject($path); + $object = $this->getContainer()->getPartialObject($path); return $object->getContentType(); } return false; @@ -392,7 +368,7 @@ class Swift extends \OC\Files\Storage\Common { $path .= '/'; } - $object = $this->container->getPartialObject($path); + $object = $this->getContainer()->getPartialObject($path); $object->saveMetadata($metadata); return true; } else { @@ -400,7 +376,7 @@ class Swift extends \OC\Files\Storage\Common { $customHeaders = array('content-type' => $mimeType); $metadataHeaders = DataObject::stockHeaders($metadata); $allHeaders = $customHeaders + $metadataHeaders; - $this->container->uploadObject($path, '', $allHeaders); + $this->getContainer()->uploadObject($path, '', $allHeaders); return true; } } @@ -416,7 +392,7 @@ class Swift extends \OC\Files\Storage\Common { $this->unlink($path2); try { - $source = $this->container->getPartialObject($path1); + $source = $this->getContainer()->getPartialObject($path1); $source->copy($this->bucket.'/'.$path2); } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); @@ -429,7 +405,7 @@ class Swift extends \OC\Files\Storage\Common { $this->unlink($path2); try { - $source = $this->container->getPartialObject($path1 . '/'); + $source = $this->getContainer()->getPartialObject($path1 . '/'); $source->copy($this->bucket.'/'.$path2 . '/'); } catch (ClientErrorResponseException $e) { \OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR); @@ -487,16 +463,71 @@ class Swift extends \OC\Files\Storage\Common { return $this->id; } + /** + * Returns the connection + * + * @return OpenCloud\ObjectStore\Service connected client + * @throws \Exception if connection could not be made + */ public function getConnection() { + if (!is_null($this->connection)) { + return $this->connection; + } + + $settings = array( + 'username' => $this->params['user'], + ); + + if (!empty($this->params['password'])) { + $settings['password'] = $this->params['password']; + } else if (!empty($this->params['key'])) { + $settings['apiKey'] = $this->params['key']; + } + + if (!empty($this->params['tenant'])) { + $settings['tenantName'] = $this->params['tenant']; + } + + if (!empty($this->params['timeout'])) { + $settings['timeout'] = $this->params['timeout']; + } + + $this->anchor = new OpenStack($this->params['url'], $settings); + + $this->connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']); + return $this->connection; } + /** + * Returns the initialized object store container. + * + * @return OpenCloud\ObjectStore\Resource\Container + */ + public function getContainer() { + if (!is_null($this->container)) { + return $this->container; + } + + try { + $this->container = $this->getConnection()->getContainer($this->bucket); + } catch (ClientErrorResponseException $e) { + $this->container = $this->getConnection()->createContainer($this->bucket); + } + + if (!$this->file_exists('.')) { + $this->mkdir('.'); + } + + return $this->container; + } + public function writeBack($tmpFile) { if (!isset(self::$tmpFiles[$tmpFile])) { return false; } $fileData = fopen($tmpFile, 'r'); - $this->container->uploadObject(self::$tmpFiles[$tmpFile], $fileData); + $this->getContainer()->uploadObject(self::$tmpFiles[$tmpFile], $fileData); unlink($tmpFile); } -- 2.39.5