|
|
@@ -1,4 +1,5 @@ |
|
|
|
<?php |
|
|
|
declare(strict_types=1); |
|
|
|
/** |
|
|
|
* @copyright Copyright (c) 2016, ownCloud, Inc. |
|
|
|
* |
|
|
@@ -39,31 +40,21 @@ |
|
|
|
|
|
|
|
namespace OCA\Files_External\Lib\Storage; |
|
|
|
|
|
|
|
use Guzzle\Http\Url; |
|
|
|
use Guzzle\Http\Exception\ClientErrorResponseException; |
|
|
|
use GuzzleHttp\Psr7\Uri; |
|
|
|
use Icewind\Streams\CallbackWrapper; |
|
|
|
use Icewind\Streams\IteratorDirectory; |
|
|
|
use Icewind\Streams\RetryWrapper; |
|
|
|
use OpenCloud; |
|
|
|
use OpenCloud\Common\Exceptions; |
|
|
|
use OpenCloud\OpenStack; |
|
|
|
use OpenCloud\Rackspace; |
|
|
|
use OpenCloud\ObjectStore\Resource\DataObject; |
|
|
|
use OC\Files\ObjectStore\SwiftFactory; |
|
|
|
use OCP\Files\StorageBadConfigException; |
|
|
|
use OpenStack\Common\Error\BadResponseError; |
|
|
|
use OpenStack\ObjectStore\v1\Models\StorageObject; |
|
|
|
|
|
|
|
class Swift extends \OC\Files\Storage\Common { |
|
|
|
|
|
|
|
/** |
|
|
|
* @var \OpenCloud\ObjectStore\Service |
|
|
|
*/ |
|
|
|
private $connection; |
|
|
|
/** @var SwiftFactory */ |
|
|
|
private $connectionFactory; |
|
|
|
/** |
|
|
|
* @var \OpenCloud\ObjectStore\Resource\Container |
|
|
|
* @var \OpenStack\ObjectStore\v1\Models\Container |
|
|
|
*/ |
|
|
|
private $container; |
|
|
|
/** |
|
|
|
* @var \OpenCloud\OpenStack |
|
|
|
*/ |
|
|
|
private $anchor; |
|
|
|
/** |
|
|
|
* @var string |
|
|
|
*/ |
|
|
@@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
*/ |
|
|
|
private $params; |
|
|
|
|
|
|
|
/** @var string */ |
|
|
|
/** @var string */ |
|
|
|
private $id; |
|
|
|
|
|
|
|
/** @var \OC\Files\ObjectStore\Swift */ |
|
|
|
private $objectStore; |
|
|
|
|
|
|
|
/** |
|
|
|
* Key value cache mapping path to data object. Maps path to |
|
|
|
* \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing |
|
|
|
* paths and path to false for not existing paths. |
|
|
|
* |
|
|
|
* @var \OCP\ICache |
|
|
|
*/ |
|
|
|
private $objectCache; |
|
|
|
|
|
|
|
/** |
|
|
|
* @param string $path |
|
|
|
* @return mixed|string |
|
|
|
*/ |
|
|
|
private function normalizePath($path) { |
|
|
|
private function normalizePath(string $path) { |
|
|
|
$path = trim($path, '/'); |
|
|
|
|
|
|
|
if (!$path) { |
|
|
@@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
* that one will be returned. |
|
|
|
* |
|
|
|
* @param string $path |
|
|
|
* @return \OpenCloud\ObjectStore\Resource\DataObject|bool object |
|
|
|
* @return StorageObject|bool object |
|
|
|
* or false if the object did not exist |
|
|
|
* @throws \OCP\Files\StorageAuthException |
|
|
|
* @throws \OCP\Files\StorageNotAvailableException |
|
|
|
*/ |
|
|
|
private function fetchObject($path) { |
|
|
|
private function fetchObject(string $path) { |
|
|
|
if ($this->objectCache->hasKey($path)) { |
|
|
|
// might be "false" if object did not exist from last check |
|
|
|
return $this->objectCache->get($path); |
|
|
|
} |
|
|
|
try { |
|
|
|
$object = $this->getContainer()->getPartialObject($path); |
|
|
|
$object = $this->getContainer()->getObject($path); |
|
|
|
$object->retrieve(); |
|
|
|
$this->objectCache->set($path, $object); |
|
|
|
return $object; |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
// this exception happens when the object does not exist, which |
|
|
|
// is expected in most cases |
|
|
|
$this->objectCache->set($path, false); |
|
|
|
return false; |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
// Expected response is "404 Not Found", so only log if it isn't |
|
|
|
if ($e->getResponse()->getStatusCode() !== 404) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
@@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
'app' => 'files_external', |
|
|
|
]); |
|
|
|
} |
|
|
|
$this->objectCache->set($path, false); |
|
|
|
return false; |
|
|
|
} |
|
|
|
} |
|
|
@@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
* @param string $path |
|
|
|
* |
|
|
|
* @return bool true if the object exist, false otherwise |
|
|
|
* @throws \OCP\Files\StorageAuthException |
|
|
|
* @throws \OCP\Files\StorageNotAvailableException |
|
|
|
*/ |
|
|
|
private function doesObjectExist($path) { |
|
|
|
return $this->fetchObject($path) !== false; |
|
|
@@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
or empty($params['user']) or empty($params['bucket']) |
|
|
|
or empty($params['region']) |
|
|
|
) { |
|
|
|
throw new \Exception("API Key or password, Username, Bucket and Region have to be configured."); |
|
|
|
throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured."); |
|
|
|
} |
|
|
|
|
|
|
|
$this->id = 'swift::' . $params['user'] . md5($params['bucket']); |
|
|
|
|
|
|
|
$bucketUrl = Url::factory($params['bucket']); |
|
|
|
if ($bucketUrl->isAbsolute()) { |
|
|
|
$this->bucket = end($bucketUrl->getPathSegments()); |
|
|
|
$params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath(); |
|
|
|
} else { |
|
|
|
$this->bucket = $params['bucket']; |
|
|
|
$bucketUrl = new Uri($params['bucket']); |
|
|
|
if ($bucketUrl->getHost()) { |
|
|
|
$params['bucket'] = basename($bucketUrl->getPath()); |
|
|
|
$params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath())); |
|
|
|
} |
|
|
|
|
|
|
|
if (empty($params['url'])) { |
|
|
@@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
$params['service_name'] = 'cloudFiles'; |
|
|
|
} |
|
|
|
|
|
|
|
$params['autocreate'] = true; |
|
|
|
|
|
|
|
$this->params = $params; |
|
|
|
// FIXME: private class... |
|
|
|
$this->objectCache = new \OC\Cache\CappedMemoryCache(); |
|
|
|
$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params); |
|
|
|
$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory); |
|
|
|
$this->bucket = $params['bucket']; |
|
|
|
} |
|
|
|
|
|
|
|
public function mkdir($path) { |
|
|
@@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
$customHeaders = array('content-type' => 'httpd/unix-directory'); |
|
|
|
$metadataHeaders = DataObject::stockHeaders(array()); |
|
|
|
$allHeaders = $customHeaders + $metadataHeaders; |
|
|
|
$this->getContainer()->uploadObject($path, '', $allHeaders); |
|
|
|
$this->getContainer()->createObject([ |
|
|
|
'name' => $path, |
|
|
|
'content' => '', |
|
|
|
'headers' => ['content-type' => 'httpd/unix-directory'] |
|
|
|
]); |
|
|
|
// invalidate so that the next access gets the real object |
|
|
|
// with all properties |
|
|
|
$this->objectCache->remove($path); |
|
|
|
} catch (Exceptions\CreateUpdateError $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
$this->getContainer()->dataObject()->setName($path . '/')->delete(); |
|
|
|
$this->objectStore->deleteObject($path . '/'); |
|
|
|
$this->objectCache->remove($path . '/'); |
|
|
|
} catch (Exceptions\DeleteError $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
$path .= '/'; |
|
|
|
} |
|
|
|
|
|
|
|
$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of # |
|
|
|
// $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of # |
|
|
|
|
|
|
|
try { |
|
|
|
$files = array(); |
|
|
|
/** @var OpenCloud\Common\Collection $objects */ |
|
|
|
$objects = $this->getContainer()->objectList(array( |
|
|
|
$files = []; |
|
|
|
$objects = $this->getContainer()->listObjects([ |
|
|
|
'prefix' => $path, |
|
|
|
'delimiter' => '/' |
|
|
|
)); |
|
|
|
]); |
|
|
|
|
|
|
|
/** @var OpenCloud\ObjectStore\Resource\DataObject $object */ |
|
|
|
/** @var StorageObject $object */ |
|
|
|
foreach ($objects as $object) { |
|
|
|
$file = basename($object->getName()); |
|
|
|
$file = basename($object->name); |
|
|
|
if ($file !== basename($path) && $file !== '.') { |
|
|
|
$files[] = $file; |
|
|
|
} |
|
|
@@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
/** @var DataObject $object */ |
|
|
|
$object = $this->fetchObject($path); |
|
|
|
if (!$object) { |
|
|
|
return false; |
|
|
|
} |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
return false; |
|
|
|
} |
|
|
|
|
|
|
|
$dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified()); |
|
|
|
if ($dateTime !== false) { |
|
|
|
$mtime = $dateTime->getTimestamp(); |
|
|
|
} else { |
|
|
|
$mtime = null; |
|
|
|
} |
|
|
|
$dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false; |
|
|
|
$mtime = $dateTime ? $dateTime->getTimestamp() : null; |
|
|
|
$objectMetadata = $object->getMetadata(); |
|
|
|
$metaTimestamp = $objectMetadata->getProperty('timestamp'); |
|
|
|
if (isset($metaTimestamp)) { |
|
|
|
$mtime = $metaTimestamp; |
|
|
|
if (isset($objectMetadata['timestamp'])) { |
|
|
|
$mtime = $objectMetadata['timestamp']; |
|
|
|
} |
|
|
|
|
|
|
|
if (!empty($mtime)) { |
|
|
@@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
$stat = array(); |
|
|
|
$stat['size'] = (int)$object->getContentLength(); |
|
|
|
$stat['size'] = (int)$object->contentLength; |
|
|
|
$stat['mtime'] = $mtime; |
|
|
|
$stat['atime'] = time(); |
|
|
|
return $stat; |
|
|
@@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
try { |
|
|
|
$this->getContainer()->dataObject()->setName($path)->delete(); |
|
|
|
$this->objectStore->deleteObject($path); |
|
|
|
$this->objectCache->remove($path); |
|
|
|
$this->objectCache->remove($path . '/'); |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
if ($e->getResponse()->getStatusCode() !== 404) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
|
]); |
|
|
|
throw $e; |
|
|
|
} |
|
|
|
return false; |
|
|
|
} |
|
|
|
|
|
|
|
return true; |
|
|
@@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
case 'r': |
|
|
|
case 'rb': |
|
|
|
try { |
|
|
|
$c = $this->getContainer(); |
|
|
|
$streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory(); |
|
|
|
/** @var \OpenCloud\Common\Http\Client $client */ |
|
|
|
$client = $c->getClient(); |
|
|
|
$streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path))); |
|
|
|
$streamInterface->rewind(); |
|
|
|
$stream = $streamInterface->getStream(); |
|
|
|
stream_context_set_option($stream, 'swift','content', $streamInterface); |
|
|
|
if(!strrpos($streamInterface |
|
|
|
->getMetaData('wrapper_data')[0], '404 Not Found')) { |
|
|
|
return RetryWrapper::wrap($stream); |
|
|
|
} |
|
|
|
return false; |
|
|
|
} catch (\Guzzle\Http\Exception\BadResponseException $e) { |
|
|
|
return $this->objectStore->readObject($path); |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
if (is_null($mtime)) { |
|
|
|
$mtime = time(); |
|
|
|
} |
|
|
|
$metadata = array('timestamp' => $mtime); |
|
|
|
$metadata = ['timestamp' => $mtime]; |
|
|
|
if ($this->file_exists($path)) { |
|
|
|
if ($this->is_dir($path) && $path !== '.') { |
|
|
|
$path .= '/'; |
|
|
|
} |
|
|
|
|
|
|
|
$object = $this->fetchObject($path); |
|
|
|
if ($object->saveMetadata($metadata)) { |
|
|
|
if ($object->mergeMetadata($metadata)) { |
|
|
|
// invalidate target object to force repopulation on fetch |
|
|
|
$this->objectCache->remove($path); |
|
|
|
} |
|
|
|
return true; |
|
|
|
} else { |
|
|
|
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path); |
|
|
|
$customHeaders = array('content-type' => $mimeType); |
|
|
|
$metadataHeaders = DataObject::stockHeaders($metadata); |
|
|
|
$allHeaders = $customHeaders + $metadataHeaders; |
|
|
|
$this->getContainer()->uploadObject($path, '', $allHeaders); |
|
|
|
$this->getContainer()->createObject([ |
|
|
|
'name' => $path, |
|
|
|
'content' => '', |
|
|
|
'headers' => ['content-type' => 'httpd/unix-directory'] |
|
|
|
]); |
|
|
|
// invalidate target object to force repopulation on fetch |
|
|
|
$this->objectCache->remove($path); |
|
|
|
return true; |
|
|
@@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
$path2 = $this->normalizePath($path2); |
|
|
|
|
|
|
|
$fileType = $this->filetype($path1); |
|
|
|
if ($fileType === 'file') { |
|
|
|
|
|
|
|
if ($fileType) { |
|
|
|
// make way |
|
|
|
$this->unlink($path2); |
|
|
|
} |
|
|
|
|
|
|
|
if ($fileType === 'file') { |
|
|
|
try { |
|
|
|
$source = $this->fetchObject($path1); |
|
|
|
$source->copy($this->bucket . '/' . $path2); |
|
|
|
$source->copy([ |
|
|
|
'destination' => $this->bucket . '/' . $path2 |
|
|
|
]); |
|
|
|
// invalidate target object to force repopulation on fetch |
|
|
|
$this->objectCache->remove($path2); |
|
|
|
$this->objectCache->remove($path2 . '/'); |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
} |
|
|
|
|
|
|
|
} else if ($fileType === 'dir') { |
|
|
|
|
|
|
|
// make way |
|
|
|
$this->unlink($path2); |
|
|
|
|
|
|
|
try { |
|
|
|
$source = $this->fetchObject($path1 . '/'); |
|
|
|
$source->copy($this->bucket . '/' . $path2 . '/'); |
|
|
|
$source->copy([ |
|
|
|
'destination' => $this->bucket . '/' . $path2 . '/' |
|
|
|
]); |
|
|
|
// invalidate target object to force repopulation on fetch |
|
|
|
$this->objectCache->remove($path2); |
|
|
|
$this->objectCache->remove($path2 . '/'); |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
} catch (BadResponseError $e) { |
|
|
|
\OC::$server->getLogger()->logException($e, [ |
|
|
|
'level' => \OCP\Util::ERROR, |
|
|
|
'app' => 'files_external', |
|
|
@@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
|
|
|
|
// cleanup |
|
|
|
if ($this->unlink($path1) === false) { |
|
|
|
throw new \Exception('failed to remove original'); |
|
|
|
$this->unlink($path2); |
|
|
|
return false; |
|
|
|
} |
|
|
@@ -567,81 +552,27 @@ class Swift extends \OC\Files\Storage\Common { |
|
|
|
return $this->id; |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* Returns the connection |
|
|
|
* |
|
|
|
* @return OpenCloud\ObjectStore\Service connected client |
|
|
|
* @throws \Exception if connection could not be made |
|
|
|
*/ |
|
|
|
public function getConnection() { |
|
|
|
if (!is_null($this->connection)) { |
|
|
|
return $this->connection; |
|
|
|
} |
|
|
|
|
|
|
|
$settings = array( |
|
|
|
'username' => $this->params['user'], |
|
|
|
); |
|
|
|
|
|
|
|
if (!empty($this->params['password'])) { |
|
|
|
$settings['password'] = $this->params['password']; |
|
|
|
} else if (!empty($this->params['key'])) { |
|
|
|
$settings['apiKey'] = $this->params['key']; |
|
|
|
} |
|
|
|
|
|
|
|
if (!empty($this->params['tenant'])) { |
|
|
|
$settings['tenantName'] = $this->params['tenant']; |
|
|
|
} |
|
|
|
|
|
|
|
if (!empty($this->params['timeout'])) { |
|
|
|
$settings['timeout'] = $this->params['timeout']; |
|
|
|
} |
|
|
|
|
|
|
|
if (isset($settings['apiKey'])) { |
|
|
|
$this->anchor = new Rackspace($this->params['url'], $settings); |
|
|
|
} else { |
|
|
|
$this->anchor = new OpenStack($this->params['url'], $settings); |
|
|
|
} |
|
|
|
|
|
|
|
$connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']); |
|
|
|
|
|
|
|
if (!empty($this->params['endpoint_url'])) { |
|
|
|
$endpoint = $connection->getEndpoint(); |
|
|
|
$endpoint->setPublicUrl($this->params['endpoint_url']); |
|
|
|
$endpoint->setPrivateUrl($this->params['endpoint_url']); |
|
|
|
$connection->setEndpoint($endpoint); |
|
|
|
} |
|
|
|
|
|
|
|
$this->connection = $connection; |
|
|
|
|
|
|
|
return $this->connection; |
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* Returns the initialized object store container. |
|
|
|
* |
|
|
|
* @return OpenCloud\ObjectStore\Resource\Container |
|
|
|
* @return \OpenStack\ObjectStore\v1\Models\Container |
|
|
|
* @throws \OCP\Files\StorageAuthException |
|
|
|
* @throws \OCP\Files\StorageNotAvailableException |
|
|
|
*/ |
|
|
|
public function getContainer() { |
|
|
|
if (!is_null($this->container)) { |
|
|
|
return $this->container; |
|
|
|
} |
|
|
|
if (is_null($this->container)) { |
|
|
|
$this->container = $this->connectionFactory->getContainer(); |
|
|
|
|
|
|
|
try { |
|
|
|
$this->container = $this->getConnection()->getContainer($this->bucket); |
|
|
|
} catch (ClientErrorResponseException $e) { |
|
|
|
$this->container = $this->getConnection()->createContainer($this->bucket); |
|
|
|
} |
|
|
|
|
|
|
|
if (!$this->file_exists('.')) { |
|
|
|
$this->mkdir('.'); |
|
|
|
if (!$this->file_exists('.')) { |
|
|
|
$this->mkdir('.'); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return $this->container; |
|
|
|
} |
|
|
|
|
|
|
|
public function writeBack($tmpFile, $path) { |
|
|
|
$fileData = fopen($tmpFile, 'r'); |
|
|
|
$this->getContainer()->uploadObject($path, $fileData); |
|
|
|
$this->objectStore->writeObject($path, $fileData); |
|
|
|
// invalidate target object to force repopulation on fetch |
|
|
|
$this->objectCache->remove($path); |
|
|
|
unlink($tmpFile); |