reworking and extending \OC\Files\Storage\AmazonS3

1 files changed, 363 insertions, 117 deletions

diff --git a/apps/files_external/lib/amazons3.php b/apps/files_external/lib/amazons3.php
index 7bcefd4176c..e24d9613759 100644
--- a/apps/files_external/lib/amazons3.php
+++ b/apps/files_external/lib/amazons3.php
@@ -4,7 +4,9 @@
  * ownCloud
  *
  * @author Michael Gapczynski
+ * @author Christian Berendt
  * @copyright 2012 Michael Gapczynski mtgap@owncloud.com
+ * @copyright 2013 Christian Berendt berendt@b1-systems.de
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
@@ -22,164 +24,286 @@
 
 namespace OC\Files\Storage;
 
-require_once 'aws-sdk/sdk.class.php';
+set_include_path(get_include_path() . PATH_SEPARATOR .
+        \OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
+require 'aws.phar';
+
+use Aws\S3\S3Client;
+use Aws\S3\Exception\S3Exception;
 
 class AmazonS3 extends \OC\Files\Storage\Common {
 
-	private $s3;
+	private $connection;
 	private $bucket;
-	private $objects = array();
-	private $id;
+	private static $tmpFiles = array();
 
-	private static $tempFiles = array();
+	private function normalizePath($path) {
+		if (substr($path, 0, 1) == '/') {
+			$path = substr($path, 1);
+		}
 
-	// TODO Update to new AWS SDK
+		if (substr($path, -1) == '/') {
+			$path = substr($path, 0, -1);
+		}
+
+		if ( ! $path) {
+			$path = '.';
+		}
+
+		return $path;
+	}
 
 	public function __construct($params) {
-		if (isset($params['key']) && isset($params['secret']) && isset($params['bucket'])) {
-			$this->id = 'amazon::' . $params['key'] . md5($params['secret']);
-			$this->s3 = new \AmazonS3(array('key' => $params['key'], 'secret' => $params['secret']));
-			$this->bucket = $params['bucket'];
-		} else {
+		if ( ! isset($params['key']) || ! isset($params['secret']) || ! isset($params['bucket'])) {
 			throw new \Exception();
 		}
-	}
 
-	private function getObject($path) {
-		if (array_key_exists($path, $this->objects)) {
-			return $this->objects[$path];
+		$this->id = 'amazon::' . $params['key'] . md5($params['secret']);
+		$this->bucket = $params['bucket'];
+		$scheme = ($params['use_ssl'] === 'false') ? 'http' : 'https';
+
+		if (isset($params['hostname']) && isset($params['port'])) {
+			$base_url = $scheme.'://'.$params['hostname'].':'.$params['port'].'/';
+			$this->connection = S3Client::factory(array(
+				'key' => $params['key'],
+				'secret' => $params['secret'],
+				'base_url' => $base_url
+			));
 		} else {
-			$response = $this->s3->get_object_metadata($this->bucket, $path);
-			if ($response) {
-				$this->objects[$path] = $response;
-				return $response;
-				// This object could be a folder, a '/' must be at the end of the path
-			} else if (substr($path, -1) != '/') {
-				$response = $this->s3->get_object_metadata($this->bucket, $path . '/');
-				if ($response) {
-					$this->objects[$path] = $response;
-					return $response;
-				}
-			}
+			$this->connection = S3Client::factory(array(
+				'key' => $params['key'],
+				'secret' => $params['secret'],
+				'scheme' => $scheme,
+				'region' => $region
+			));
 		}
-		return false;
-	}
 
-	public function getId() {
-		return $this->id;
+		if ( ! $this->connection->doesBucketExist($this->bucket)) {
+			$result = $this->connection->createBucket(array(
+				'Bucket' => $this->bucket
+			));
+		}
+
+		if ( ! $this->file_exists('.')) {
+			$result = $this->connection->putObject(array(
+				'Bucket' => $this->bucket,
+				'Key'    => '.',
+				'Body'   => '',
+				'ContentType' => 'httpd/unix-directory',
+				'ContentLength' => 0
+			));
+		}
 	}
 
 	public function mkdir($path) {
-		// Folders in Amazon S3 are 0 byte objects with a '/' at the end of the name
-		if (substr($path, -1) != '/') {
+		$path = $this->normalizePath($path);
+
+		if($this->is_dir($path)) {
+			return false;
+		}
+
+		try {
+			$result = $this->connection->putObject(array(
+				'Bucket' => $this->bucket,
+				'Key'    => $path . '/',
+				'Body'   => '',
+				'ContentType' => 'httpd/unix-directory',
+				'ContentLength' => 0
+			));
+		} catch (S3Exception $e) {
+			return false;
+		}
+
+		return true;
+	}
+
+	public function file_exists($path) {
+		$path = $this->normalizePath($path);
+
+		if ( ! $path) {
+			$path = '.';
+		} else if ($path != '.' && $this->is_dir($path)) {
 			$path .= '/';
 		}
-		$response = $this->s3->create_object($this->bucket, $path, array('body' => ''));
-		return $response->isOK();
+
+		try {
+			$result = $this->connection->doesObjectExist(
+				$this->bucket,
+				$path
+			);
+		} catch (S3Exception $e) {
+			return false;
+		}
+
+		return $result;
 	}
 
+
 	public function rmdir($path) {
-		if (substr($path, -1) != '/') {
-			$path .= '/';
+		$path = $this->normalizePath($path);
+
+		if ( ! $this->file_exists($path)) {
+			return false;
+		}
+
+		$dh = $this->opendir($path);
+		while ($file = readdir($dh)) {
+			if ($file == '.' || $file != '..') {
+				continue;
+			}
+
+			if ($this->is_dir(stripcslashes($file))) {
+				$this->rmdir(stripcslashes($file));
+			} else {
+				$this->unlink(stripcslashes($file));
+			}
+               	}
+
+		try {
+			$result = $this->connection->deleteObject(array(
+				'Bucket' => $this->bucket,
+				'Key' => $path . '/'
+			));
+		} catch (S3Exception $e) {
+			return false;
 		}
-		return $this->unlink($path);
+
+		return true;
 	}
 
 	public function opendir($path) {
-		if ($path == '' || $path == '/') {
-			// Use the '/' delimiter to only fetch objects inside the folder
-			$opt = array('delimiter' => '/');
-		} else {
-			if (substr($path, -1) != '/') {
-				$path .= '/';
-			}
-			$opt = array('delimiter' => '/', 'prefix' => $path);
+		$path = $this->normalizePath($path);
+
+		if ($path == '.') {
+			$path = '';
+		} else if ($path) {
+			$path .= '/';
 		}
-		$response = $this->s3->list_objects($this->bucket, $opt);
-		if ($response->isOK()) {
+
+		try {
 			$files = array();
-			foreach ($response->body->Contents as $object) {
-				// The folder being opened also shows up in the list of objects, don't add it to the files
-				if ($object->Key != $path) {
-					$files[] = basename($object->Key);
+			$result = $this->connection->getIterator('ListObjects', array(
+				'Bucket' => $this->bucket,
+				'Delimiter' => '/',
+				'Prefix' => $path
+			), array('return_prefixes' => true));
+
+			foreach ($result as $object) {
+				$file = basename(
+					isset($object['Key']) ? $object['Key'] : $object['Prefix']
+				);
+
+				if ( $file != basename($path)) {
+					$files[] = $file;
 				}
 			}
-			// Sub folders show up as CommonPrefixes
-			foreach ($response->body->CommonPrefixes as $object) {
-				$files[] = basename($object->Prefix);
-			}
+
 			\OC\Files\Stream\Dir::register('amazons3' . $path, $files);
+
 			return opendir('fakedir://amazons3' . $path);
+		} catch (S3Exception $e) {
+			return false;
 		}
-		return false;
 	}
 
 	public function stat($path) {
-		if ($path == '' || $path == '/') {
-			$stat['size'] = $this->s3->get_bucket_filesize($this->bucket);
-			$stat['atime'] = time();
-			$stat['mtime'] = $stat['atime'];
-		} else if ($object = $this->getObject($path)) {
-			$stat['size'] = $object['Size'];
+		$path = $this->normalizePath($path);
+
+		try {
+			if ($this->is_dir($path) && $path != '.') {
+				$path .= '/';
+			}
+
+			$result = $this->connection->headObject(array(
+				'Bucket' => $this->bucket,
+				'Key' => $path
+			));
+
+			$stat = array();
+			$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
+			if ($result['Metadata']['lastmodified']) {
+				$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
+			} else {
+				$stat['mtime'] = strtotime($result['LastModified']);
+			}
 			$stat['atime'] = time();
-			$stat['mtime'] = strtotime($object['LastModified']);
-		}
-		if (isset($stat)) {
+
 			return $stat;
+		} catch(S3Exception $e) {
+			return false;
 		}
-		return false;
 	}
 
 	public function filetype($path) {
-		if ($path == '' || $path == '/') {
-			return 'dir';
-		} else {
-			$object = $this->getObject($path);
-			if ($object) {
-				// Amazon S3 doesn't have typical folders, this is an alternative method to detect a folder
-				if (substr($object['Key'], -1) == '/' && $object['Size'] == 0) {
-					return 'dir';
-				} else {
-					return 'file';
-				}
+		$path = $this->normalizePath($path);
+
+		try {
+			if ($path != '.' && $this->connection->doesObjectExist($this->bucket, $path)) {
+				return 'file';
+			}
+
+			if ($path != '.') {
+				$path .= '/';
 			}
+
+			if ($this->connection->doesObjectExist($this->bucket, $path)) {
+				return 'dir';
+			}
+		} catch (S3Exception $e) {
+			return false;
 		}
+
 		return false;
 	}
 
 	public function isReadable($path) {
-		// TODO Check acl and determine who grantee is
 		return true;
 	}
 
 	public function isUpdatable($path) {
-		// TODO Check acl and determine who grantee is
 		return true;
 	}
 
-	public function file_exists($path) {
-		if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') {
-			$path .= '/';
+	public function unlink($path) {
+		$path = $this->normalizePath($path);
+
+		if ( ! $this->file_exists($path)) {
+			return false;
 		}
-		return $this->s3->if_object_exists($this->bucket, $path);
-	}
 
-	public function unlink($path) {
-		$response = $this->s3->delete_object($this->bucket, $path);
-		return $response->isOK();
+		try {
+			$result = $this->connection->deleteObject(array(
+				'Bucket' => $this->bucket,
+				'Key' => $path
+			));
+		} catch (S3Exception $e) {
+			return false;
+		}
+
+		$this->touch(dirname($path));
+		return true;
 	}
 
 	public function fopen($path, $mode) {
+		$path = $this->normalizePath($path);
+
 		switch ($mode) {
 			case 'r':
 			case 'rb':
 				$tmpFile = \OC_Helper::tmpFile();
-				$handle = fopen($tmpFile, 'w');
-				$response = $this->s3->get_object($this->bucket, $path, array('fileDownload' => $handle));
-				if ($response->isOK()) {
-					return fopen($tmpFile, 'r');
+				self::$tmpFiles[$tmpFile] = $path;
+
+				try {
+					$result = $this->connection->getObject(array(
+						'Bucket' => $this->bucket,
+						'Key' => $path,
+						'SaveAs' => $tmpFile
+					));
+				} catch (S3Exception $e) {
+					return false;
 				}
-				break;
+
+				return fopen($tmpFile, 'r');
 			case 'w':
 			case 'wb':
 			case 'a':
@@ -203,45 +327,139 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 					$source = $this->fopen($path, 'r');
 					file_put_contents($tmpFile, $source);
 				}
-				self::$tempFiles[$tmpFile] = $path;
+				self::$tmpFiles[$tmpFile] = $path;
+
 				return fopen('close://' . $tmpFile, $mode);
 		}
 		return false;
 	}
 
-	public function writeBack($tmpFile) {
-		if (isset(self::$tempFiles[$tmpFile])) {
-			$handle = fopen($tmpFile, 'r');
-			$response = $this->s3->create_object($this->bucket,
-				self::$tempFiles[$tmpFile],
-				array('fileUpload' => $handle));
-			if ($response->isOK()) {
-				unlink($tmpFile);
-			}
-		}
-	}
-
 	public function getMimeType($path) {
-		if ($this->filetype($path) == 'dir') {
+		$path = $this->normalizePath($path);
+
+		if ($this->is_dir($path)) {
 			return 'httpd/unix-directory';
-		} else {
-			$object = $this->getObject($path);
-			if ($object) {
-				return $object['ContentType'];
+		} else if ($this->file_exists($path)) {
+			try {
+				$result = $this->connection->headObject(array(
+					'Bucket' => $this->bucket,
+					'Key' => $path
+				));
+			} catch (S3Exception $e) {
+				return false;
 			}
+
+			return $result['ContentType'];
 		}
 		return false;
 	}
 
 	public function touch($path, $mtime = null) {
-		if (is_null($mtime)) {
-			$mtime = time();
+		$path = $this->normalizePath($path);
+
+		$metadata = array();
+		if ( ! is_null($mtime)) {
+			$metadata = array('lastmodified' => $mtime);
 		}
-		if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') {
-			$path .= '/';
+
+		try {
+			if ($this->file_exists($path)) {
+				if ($this->is_dir($path) && $path != '.') {
+					$path .= '/';
+				}
+				$result = $this->connection->copyObject(array(
+					'Bucket' => $this->bucket,
+					'Key' => $path,
+					'Metadata' => $metadata,
+					'CopySource' => $this->bucket . '/' . $path
+				));
+			} else {
+				$result = $this->connection->putObject(array(
+					'Bucket' => $this->bucket,
+					'Key' => $path,
+					'Metadata' => $metadata
+				));
+			}
+		} catch (S3Exception $e) {
+			return false;
+		}
+
+		return true;
+	}
+
+	public function copy($path1, $path2) {
+		$path1 = $this->normalizePath($path1);
+		$path2 = $this->normalizePath($path2);
+
+		if ($this->is_file($path1)) {
+			try {
+				$result = $this->connection->copyObject(array(
+					'Bucket' => $this->bucket,
+					'Key' => $path2,
+					'CopySource' => $this->bucket . '/' . $path1
+				));
+			} catch (S3Exception $e) {
+				return false;
+			}
+		} else {
+			if ($this->file_exists($path2)) {
+				return false;
+			}
+
+			try {
+				$result = $this->connection->copyObject(array(
+					'Bucket' => $this->bucket,
+					'Key' => $path2 . '/',
+					'CopySource' => $this->bucket . '/' . $path1 . '/'
+				));
+			} catch (S3Exception $e) {
+				return false;
+			}
+
+			$dh = $this->opendir($path1);
+			while ($file = readdir($dh)) {
+				if ($file == '.' || $file == '..') {
+					continue;
+				}
+
+				$source = $path1 . '/' . $file;
+				$target = $path2 . '/' . $file;
+				$this->copy($source, $target);
+                	}
+		}
+
+		return true;
+	}
+
+	public function rename($path1, $path2) {
+		$path1 = $this->normalizePath($path1);
+		$path2 = $this->normalizePath($path2);
+
+		if ($this->is_file($path1)) {
+			if ($this->copy($path1, $path2) == false) {
+				return false;
+			}
+
+			if ($this->unlink($path1) == false) {
+				$this->unlink($path2);
+				return false;
+			}
+		} else {
+			if ($this->file_exists($path2)) {
+				return false;
+			}
+
+			if ($this->copy($path1, $path2) == false) {
+				return false;
+			}
+
+			if($this->rmdir($path1) == false) {
+				$this->rmdir($path2);
+				return false;
+			}
 		}
-		$response = $this->s3->update_object($this->bucket, $path, array('meta' => array('LastModified' => $mtime)));
-		return $response->isOK();
+
+		return true;
 	}
 
 	public function test() {
@@ -252,4 +470,32 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return false;
 	}
 
+	public function getId() {
+		return $this->id;
+	}
+
+	public function getConnection() {
+		return $this->connection;
+	}
+
+	public function writeBack($tmpFile) {
+		if ( ! isset(self::$tmpFiles[$tmpFile])) {
+			return false;
+		}
+
+		try {
+			$result= $this->connection->putObject(array(
+				'Bucket' => $this->bucket,
+				'Key' => self::$tmpFiles[$tmpFile],
+				'SourceFile' => $tmpFile,
+				'ContentType' => \OC_Helper::getMimeType($tmpFile),
+				'ContentLength' => filesize($tmpFile)
+			));
+
+			unlink($tmpFile);
+			$this->touch(dirname(self::$tmpFiles[$tmpFile]));
+		} catch (S3Exception $e) {
+			return false;
+		}
+	}
 }