Add method to storage backends to get directory content with metadata

Currently you need to use `opendir` and then call `getMetadata` for
every file, which adds overhead because most storage backends already
get the metadata when doing the `opendir`.

While storagebackends can (and do) use caching to relief this problem,
this adds cache invalidation dificulties and only a limited number of
items are generally cached (to prevent memory usage exploding when
scanning large storages)

With this new methods storage backends can use the child metadata they
got from listing the folder to return metadata without having to keep
seperate caches.

Signed-off-by: Robin Appelman <robin@icewind.nl>

3 files changed, 59 insertions, 10 deletions

diff --git a/apps/files_external/lib/Lib/Storage/SMB.php b/apps/files_external/lib/Lib/Storage/SMB.php
index d010fb54840..3ff8179c7b4 100644
--- a/apps/files_external/lib/Lib/Storage/SMB.php
+++ b/apps/files_external/lib/Lib/Storage/SMB.php
@@ -55,6 +55,7 @@ use OC\Cache\CappedMemoryCache;
 use OC\Files\Filesystem;
 use OC\Files\Storage\Common;
 use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
+use OCP\Constants;
 use OCP\Files\Notify\IChange;
 use OCP\Files\Notify\IRenameChange;
 use OCP\Files\Storage\INotifyStorage;
@@ -97,7 +98,7 @@ class SMB extends Common implements INotifyStorage {
 		if (isset($params['auth'])) {
 			$auth = $params['auth'];
 		} elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
-			list($workgroup, $user) = $this->splitUser($params['user']);
+			[$workgroup, $user] = $this->splitUser($params['user']);
 			$auth = new BasicAuth($user, $workgroup, $params['password']);
 		} else {
 			throw new \Exception('Invalid configuration, no credentials provided');
@@ -206,14 +207,15 @@ class SMB extends Common implements INotifyStorage {
 	 * @return \Icewind\SMB\IFileInfo[]
 	 * @throws StorageNotAvailableException
 	 */
-	protected function getFolderContents($path) {
+	protected function getFolderContents($path): iterable {
 		try {
 			$path = ltrim($this->buildPath($path), '/');
 			$files = $this->share->dir($path);
 			foreach ($files as $file) {
 				$this->statCache[$path . '/' . $file->getName()] = $file;
 			}
-			return array_filter($files, function (IFileInfo $file) {
+
+			foreach ($files as $file) {
 				try {
 					// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
 					// so we trigger the below exceptions where applicable
@@ -221,15 +223,15 @@ class SMB extends Common implements INotifyStorage {
 					if ($hide) {
 						$this->logger->debug('hiding hidden file ' . $file->getName());
 					}
-					return !$hide;
+					if (!$hide) {
+						yield $file;
+					}
 				} catch (ForbiddenException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
-					return false;
 				} catch (NotFoundException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
-					return false;
 				}
-			});
+			}
 		} catch (ConnectException $e) {
 			$this->logger->logException($e, ['message' => 'Error while getting folder content']);
 			throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -508,6 +510,46 @@ class SMB extends Common implements INotifyStorage {
 		}
 	}
 
+	public function getMetaData($path) {
+		$fileInfo = $this->getFileInfo($path);
+		if (!$fileInfo) {
+			return null;
+		}
+
+		return $this->getMetaDataFromFileInfo($fileInfo);
+	}
+
+	private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
+		$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
+
+		if (!$fileInfo->isReadOnly()) {
+			$permissions += Constants::PERMISSION_DELETE;
+			$permissions += Constants::PERMISSION_UPDATE;
+			if ($fileInfo->isDirectory()) {
+				$permissions += Constants::PERMISSION_CREATE;
+			}
+		}
+
+		$data = [];
+		if ($fileInfo->isDirectory()) {
+			$data['mimetype'] = 'httpd/unix-directory';
+		} else {
+			$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
+		}
+		$data['mtime'] = $fileInfo->getMTime();
+		if ($fileInfo->isDirectory()) {
+			$data['size'] = -1; //unknown
+		} else {
+			$data['size'] = $fileInfo->getSize();
+		}
+		$data['etag'] = $this->getETag($fileInfo->getPath());
+		$data['storage_mtime'] = $data['mtime'];
+		$data['permissions'] = $permissions;
+		$data['name'] = $fileInfo->getName();
+
+		return $data;
+	}
+
 	public function opendir($path) {
 		try {
 			$files = $this->getFolderContents($path);
@@ -519,10 +561,17 @@ class SMB extends Common implements INotifyStorage {
 		$names = array_map(function ($info) {
 			/** @var \Icewind\SMB\IFileInfo $info */
 			return $info->getName();
-		}, $files);
+		}, iterator_to_array($files));
 		return IteratorDirectory::wrap($names);
 	}
 
+	public function getDirectoryContent($directory): \Traversable {
+		$files = $this->getFolderContents($directory);
+		foreach ($files as $file) {
+			yield $this->getMetaDataFromFileInfo($file);
+		}
+	}
+
 	public function filetype($path) {
 		try {
 			return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';
diff --git a/apps/files_sharing/lib/External/Scanner.php b/apps/files_sharing/lib/External/Scanner.php
index 8962bb68c07..ebe0361c989 100644
--- a/apps/files_sharing/lib/External/Scanner.php
+++ b/apps/files_sharing/lib/External/Scanner.php
@@ -57,7 +57,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		try {
 			return parent::scanFile($file, $reuseExisting);
 		} catch (ForbiddenException $e) {
diff --git a/apps/files_sharing/lib/Scanner.php b/apps/files_sharing/lib/Scanner.php
index 7115b602aed..36a412800e5 100644
--- a/apps/files_sharing/lib/Scanner.php
+++ b/apps/files_sharing/lib/Scanner.php
@@ -71,7 +71,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
 		}
 	}
 
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		$sourceScanner = $this->getSourceScanner();
 		if ($sourceScanner instanceof NoopScanner) {
 			return [];