Add method to storage backends to get directory content with metadata

Currently you need to use `opendir` and then call `getMetadata` for
every file, which adds overhead because most storage backends already
get the metadata when doing the `opendir`.

While storagebackends can (and do) use caching to relief this problem,
this adds cache invalidation dificulties and only a limited number of
items are generally cached (to prevent memory usage exploding when
scanning large storages)

With this new methods storage backends can use the child metadata they
got from listing the folder to return metadata without having to keep
seperate caches.

Signed-off-by: Robin Appelman <robin@icewind.nl>

11 files changed, 118 insertions, 59 deletions

diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php
index 2a5a6536697..f895948574b 100644
--- a/lib/private/Files/Cache/Scanner.php
+++ b/lib/private/Files/Cache/Scanner.php
@@ -126,11 +126,11 @@ class Scanner extends BasicEmitter implements IScanner {
 	 * @param int $parentId
 	 * @param array|null|false $cacheData existing data in the cache for the file to be scanned
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
+	 * @param null $data the metadata for the file, as returned by the storage
 	 * @return array an array of metadata of the scanned file
-	 * @throws \OC\ServerNotAvailableException
 	 * @throws \OCP\Lock\LockedException
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		if ($file !== '') {
 			try {
 				$this->storage->verifyPath(dirname($file), basename($file));
@@ -149,7 +149,7 @@ class Scanner extends BasicEmitter implements IScanner {
 			}
 
 			try {
-				$data = $this->getData($file);
+				$data = $data ?? $this->getData($file);
 			} catch (ForbiddenException $e) {
 				if ($lock) {
 					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -367,26 +367,6 @@ class Scanner extends BasicEmitter implements IScanner {
 	}
 
 	/**
-	 * Get the children from the storage
-	 *
-	 * @param string $folder
-	 * @return string[]
-	 */
-	protected function getNewChildren($folder) {
-		$children = [];
-		if ($dh = $this->storage->opendir($folder)) {
-			if (is_resource($dh)) {
-				while (($file = readdir($dh)) !== false) {
-					if (!Filesystem::isIgnoredDir($file)) {
-						$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
-					}
-				}
-			}
-		}
-		return $children;
-	}
-
-	/**
 	 * scan all the files and folders in a folder
 	 *
 	 * @param string $path
@@ -425,7 +405,7 @@ class Scanner extends BasicEmitter implements IScanner {
 	private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
 		// we put this in it's own function so it cleans up the memory before we start recursing
 		$existingChildren = $this->getExistingChildren($folderId);
-		$newChildren = $this->getNewChildren($path);
+		$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
 
 		if ($this->useTransactions) {
 			\OC::$server->getDatabaseConnection()->beginTransaction();
@@ -433,11 +413,14 @@ class Scanner extends BasicEmitter implements IScanner {
 
 		$exceptionOccurred = false;
 		$childQueue = [];
-		foreach ($newChildren as $file) {
+		$newChildNames = [];
+		foreach ($newChildren as $fileMeta) {
+			$file = $fileMeta['name'];
+			$newChildNames[] = $file;
 			$child = $path ? $path . '/' . $file : $file;
 			try {
 				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
-				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
+				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
 				if ($data) {
 					if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
 						$childQueue[$child] = $data['fileid'];
@@ -471,7 +454,7 @@ class Scanner extends BasicEmitter implements IScanner {
 				throw $e;
 			}
 		}
-		$removedChildren = \array_diff(array_keys($existingChildren), $newChildren);
+		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
 		foreach ($removedChildren as $childName) {
 			$child = $path ? $path . '/' . $childName : $childName;
 			$this->removeFromCache($child);
diff --git a/lib/private/Files/ObjectStore/NoopScanner.php b/lib/private/Files/ObjectStore/NoopScanner.php
index 57a94aba294..25b52416efd 100644
--- a/lib/private/Files/ObjectStore/NoopScanner.php
+++ b/lib/private/Files/ObjectStore/NoopScanner.php
@@ -44,7 +44,7 @@ class NoopScanner extends Scanner {
 	 * @param array|null $cacheData existing data in the cache for the file to be scanned
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		return [];
 	}
 
diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php
index bb672f3a0ea..86252f5c3dd 100644
--- a/lib/private/Files/Storage/Common.php
+++ b/lib/private/Files/Storage/Common.php
@@ -234,7 +234,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		} else {
 			$source = $this->fopen($path1, 'r');
 			$target = $this->fopen($path2, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if (!$result) {
 				\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
 			}
@@ -626,7 +626,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 			// are not the same as the original one.Once this is fixed we also
 			// need to adjust the encryption wrapper.
 			$target = $this->fopen($targetInternalPath, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if ($result and $preserveMtime) {
 				$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
 			}
@@ -719,6 +719,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		$data['etag'] = $this->getETag($path);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}
@@ -867,4 +868,17 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		}
 		return $count;
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		$dh = $this->opendir($directory);
+		if (is_resource($dh)) {
+			$basePath = rtrim($directory, '/');
+			while (($file = readdir($dh)) !== false) {
+				if (!Filesystem::isIgnoredDir($file)) {
+					$childPath = $basePath . '/' . trim($file, '/');
+					yield $this->getMetaData($childPath);
+				}
+			}
+		}
+	}
 }
diff --git a/lib/private/Files/Storage/Local.php b/lib/private/Files/Storage/Local.php
index 89b318b4770..2019c67bede 100644
--- a/lib/private/Files/Storage/Local.php
+++ b/lib/private/Files/Storage/Local.php
@@ -196,6 +196,7 @@ class Local extends \OC\Files\Storage\Common {
 		$data['etag'] = $this->calculateEtag($path, $stat);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}
diff --git a/lib/private/Files/Storage/Storage.php b/lib/private/Files/Storage/Storage.php
index 7e5a0cef4d7..76dab07b6a7 100644
--- a/lib/private/Files/Storage/Storage.php
+++ b/lib/private/Files/Storage/Storage.php
@@ -119,4 +119,22 @@ interface Storage extends \OCP\Files\Storage {
 	 * @throws \OCP\Lock\LockedException
 	 */
 	public function changeLock($path, $type, ILockingProvider $provider);
+
+	/**
+	 * Get the contents of a directory with metadata
+	 *
+	 * @param string $directory
+	 * @return \Traversable an iterator, containing file metadata
+	 *
+	 * The metadata array will contain the following fields
+	 *
+	 * - name
+	 * - mimetype
+	 * - mtime
+	 * - size
+	 * - etag
+	 * - storage_mtime
+	 * - permissions
+	 */
+	public function getDirectoryContent($directory): \Traversable;
 }
diff --git a/lib/private/Files/Storage/Wrapper/Availability.php b/lib/private/Files/Storage/Wrapper/Availability.php
index 300ba7e7f7c..ea210128e8f 100644
--- a/lib/private/Files/Storage/Wrapper/Availability.php
+++ b/lib/private/Files/Storage/Wrapper/Availability.php
@@ -461,4 +461,15 @@ class Availability extends Wrapper {
 		$this->getStorageCache()->setAvailability(false, $delay);
 		throw $e;
 	}
+
+
+
+	public function getDirectoryContent($directory): \Traversable {
+		$this->checkAvailability();
+		try {
+			return parent::getDirectoryContent($directory);
+		} catch (StorageNotAvailableException $e) {
+			$this->setUnavailable($e);
+		}
+	}
 }
diff --git a/lib/private/Files/Storage/Wrapper/Encoding.php b/lib/private/Files/Storage/Wrapper/Encoding.php
index d1e0622808b..4e81588de80 100644
--- a/lib/private/Files/Storage/Wrapper/Encoding.php
+++ b/lib/private/Files/Storage/Wrapper/Encoding.php
@@ -534,4 +534,8 @@ class Encoding extends Wrapper {
 	public function getMetaData($path) {
 		return $this->storage->getMetaData($this->findPathToUse($path));
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->storage->getDirectoryContent($this->findPathToUse($directory));
+	}
 }
diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php
index 1ea2664877b..897624ff6ae 100644
--- a/lib/private/Files/Storage/Wrapper/Encryption.php
+++ b/lib/private/Files/Storage/Wrapper/Encryption.php
@@ -105,17 +105,17 @@ class Encryption extends Wrapper {
 	 * @param ArrayCache $arrayCache
 	 */
 	public function __construct(
-			$parameters,
-			IManager $encryptionManager = null,
-			Util $util = null,
-			ILogger $logger = null,
-			IFile $fileHelper = null,
-			$uid = null,
-			IStorage $keyStorage = null,
-			Update $update = null,
-			Manager $mountManager = null,
-			ArrayCache $arrayCache = null
-		) {
+		$parameters,
+		IManager $encryptionManager = null,
+		Util $util = null,
+		ILogger $logger = null,
+		IFile $fileHelper = null,
+		$uid = null,
+		IStorage $keyStorage = null,
+		Update $update = null,
+		Manager $mountManager = null,
+		ArrayCache $arrayCache = null
+	) {
 		$this->mountPoint = $parameters['mountPoint'];
 		$this->mount = $parameters['mount'];
 		$this->encryptionManager = $encryptionManager;
@@ -169,15 +169,7 @@ class Encryption extends Wrapper {
 		return $this->storage->filesize($path);
 	}
 
-	/**
-	 * @param string $path
-	 * @return array
-	 */
-	public function getMetaData($path) {
-		$data = $this->storage->getMetaData($path);
-		if (is_null($data)) {
-			return null;
-		}
+	private function modifyMetaData(string $path, array $data): array {
 		$fullPath = $this->getFullPath($path);
 		$info = $this->getCache()->get($path);
 
@@ -199,6 +191,25 @@ class Encryption extends Wrapper {
 	}
 
 	/**
+	 * @param string $path
+	 * @return array
+	 */
+	public function getMetaData($path) {
+		$data = $this->storage->getMetaData($path);
+		if (is_null($data)) {
+			return null;
+		}
+		return $this->modifyMetaData($path, $data);
+	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		$parent = rtrim($directory, '/');
+		foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
+			yield $this->modifyMetaData($parent . '/' . $data['name'], $data);
+		}
+	}
+
+	/**
 	 * see http://php.net/manual/en/function.file_get_contents.php
 	 *
 	 * @param string $path
@@ -493,7 +504,7 @@ class Encryption extends Wrapper {
 				try {
 					$result = $this->fixUnencryptedSize($path, $size, $unencryptedSize);
 				} catch (\Exception $e) {
-					$this->logger->error('Couldn\'t re-calculate unencrypted size for '. $path);
+					$this->logger->error('Couldn\'t re-calculate unencrypted size for ' . $path);
 					$this->logger->logException($e);
 				}
 				unset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]);
@@ -546,7 +557,7 @@ class Encryption extends Wrapper {
 		// next highest is end of chunks, one subtracted is last one
 		// we have to read the last chunk, we can't just calculate it (because of padding etc)
 
-		$lastChunkNr = ceil($size/ $blockSize)-1;
+		$lastChunkNr = ceil($size / $blockSize) - 1;
 		// calculate last chunk position
 		$lastChunkPos = ($lastChunkNr * $blockSize);
 		// try to fseek to the last chunk, if it fails we have to read the whole file
@@ -554,16 +565,16 @@ class Encryption extends Wrapper {
 			$newUnencryptedSize += $lastChunkNr * $unencryptedBlockSize;
 		}
 
-		$lastChunkContentEncrypted='';
+		$lastChunkContentEncrypted = '';
 		$count = $blockSize;
 
 		while ($count > 0) {
-			$data=fread($stream, $blockSize);
-			$count=strlen($data);
+			$data = fread($stream, $blockSize);
+			$count = strlen($data);
 			$lastChunkContentEncrypted .= $data;
 			if (strlen($lastChunkContentEncrypted) > $blockSize) {
 				$newUnencryptedSize += $unencryptedBlockSize;
-				$lastChunkContentEncrypted=substr($lastChunkContentEncrypted, $blockSize);
+				$lastChunkContentEncrypted = substr($lastChunkContentEncrypted, $blockSize);
 			}
 		}
 
@@ -743,7 +754,7 @@ class Encryption extends Wrapper {
 			try {
 				$source = $sourceStorage->fopen($sourceInternalPath, 'r');
 				$target = $this->fopen($targetInternalPath, 'w');
-				list(, $result) = \OC_Helper::streamCopy($source, $target);
+				[, $result] = \OC_Helper::streamCopy($source, $target);
 				fclose($source);
 				fclose($target);
 			} catch (\Exception $e) {
@@ -889,7 +900,7 @@ class Encryption extends Wrapper {
 				$header = substr($header, 0, $endAt + strlen(Util::HEADER_END));
 
 				// +1 to not start with an ':' which would result in empty element at the beginning
-				$exploded = explode(':', substr($header, strlen(Util::HEADER_START)+1));
+				$exploded = explode(':', substr($header, strlen(Util::HEADER_START) + 1));
 
 				$element = array_shift($exploded);
 				while ($element !== Util::HEADER_END) {
@@ -1023,7 +1034,7 @@ class Encryption extends Wrapper {
 	public function writeStream(string $path, $stream, int $size = null): int {
 		// always fall back to fopen
 		$target = $this->fopen($path, 'w');
-		list($count, $result) = \OC_Helper::streamCopy($stream, $target);
+		[$count, $result] = \OC_Helper::streamCopy($stream, $target);
 		fclose($target);
 		return $count;
 	}
diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php
index 93380baac5e..929a4942562 100644
--- a/lib/private/Files/Storage/Wrapper/Jail.php
+++ b/lib/private/Files/Storage/Wrapper/Jail.php
@@ -539,4 +539,8 @@ class Jail extends Wrapper {
 			return $count;
 		}
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->getWrapperStorage()->getDirectoryContent($this->getJailedPath($directory));
+	}
 }
diff --git a/lib/private/Files/Storage/Wrapper/PermissionsMask.php b/lib/private/Files/Storage/Wrapper/PermissionsMask.php
index c2cea31bd5a..6b453b4380a 100644
--- a/lib/private/Files/Storage/Wrapper/PermissionsMask.php
+++ b/lib/private/Files/Storage/Wrapper/PermissionsMask.php
@@ -157,4 +157,13 @@ class PermissionsMask extends Wrapper {
 		}
 		return parent::getScanner($path, $storage);
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
+			$data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions'];
+			$data['permissions'] &= $this->mask;
+
+			yield $data;
+		}
+	}
 }
diff --git a/lib/private/Files/Storage/Wrapper/Wrapper.php b/lib/private/Files/Storage/Wrapper/Wrapper.php
index d67af5cb22e..4584bebe076 100644
--- a/lib/private/Files/Storage/Wrapper/Wrapper.php
+++ b/lib/private/Files/Storage/Wrapper/Wrapper.php
@@ -637,4 +637,8 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea
 			return $count;
 		}
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->getWrapperStorage()->getDirectoryContent($directory);
+	}
 }