Merge pull request #3764 from owncloud/scanfolder-existing

Improve re-using existing data in the filescanner

1 files changed, 48 insertions, 31 deletions

diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php
index 8f9a7921956..e7fbd856d5a 100644
--- a/lib/files/cache/scanner.php
+++ b/lib/files/cache/scanner.php
@@ -8,6 +8,8 @@
 
 namespace OC\Files\Cache;
 
+use OC\Files\Filesystem;
+
 class Scanner {
 	/**
 	 * @var \OC\Files\Storage\Storage $storage
@@ -27,6 +29,9 @@ class Scanner {
 	const SCAN_RECURSIVE = true;
 	const SCAN_SHALLOW = false;
 
+	const REUSE_ETAG = 1;
+	const REUSE_SIZE = 2;
+
 	public function __construct(\OC\Files\Storage\Storage $storage) {
 		$this->storage = $storage;
 		$this->storageId = $this->storage->getId();
@@ -59,12 +64,12 @@ class Scanner {
 	 * scan a single file and store it in the cache
 	 *
 	 * @param string $file
-	 * @param bool $checkExisting check existing folder sizes in the cache instead of always using -1 for folder size
+	 * @param int $reuseExisting
 	 * @return array with metadata of the scanned file
 	 */
-	public function scanFile($file, $checkExisting = false) {
-		if ( ! self::isPartialFile($file)
-			and ! \OC\Files\Filesystem::isFileBlacklisted($file)
+	public function scanFile($file, $reuseExisting = 0) {
+		if (!self::isPartialFile($file)
+			and !Filesystem::isFileBlacklisted($file)
 		) {
 			\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId));
 			$data = $this->getData($file);
@@ -79,20 +84,22 @@ class Scanner {
 					}
 				}
 				$newData = $data;
-				if ($cacheData = $this->cache->get($file)) {
-					if ($checkExisting && $data['size'] === -1) {
-						$data['size'] = $cacheData['size'];
-					}
-					if ($data['mtime'] === $cacheData['mtime'] &&
-						$data['size'] === $cacheData['size']) {
-						$data['etag'] = $cacheData['etag'];
+				if ($reuseExisting and $cacheData = $this->cache->get($file)) {
+					// only reuse data if the file hasn't explicitly changed
+					if ($data['mtime'] === $cacheData['mtime']) {
+						if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
+							$data['size'] = $cacheData['size'];
+						}
+						if ($reuseExisting & self::REUSE_ETAG) {
+							$data['etag'] = $cacheData['etag'];
+						}
 					}
 					// Only update metadata that has changed
 					$newData = array_diff($data, $cacheData);
 				}
-				if (!empty($newData)) {
-					$this->cache->put($file, $newData);
-				}
+			}
+			if (!empty($newData)) {
+				$this->cache->put($file, $newData);
 			}
 			return $data;
 		}
@@ -100,38 +107,50 @@ class Scanner {
 	}
 
 	/**
-	 * scan all the files in a folder and store them in the cache
+	 * scan a folder and all it's children
 	 *
 	 * @param string $path
 	 * @param bool $recursive
-	 * @param bool $onlyChilds
+	 * @param int $reuse
 	 * @return int the size of the scanned folder or -1 if the size is unknown at this stage
 	 */
-	public function scan($path, $recursive = self::SCAN_RECURSIVE, $onlyChilds = false) {
-		\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_folder', array('path' => $path, 'storage' => $this->storageId));
-		$childQueue = array();
-		if (!$onlyChilds) {
-			$this->scanFile($path);
+	public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {
+		if ($reuse === -1) {
+			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : 0;
 		}
+		$this->scanFile($path, $reuse);
+		return $this->scanChildren($path, $recursive, $reuse);
+	}
 
+	/**
+	 * scan all the files and folders in a folder
+	 *
+	 * @param string $path
+	 * @param bool $recursive
+	 * @param int $reuse
+	 * @return int the size of the scanned folder or -1 if the size is unknown at this stage
+	 */
+	public function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {
+		if ($reuse === -1) {
+			$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : 0;
+		}
+		\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_folder', array('path' => $path, 'storage' => $this->storageId));
 		$size = 0;
+		$childQueue = array();
 		if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
 			\OC_DB::beginTransaction();
 			while ($file = readdir($dh)) {
 				$child = ($path) ? $path . '/' . $file : $file;
-				if (!\OC\Files\Filesystem::isIgnoredDir($file)) {
-					$data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW);
+				if (!Filesystem::isIgnoredDir($file)) {
+					$data = $this->scanFile($child, $reuse);
 					if ($data) {
 						if ($data['size'] === -1) {
 							if ($recursive === self::SCAN_RECURSIVE) {
 								$childQueue[] = $child;
-								$data['size'] = 0;
 							} else {
 								$size = -1;
 							}
-						}
-
-						if ($size !== -1) {
+						} else if ($size !== -1) {
 							$size += $data['size'];
 						}
 					}
@@ -139,16 +158,14 @@ class Scanner {
 			}
 			\OC_DB::commit();
 			foreach ($childQueue as $child) {
-				$childSize = $this->scan($child, self::SCAN_RECURSIVE, true);
+				$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE);
 				if ($childSize === -1) {
 					$size = -1;
 				} else {
 					$size += $childSize;
 				}
 			}
-			if ($size !== -1) {
-				$this->cache->put($path, array('size' => $size));
-			}
+			$this->cache->put($path, array('size' => $size));
 		}
 		return $size;
 	}