Merge pull request #8822 from owncloud/cache-change-propagator

[WIP] Improved propagation of cache changes

2 files changed, 147 insertions, 46 deletions

diff --git a/lib/private/files/cache/changepropagator.php b/lib/private/files/cache/changepropagator.php
new file mode 100644
index 00000000000..30f2e675e2e
--- /dev/null
+++ b/lib/private/files/cache/changepropagator.php
@@ -0,0 +1,98 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\Files\Cache;
+
+/**
+ * Propagates changes in etag and mtime up the filesystem tree
+ *
+ * @package OC\Files\Cache
+ */
+class ChangePropagator {
+	/**
+	 * @var string[]
+	 */
+	protected $changedFiles = array();
+
+	/**
+	 * @var \OC\Files\View
+	 */
+	protected $view;
+
+	/**
+	 * @param \OC\Files\View $view
+	 */
+	public function __construct(\OC\Files\View $view) {
+		$this->view = $view;
+	}
+
+	public function addChange($path) {
+		$this->changedFiles[] = $path;
+	}
+
+	public function getChanges() {
+		return $this->changedFiles;
+	}
+
+	/**
+	 * propagate the registered changes to their parent folders
+	 *
+	 * @param int $time (optional) the mtime to set for the folders, if not set the current time is used
+	 */
+	public function propagateChanges($time = null) {
+		$parents = $this->getAllParents();
+		$this->changedFiles = array();
+		if (!$time) {
+			$time = time();
+		}
+		foreach ($parents as $parent) {
+			/**
+			 * @var \OC\Files\Storage\Storage $storage
+			 * @var string $internalPath
+			 */
+
+			list($storage, $internalPath) = $this->view->resolvePath($parent);
+			$cache = $storage->getCache();
+			$id = $cache->getId($internalPath);
+			$cache->update($id, array('mtime' => $time, 'etag' => $storage->getETag($internalPath)));
+		}
+	}
+
+	/**
+	 * @return string[]
+	 */
+	public function getAllParents() {
+		$parents = array();
+		foreach ($this->getChanges() as $path) {
+			$parents = array_values(array_unique(array_merge($parents, $this->getParents($path))));
+		}
+		return $parents;
+	}
+
+	/**
+	 * get all parent folders of $path
+	 *
+	 * @param string $path
+	 * @return string[]
+	 */
+	protected function getParents($path) {
+		$parts = explode('/', $path);
+
+		// remove the singe file
+		array_pop($parts);
+		$result = array('/');
+		$resultPath = '';
+		foreach ($parts as $part) {
+			if ($part) {
+				$resultPath .= '/' . $part;
+				$result[] = $resultPath;
+			}
+		}
+		return $result;
+	}
+}
diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php
index b97070fcdf0..6cb9807f5a7 100644
--- a/lib/private/files/cache/scanner.php
+++ b/lib/private/files/cache/scanner.php
@@ -124,10 +124,8 @@ class Scanner extends BasicEmitter {
 						// prevent empty etag
 						if (empty($cacheData['etag'])) {
 							$etag = $data['etag'];
-							$propagateETagChange = true;
 						} else {
 							$etag = $cacheData['etag'];
-							$propagateETagChange = false;
 						}
 						// only reuse data if the file hasn't explicitly changed
 						if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
@@ -136,22 +134,6 @@ class Scanner extends BasicEmitter {
 							}
 							if ($reuseExisting & self::REUSE_ETAG) {
 								$data['etag'] = $etag;
-								if ($propagateETagChange) {
-									$parent = $file;
-									while ($parent !== '') {
-										$parent = dirname($parent);
-										if ($parent === '.') {
-											$parent = '';
-										}
-										$parentCacheData = $this->cache->get($parent);
-										\OC_Hook::emit('Scanner', 'updateCache', array('file' => $file, 'data' => $data));
-										if($this->cacheActive) {
-											$this->cache->update($parentCacheData['fileid'], array(
-												'etag' => $this->storage->getETag($parent),
-											));
-										}
-									}
-								}
 							}
 						}
 						// Only update metadata that has changed
@@ -166,24 +148,53 @@ class Scanner extends BasicEmitter {
 					}
 				}
 				if (!empty($newData)) {
-					\OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData));
-					if($this->cacheActive) {
-						$data['fileid'] = $this->cache->put($file, $newData);
-					}
+					$data['fileid'] = $this->addToCache($file, $newData);
 					$this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId));
 					\OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId));
 				}
 			} else {
-				\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file));
-				if($this->cacheActive) {
-					$this->cache->remove($file);
-				}
+				$this->removeFromCache($file);
 			}
 			return $data;
 		}
 		return null;
 	}
 
+	protected function removeFromCache($path) {
+		\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path));
+		$this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path));
+		if ($this->cacheActive) {
+			$this->cache->remove($path);
+		}
+	}
+
+	/**
+	 * @param string $path
+	 * @param array $data
+	 * @return int the id of the added file
+	 */
+	protected function addToCache($path, $data) {
+		\OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data));
+		$this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data));
+		if ($this->cacheActive) {
+			return $this->cache->put($path, $data);
+		} else {
+			return -1;
+		}
+	}
+
+	/**
+	 * @param string $path
+	 * @param array $data
+	 */
+	protected function updateCache($path, $data) {
+		\OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data));
+		$this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data));
+		if ($this->cacheActive) {
+			$this->cache->put($path, $data);
+		}
+	}
+
 	/**
 	 * scan a folder and all it's children
 	 *
@@ -236,18 +247,15 @@ class Scanner extends BasicEmitter {
 						try {
 							$data = $this->scanFile($child, $reuse, true);
 							if ($data) {
-								if ($data['size'] === -1) {
-									if ($recursive === self::SCAN_RECURSIVE) {
-										$childQueue[] = $child;
-									} else {
-										$size = -1;
-									}
+								if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
+									$childQueue[] = $child;
+								} else if ($data['size'] === -1) {
+									$size = -1;
 								} else if ($size !== -1) {
 									$size += $data['size'];
 								}
 							}
-						}
-						catch (\Doctrine\DBAL\DBALException $ex){
+						} catch (\Doctrine\DBAL\DBALException $ex) {
 							// might happen if inserting duplicate while a scanning
 							// process is running in parallel
 							// log and ignore
@@ -260,13 +268,10 @@ class Scanner extends BasicEmitter {
 			$removedChildren = \array_diff($existingChildren, $newChildren);
 			foreach ($removedChildren as $childName) {
 				$child = ($path) ? $path . '/' . $childName : $childName;
-				\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child));
-				if($this->cacheActive) {
-					$this->cache->remove($child);
-				}
+				$this->removeFromCache($child);
 			}
 			\OC_DB::commit();
-			if ($exceptionOccurred){
+			if ($exceptionOccurred) {
 				// It might happen that the parallel scan process has already
 				// inserted mimetypes but those weren't available yet inside the transaction
 				// To make sure to have the updated mime types in such cases,
@@ -278,15 +283,11 @@ class Scanner extends BasicEmitter {
 				$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
 				if ($childSize === -1) {
 					$size = -1;
-				} else {
+				} else if ($size !== -1) {
 					$size += $childSize;
 				}
 			}
-			$newData = array('size' => $size);
-			\OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'data' => $newData));
-			if($this->cacheActive) {
-				$this->cache->put($path, $newData);
-			}
+			$this->updateCache($path, array('size' => $size));
 		}
 		$this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId));
 		return $size;
@@ -296,6 +297,7 @@ class Scanner extends BasicEmitter {
 	 * check if the file should be ignored when scanning
 	 * NOTE: files with a '.part' extension are ignored as well!
 	 *       prevents unfinished put requests to be scanned
+	 *
 	 * @param string $file
 	 * @return boolean
 	 */
@@ -314,7 +316,7 @@ class Scanner extends BasicEmitter {
 		while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
 			$this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG);
 			\OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path));
-			if($this->cacheActive) {
+			if ($this->cacheActive) {
 				$this->cache->correctFolderSize($path);
 			}
 			$lastPath = $path;
@@ -323,6 +325,7 @@ class Scanner extends BasicEmitter {
 
 	/**
 	 * Set whether the cache is affected by scan operations
+	 *
 	 * @param boolean $active The active state of the cache
 	 */
 	public function setCacheActive($active) {