Merge pull request #20439 from owncloud/etag-propagate-in-storage

Take submount etag into account for folder etags

8 files changed, 242 insertions, 110 deletions

diff --git a/lib/private/files/cache/cache.php b/lib/private/files/cache/cache.php
index 40477243324..e6110c1925d 100644
--- a/lib/private/files/cache/cache.php
+++ b/lib/private/files/cache/cache.php
@@ -160,6 +160,7 @@ class Cache {
 		} else {
 			//fix types
 			$data['fileid'] = (int)$data['fileid'];
+			$data['parent'] = (int)$data['parent'];
 			$data['size'] = 0 + $data['size'];
 			$data['mtime'] = (int)$data['mtime'];
 			$data['storage_mtime'] = (int)$data['storage_mtime'];
@@ -391,12 +392,17 @@ class Cache {
 		if ($file === '') {
 			return -1;
 		} else {
-			$parent = dirname($file);
-			if ($parent === '.') {
-				$parent = '';
-			}
-			return $this->getId($parent);
+			$parent = $this->getParentPath($file);
+			return (int) $this->getId($parent);
+		}
+	}
+
+	private function getParentPath($path) {
+		$parent = dirname($path);
+		if ($parent === '.') {
+			$parent = '';
 		}
+		return $parent;
 	}
 
 	/**
diff --git a/lib/private/files/cache/changepropagator.php b/lib/private/files/cache/changepropagator.php
index 9696a82257e..2a48eb13c59 100644
--- a/lib/private/files/cache/changepropagator.php
+++ b/lib/private/files/cache/changepropagator.php
@@ -22,6 +22,7 @@
 
 namespace OC\Files\Cache;
 
+use OC\Files\Filesystem;
 use OC\Hooks\BasicEmitter;
 
 /**
@@ -61,23 +62,30 @@ class ChangePropagator extends BasicEmitter {
 	 * @param int $time (optional) the mtime to set for the folders, if not set the current time is used
 	 */
 	public function propagateChanges($time = null) {
-		$parents = $this->getAllParents();
-		$this->changedFiles = array();
+		$changes = $this->getChanges();
+		$this->changedFiles = [];
 		if (!$time) {
 			$time = time();
 		}
-		foreach ($parents as $parent) {
+		foreach ($changes as $change) {
 			/**
 			 * @var \OC\Files\Storage\Storage $storage
 			 * @var string $internalPath
 			 */
 
-			list($storage, $internalPath) = $this->view->resolvePath($parent);
+			$absolutePath = $this->view->getAbsolutePath($change);
+			$mount = $this->view->getMount($change);
+			$storage = $mount->getStorage();
+			$internalPath = $mount->getInternalPath($absolutePath);
 			if ($storage) {
-				$cache = $storage->getCache();
-				$entry = $cache->get($internalPath);
-				$cache->update($entry['fileid'], array('mtime' => max($time, $entry['mtime']), 'etag' => $storage->getETag($internalPath)));
-				$this->emit('\OC\Files', 'propagate', [$parent, $entry]);
+				$propagator = $storage->getPropagator();
+				$propagatedEntries = $propagator->propagateChange($internalPath, $time);
+
+				foreach ($propagatedEntries as $entry) {
+					$absolutePath = Filesystem::normalizePath($mount->getMountPoint() . '/' . $entry['path']);
+					$relativePath = $this->view->getRelativePath($absolutePath);
+					$this->emit('\OC\Files', 'propagate', [$relativePath, $entry]);
+				}
 			}
 		}
 	}
diff --git a/lib/private/files/cache/propagator.php b/lib/private/files/cache/propagator.php
new file mode 100644
index 00000000000..bd11cef5990
--- /dev/null
+++ b/lib/private/files/cache/propagator.php
@@ -0,0 +1,66 @@
+<?php
+/**
+ * @author Robin Appelman <icewind@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Files\Cache;
+
+/**
+ * Propagate etags and mtimes within the storage
+ */
+class Propagator {
+	/**
+	 * @var \OC\Files\Storage\Storage
+	 */
+	protected $storage;
+
+	/**
+	 * @param \OC\Files\Storage\Storage $storage
+	 */
+	public function __construct(\OC\Files\Storage\Storage $storage) {
+		$this->storage = $storage;
+	}
+
+
+	/**
+	 * @param string $internalPath
+	 * @param int $time
+	 * @return array[] all propagated entries
+	 */
+	public function propagateChange($internalPath, $time) {
+		$cache = $this->storage->getCache($internalPath);
+
+		$parentId = $cache->getParentId($internalPath);
+		$propagatedEntries = [];
+		while ($parentId !== -1) {
+			$entry = $cache->get($parentId);
+			$propagatedEntries[] = $entry;
+			if (!$entry) {
+				return $propagatedEntries;
+			}
+			$mtime = max($time, $entry['mtime']);
+
+			$cache->update($parentId, ['mtime' => $mtime, 'etag' => $this->storage->getETag($entry['path'])]);
+
+			$parentId = $entry['parent'];
+		}
+
+		return $propagatedEntries;
+	}
+}
diff --git a/lib/private/files/fileinfo.php b/lib/private/files/fileinfo.php
index bb810dd45ed..5b5e8697004 100644
--- a/lib/private/files/fileinfo.php
+++ b/lib/private/files/fileinfo.php
@@ -62,6 +62,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	private $owner;
 
 	/**
+	 * @var string[]
+	 */
+	private $childEtags = [];
+
+	/**
 	 * @param string|boolean $path
 	 * @param Storage\Storage $storage
 	 * @param string $internalPath
@@ -93,6 +98,8 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	public function offsetGet($offset) {
 		if ($offset === 'type') {
 			return $this->getType();
+		} else if ($offset === 'etag') {
+			return $this->getEtag();
 		} elseif (isset($this->data[$offset])) {
 			return $this->data[$offset];
 		} else {
@@ -153,7 +160,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	 * @return string
 	 */
 	public function getEtag() {
-		return $this->data['etag'];
+		if (count($this->childEtags) > 0) {
+			$combinedEtag = $this->data['etag'] . '::' . implode('::', $this->childEtags);
+			return md5($combinedEtag);
+		} else {
+			return $this->data['etag'];
+		}
 	}
 
 	/**
@@ -285,4 +297,26 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	public function getOwner() {
 		return $this->owner;
 	}
+
+	/**
+	 * Add a cache entry which is the child of this folder
+	 *
+	 * Sets the size, etag and size to for cross-storage childs
+	 *
+	 * @param array $data cache entry for the child
+	 * @param string $entryPath full path of the child entry
+	 */
+	public function addSubEntry($data, $entryPath) {
+		$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
+		if (isset($data['mtime'])) {
+			$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
+		}
+		if (isset($data['etag'])) {
+			// prefix the etag with the relative path of the subentry to propagate etag on mount moves
+			$relativeEntryPath = substr($entryPath, strlen($this->getPath()));
+			// attach the permissions to propagate etag on permision changes of submounts
+			$permissions = isset($data['permissions']) ? $data['permissions'] : 0;
+			$this->childEtags[] = $relativeEntryPath . '/' . $data['etag'] . $permissions;
+		}
+	}
 }
diff --git a/lib/private/files/storage/common.php b/lib/private/files/storage/common.php
index 8e4958a930d..3772b442b59 100644
--- a/lib/private/files/storage/common.php
+++ b/lib/private/files/storage/common.php
@@ -37,6 +37,7 @@
 namespace OC\Files\Storage;
 
 use OC\Files\Cache\Cache;
+use OC\Files\Cache\Propagator;
 use OC\Files\Cache\Scanner;
 use OC\Files\Filesystem;
 use OC\Files\Cache\Watcher;
@@ -64,6 +65,7 @@ abstract class Common implements Storage {
 	protected $cache;
 	protected $scanner;
 	protected $watcher;
+	protected $propagator;
 	protected $storageCache;
 
 	protected $mountOptions = [];
@@ -345,6 +347,22 @@ abstract class Common implements Storage {
 		return $this->watcher;
 	}
 
+	/**
+	 * get a propagator instance for the cache
+	 *
+	 * @param \OC\Files\Storage\Storage (optional) the storage to pass to the watcher
+	 * @return \OC\Files\Cache\Propagator
+	 */
+	public function getPropagator($storage = null) {
+		if (!$storage) {
+			$storage = $this;
+		}
+		if (!isset($this->propagator)) {
+			$this->propagator = new Propagator($storage);
+		}
+		return $this->propagator;
+	}
+
 	public function getStorageCache($storage = null) {
 		if (!$storage) {
 			$storage = $this;
diff --git a/lib/private/files/storage/storage.php b/lib/private/files/storage/storage.php
index f46ac544b56..3399c717890 100644
--- a/lib/private/files/storage/storage.php
+++ b/lib/private/files/storage/storage.php
@@ -68,6 +68,14 @@ interface Storage extends \OCP\Files\Storage {
 	public function getWatcher($path = '', $storage = null);
 
 	/**
+	 * get a propagator instance for the cache
+	 *
+	 * @param \OC\Files\Storage\Storage (optional) the storage to pass to the watcher
+	 * @return \OC\Files\Cache\Propagator
+	 */
+	public function getPropagator($storage = null);
+
+	/**
 	 * @return \OC\Files\Cache\Storage
 	 */
 	public function getStorageCache();
diff --git a/lib/private/files/storage/wrapper/wrapper.php b/lib/private/files/storage/wrapper/wrapper.php
index 048738170db..5327b94211b 100644
--- a/lib/private/files/storage/wrapper/wrapper.php
+++ b/lib/private/files/storage/wrapper/wrapper.php
@@ -430,6 +430,13 @@ class Wrapper implements \OC\Files\Storage\Storage {
 		return $this->storage->getWatcher($path, $storage);
 	}
 
+	public function getPropagator($storage = null) {
+		if (!$storage) {
+			$storage = $this;
+		}
+		return $this->storage->getPropagator($storage);
+	}
+
 	/**
 	 * @return \OC\Files\Cache\Storage
 	 */
diff --git a/lib/private/files/view.php b/lib/private/files/view.php
index cee4b182425..3ad9d36a384 100644
--- a/lib/private/files/view.php
+++ b/lib/private/files/view.php
@@ -707,10 +707,6 @@ class View {
 				} else if ($result) {
 					if ($internalPath1 !== '') { // dont do a cache update for moved mounts
 						$this->updater->rename($path1, $path2);
-					} else { // only do etag propagation
-						$this->getUpdater()->getPropagator()->addChange($path1);
-						$this->getUpdater()->getPropagator()->addChange($path2);
-						$this->getUpdater()->getPropagator()->propagateChanges();
 					}
 				}
 
@@ -1179,6 +1175,48 @@ class View {
 	}
 
 	/**
+	 * Get file info from cache
+	 *
+	 * If the file is not in cached it will be scanned
+	 * If the file has changed on storage the cache will be updated
+	 *
+	 * @param \OC\Files\Storage\Storage $storage
+	 * @param string $internalPath
+	 * @param string $relativePath
+	 * @return array|bool
+	 */
+	private function getCacheEntry($storage, $internalPath, $relativePath) {
+		$cache = $storage->getCache($internalPath);
+		$data = $cache->get($internalPath);
+		$watcher = $storage->getWatcher($internalPath);
+
+		try {
+			// if the file is not in the cache or needs to be updated, trigger the scanner and reload the data
+			if (!$data || $data['size'] === -1) {
+				$this->lockFile($relativePath, ILockingProvider::LOCK_SHARED);
+				if (!$storage->file_exists($internalPath)) {
+					$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
+					return false;
+				}
+				$scanner = $storage->getScanner($internalPath);
+				$scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW);
+				$data = $cache->get($internalPath);
+				$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
+			} else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) {
+				$this->lockFile($relativePath, ILockingProvider::LOCK_SHARED);
+				$watcher->update($internalPath, $data);
+				$this->updater->propagate($relativePath);
+				$data = $cache->get($internalPath);
+				$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
+			}
+		} catch (LockedException $e) {
+			// if the file is locked we just use the old cache info
+		}
+
+		return $data;
+	}
+
+	/**
 	 * get the filesystem info
 	 *
 	 * @param string $path
@@ -1189,9 +1227,8 @@ class View {
 	 */
 	public function getFileInfo($path, $includeMountPoints = true) {
 		$this->assertPathLength($path);
-		$data = array();
 		if (!Filesystem::isValidPath($path)) {
-			return $data;
+			return false;
 		}
 		if (Cache\Scanner::isPartialFile($path)) {
 			return $this->getPartFileInfo($path);
@@ -1202,48 +1239,27 @@ class View {
 		$mount = Filesystem::getMountManager()->find($path);
 		$storage = $mount->getStorage();
 		$internalPath = $mount->getInternalPath($path);
-		$data = null;
 		if ($storage) {
-			$cache = $storage->getCache($internalPath);
+			$data = $this->getCacheEntry($storage, $internalPath, $relativePath);
 
-			$data = $cache->get($internalPath);
-			$watcher = $storage->getWatcher($internalPath);
+			if(!is_array($data)) {
+				return false;
+			}
 
-			try {
-				// if the file is not in the cache or needs to be updated, trigger the scanner and reload the data
-				if (!$data) {
-					$this->lockFile($relativePath, ILockingProvider::LOCK_SHARED);
-					if (!$storage->file_exists($internalPath)) {
-						$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
-						return false;
-					}
-					$scanner = $storage->getScanner($internalPath);
-					$scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW);
-					$data = $cache->get($internalPath);
-					$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
-				} else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) {
-					$this->lockFile($relativePath, ILockingProvider::LOCK_SHARED);
-					$watcher->update($internalPath, $data);
-					$this->updater->propagate($path);
-					$data = $cache->get($internalPath);
-					$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
-				}
-			} catch (LockedException $e) {
-				// if the file is locked we just use the old cache info
+			if ($mount instanceof MoveableMount && $internalPath === '') {
+				$data['permissions'] |= \OCP\Constants::PERMISSION_DELETE;
 			}
 
+			$owner = $this->getUserObjectForOwner($storage->getOwner($internalPath));
+			$info = new FileInfo($path, $storage, $internalPath, $data, $mount, $owner);
+
 			if ($data and isset($data['fileid'])) {
-				// upgrades from oc6 or lower might not have the permissions set in the file cache
-				if ($data['permissions'] === 0) {
-					$data['permissions'] = $storage->getPermissions($data['path']);
-					$cache->update($data['fileid'], array('permissions' => $data['permissions']));
-				}
 				if ($includeMountPoints and $data['mimetype'] === 'httpd/unix-directory') {
 					//add the sizes of other mount points to the folder
 					$extOnly = ($includeMountPoints === 'ext');
-					$mountPoints = Filesystem::getMountPoints($path);
-					foreach ($mountPoints as $mountPoint) {
-						$subStorage = Filesystem::getStorage($mountPoint);
+					$mounts = Filesystem::getMountManager()->findIn($path);
+					foreach ($mounts as $mount) {
+						$subStorage = $mount->getStorage();
 						if ($subStorage) {
 							// exclude shared storage ?
 							if ($extOnly && $subStorage instanceof \OC\Files\Storage\Shared) {
@@ -1251,22 +1267,16 @@ class View {
 							}
 							$subCache = $subStorage->getCache('');
 							$rootEntry = $subCache->get('');
-							$data['size'] += isset($rootEntry['size']) ? $rootEntry['size'] : 0;
+							$info->addSubEntry($rootEntry, $mount->getMountPoint());
 						}
 					}
 				}
 			}
-		}
-		if (!$data) {
-			return false;
-		}
 
-		if ($mount instanceof MoveableMount && $internalPath === '') {
-			$data['permissions'] |= \OCP\Constants::PERMISSION_DELETE;
+			return $info;
 		}
 
-		$owner = $this->getUserObjectForOwner($storage->getOwner($internalPath));
-		return new FileInfo($path, $storage, $internalPath, $data, $mount, $owner);
+		return false;
 	}
 
 	/**
@@ -1278,9 +1288,8 @@ class View {
 	 */
 	public function getDirectoryContent($directory, $mimetype_filter = '') {
 		$this->assertPathLength($directory);
-		$result = array();
 		if (!Filesystem::isValidPath($directory)) {
-			return $result;
+			return [];
 		}
 		$path = $this->getAbsolutePath($directory);
 		$path = Filesystem::normalizePath($path);
@@ -1291,50 +1300,25 @@ class View {
 			$cache = $storage->getCache($internalPath);
 			$user = \OC_User::getUser();
 
-			/**
-			 * @var \OC\Files\FileInfo[] $files
-			 */
-			$files = array();
+			$data = $this->getCacheEntry($storage, $internalPath, $directory);
 
-			$data = $cache->get($internalPath);
-			$watcher = $storage->getWatcher($internalPath);
-			try {
-				if (!$data or $data['size'] === -1) {
-					$this->lockFile($directory, ILockingProvider::LOCK_SHARED);
-					if (!$storage->file_exists($internalPath)) {
-						$this->unlockFile($directory, ILockingProvider::LOCK_SHARED);
-						return array();
-					}
-					$scanner = $storage->getScanner($internalPath);
-					$scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW);
-					$data = $cache->get($internalPath);
-					$this->unlockFile($directory, ILockingProvider::LOCK_SHARED);
-				} else if ($watcher->needsUpdate($internalPath, $data)) {
-					$this->lockFile($directory, ILockingProvider::LOCK_SHARED);
-					$watcher->update($internalPath, $data);
-					$this->updater->propagate($path);
-					$data = $cache->get($internalPath);
-					$this->unlockFile($directory, ILockingProvider::LOCK_SHARED);
-				}
-			} catch (LockedException $e) {
-				// if the file is locked we just use the old cache info
+			if (!is_array($data) || !isset($data['fileid'])) {
+				return [];
 			}
 
 			$folderId = $data['fileid'];
 			$contents = $cache->getFolderContentsById($folderId); //TODO: mimetype_filter
 
-			foreach ($contents as $content) {
-				if ($content['permissions'] === 0) {
-					$content['permissions'] = $storage->getPermissions($content['path']);
-					$cache->update($content['fileid'], array('permissions' => $content['permissions']));
-				}
-				// if sharing was disabled for the user we remove the share permissions
+			/**
+			 * @var \OC\Files\FileInfo[] $files
+			 */
+			$files = array_map(function (array $content) use ($path, $storage, $mount) {
 				if (\OCP\Util::isSharingDisabledForUser()) {
 					$content['permissions'] = $content['permissions'] & ~\OCP\Constants::PERMISSION_SHARE;
 				}
 				$owner = $this->getUserObjectForOwner($storage->getOwner($content['path']));
-				$files[] = new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner);
-			}
+				return new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner);
+			}, $contents);
 
 			//add a folder for any mountpoint in this directory and add the sizes of other mountpoints to the folders
 			$mounts = Filesystem::getMountManager()->findIn($path);
@@ -1345,7 +1329,8 @@ class View {
 				if ($subStorage) {
 					$subCache = $subStorage->getCache('');
 
-					if ($subCache->getStatus('') === Cache\Cache::NOT_FOUND) {
+					$rootEntry = $subCache->get('');
+					if (!$rootEntry) {
 						$subScanner = $subStorage->getScanner('');
 						try {
 							$subScanner->scanFile('');
@@ -1363,17 +1348,17 @@ class View {
 							);
 							continue;
 						}
+						$rootEntry = $subCache->get('');
 					}
 
-					$rootEntry = $subCache->get('');
 					if ($rootEntry) {
 						$relativePath = trim(substr($mountPoint, $dirLength), '/');
 						if ($pos = strpos($relativePath, '/')) {
 							//mountpoint inside subfolder add size to the correct folder
 							$entryName = substr($relativePath, 0, $pos);
 							foreach ($files as &$entry) {
-								if ($entry['name'] === $entryName) {
-									$entry['size'] += $rootEntry['size'];
+								if ($entry->getName() === $entryName) {
+									$entry->addSubEntry($rootEntry, $mountPoint);
 								}
 							}
 						} else { //mountpoint in this folder, add an entry for it
@@ -1410,23 +1395,23 @@ class View {
 			}
 
 			if ($mimetype_filter) {
-				foreach ($files as $file) {
+				$files = array_filter($files, function (FileInfo $file) use ($mimetype_filter) {
 					if (strpos($mimetype_filter, '/')) {
-						if ($file['mimetype'] === $mimetype_filter) {
+						if ($file->getMimetype() === $mimetype_filter) {
 							$result[] = $file;
 						}
 					} else {
-						if ($file['mimepart'] === $mimetype_filter) {
+						if ($file->getMimePart() === $mimetype_filter) {
 							$result[] = $file;
 						}
 					}
-				}
-			} else {
-				$result = $files;
+				});
 			}
-		}
 
-		return $result;
+			return $files;
+		} else {
+			return [];
+		}
 	}
 
 	/**