Merge pull request #5424 from nextcloud/moveFromCache-from-shared

fix moving folders out of a cache jail

7 files changed, 313 insertions, 3 deletions

diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index ee1dc08021c..5a8a921f219 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -751,6 +751,7 @@ return array(
     'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => $baseDir . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php',
     'OC\\Repair\\NC12\\InstallCoreBundle' => $baseDir . '/lib/private/Repair/NC12/InstallCoreBundle.php',
     'OC\\Repair\\NC12\\UpdateLanguageCodes' => $baseDir . '/lib/private/Repair/NC12/UpdateLanguageCodes.php',
+    'OC\\Repair\\NC13\\RepairInvalidPaths' => $baseDir . '/lib/private/Repair/NC13/RepairInvalidPaths.php',
     'OC\\Repair\\OldGroupMembershipShares' => $baseDir . '/lib/private/Repair/OldGroupMembershipShares.php',
     'OC\\Repair\\Owncloud\\SaveAccountsTableData' => $baseDir . '/lib/private/Repair/Owncloud/SaveAccountsTableData.php',
     'OC\\Repair\\RemoveRootShares' => $baseDir . '/lib/private/Repair/RemoveRootShares.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 7b133d54a20..b4aa6cc322b 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -781,6 +781,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php',
         'OC\\Repair\\NC12\\InstallCoreBundle' => __DIR__ . '/../../..' . '/lib/private/Repair/NC12/InstallCoreBundle.php',
         'OC\\Repair\\NC12\\UpdateLanguageCodes' => __DIR__ . '/../../..' . '/lib/private/Repair/NC12/UpdateLanguageCodes.php',
+        'OC\\Repair\\NC13\\RepairInvalidPaths' => __DIR__ . '/../../..' . '/lib/private/Repair/NC13/RepairInvalidPaths.php',
         'OC\\Repair\\OldGroupMembershipShares' => __DIR__ . '/../../..' . '/lib/private/Repair/OldGroupMembershipShares.php',
         'OC\\Repair\\Owncloud\\SaveAccountsTableData' => __DIR__ . '/../../..' . '/lib/private/Repair/Owncloud/SaveAccountsTableData.php',
         'OC\\Repair\\RemoveRootShares' => __DIR__ . '/../../..' . '/lib/private/Repair/RemoveRootShares.php',
diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php
index ebab20fbaed..8f12ca77ee6 100644
--- a/lib/private/Files/Cache/Wrapper/CacheJail.php
+++ b/lib/private/Files/Cache/Wrapper/CacheJail.php
@@ -94,7 +94,7 @@ class CacheJail extends CacheWrapper {
 	 * get the stored metadata of a file or folder
 	 *
 	 * @param string /int $file
-	 * @return array|false
+	 * @return ICacheEntry|false
 	 */
 	public function get($file) {
 		if (is_string($file) or $file == '') {
@@ -176,6 +176,16 @@ class CacheJail extends CacheWrapper {
 	}
 
 	/**
+	 * Get the storage id and path needed for a move
+	 *
+	 * @param string $path
+	 * @return array [$storageId, $internalPath]
+	 */
+	protected function getMoveInfo($path) {
+		return [$this->getNumericStorageId(), $this->getSourcePath($path)];
+	}
+
+	/**
 	 * remove all entries for files that are stored on the storage from the cache
 	 */
 	public function clear() {
diff --git a/lib/private/Repair.php b/lib/private/Repair.php
index 4d14bf2550c..dae328e6340 100644
--- a/lib/private/Repair.php
+++ b/lib/private/Repair.php
@@ -42,6 +42,7 @@ use OC\Repair\NC12\UpdateLanguageCodes;
 use OC\Repair\OldGroupMembershipShares;
 use OC\Repair\Owncloud\SaveAccountsTableData;
 use OC\Repair\RemoveRootShares;
+use OC\Repair\NC13\RepairInvalidPaths;
 use OC\Repair\SqliteAutoincrement;
 use OC\Repair\RepairMimeTypes;
 use OC\Repair\RepairInvalidShares;
@@ -143,7 +144,8 @@ class Repair implements IOutput{
 				\OC::$server->query(BundleFetcher::class),
 				\OC::$server->getConfig(),
 				\OC::$server->query(Installer::class)
-			)
+			),
+			new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), \OC::$server->getConfig())
 		];
 	}
 
@@ -155,7 +157,7 @@ class Repair implements IOutput{
 	 */
 	public static function getExpensiveRepairSteps() {
 		return [
-			new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()),
+			new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager())
 		];
 	}
 
diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php
new file mode 100644
index 00000000000..076fbb735c8
--- /dev/null
+++ b/lib/private/Repair/NC13/RepairInvalidPaths.php
@@ -0,0 +1,129 @@
+<?php
+/**
+ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Repair\NC13;
+
+
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class RepairInvalidPaths implements IRepairStep {
+	/** @var IDBConnection */
+	private $connection;
+	/** @var IConfig */
+	private $config;
+
+	public function __construct(IDBConnection $connection, IConfig $config) {
+		$this->connection = $connection;
+		$this->config = $config;
+	}
+
+
+	public function getName() {
+		return 'Repair invalid paths in file cache';
+	}
+
+	private function getInvalidEntries() {
+		$builder = $this->connection->getQueryBuilder();
+
+		$computedPath = $builder->func()->concat(
+			'p.path',
+			$builder->func()->concat($builder->createNamedParameter('/'), 'f.name')
+		);
+
+		//select f.path, f.parent,p.path from oc_filecache f inner join oc_filecache p on f.parent=p.fileid and p.path!='' where f.path != p.path || '/' || f.name;
+		$query = $builder->select('f.fileid', 'f.path', 'p.path AS parent_path', 'f.name', 'f.parent', 'f.storage')
+			->from('filecache', 'f')
+			->innerJoin('f', 'filecache', 'p', $builder->expr()->andX(
+				$builder->expr()->eq('f.parent', 'p.fileid'),
+				$builder->expr()->neq('p.name', $builder->createNamedParameter(''))
+			))
+			->where($builder->expr()->neq('f.path', $computedPath));
+
+		return $query->execute()->fetchAll();
+	}
+
+	private function getId($storage, $path) {
+		$builder = $this->connection->getQueryBuilder();
+
+		$query = $builder->select('fileid')
+			->from('filecache')
+			->where($builder->expr()->eq('storage', $builder->createNamedParameter($storage)))
+			->andWhere($builder->expr()->eq('path', $builder->createNamedParameter($path)));
+
+		return $query->execute()->fetchColumn();
+	}
+
+	private function update($fileid, $newPath) {
+		$builder = $this->connection->getQueryBuilder();
+
+		$query = $builder->update('filecache')
+			->set('path', $builder->createNamedParameter($newPath))
+			->set('path_hash', $builder->createNamedParameter(md5($newPath)))
+			->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid)));
+
+		$query->execute();
+	}
+
+	private function reparent($from, $to) {
+		$builder = $this->connection->getQueryBuilder();
+
+		$query = $builder->update('filecache')
+			->set('parent', $builder->createNamedParameter($to))
+			->where($builder->expr()->eq('fileid', $builder->createNamedParameter($from)));
+		$query->execute();
+	}
+
+	private function delete($fileid) {
+		$builder = $this->connection->getQueryBuilder();
+
+		$query = $builder->delete('filecache')
+			->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid)));
+		$query->execute();
+	}
+
+	private function repair() {
+		$entries = $this->getInvalidEntries();
+		foreach ($entries as $entry) {
+			$calculatedPath = $entry['parent_path'] . '/' . $entry['name'];
+			if ($newId = $this->getId($entry['storage'], $calculatedPath)) {
+				// a new entry with the correct path has already been created, reuse that one and delete the incorrect entry
+				$this->reparent($entry['fileid'], $newId);
+				$this->delete($entry['fileid']);
+			} else {
+				$this->update($entry['fileid'], $calculatedPath);
+			}
+		}
+		return count($entries);
+	}
+
+	public function run(IOutput $output) {
+		$versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0');
+		// was added to 12.0.0.30 and 13.0.0.1
+		if (version_compare($versionFromBeforeUpdate, '12.0.0.30', '<') || version_compare($versionFromBeforeUpdate, '13.0.0.0', '==')) {
+			$count = $this->repair();
+
+			$output->info('Repaired ' . $count . ' paths');
+		}
+	}
+}
diff --git a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php
index e3043c50d57..f26e3a59f1c 100644
--- a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php
+++ b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php
@@ -8,6 +8,7 @@
 
 namespace Test\Files\Cache\Wrapper;
 
+use OC\Files\Cache\Wrapper\CacheJail;
 use Test\Files\Cache\CacheTest;
 
 /**
@@ -80,4 +81,53 @@ class CacheJailTest extends CacheTest {
 		//not supported
 		$this->assertTrue(true);
 	}
+
+	function testMoveFromJail() {
+		$folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory');
+
+		$this->sourceCache->put('source', $folderData);
+		$this->sourceCache->put('source/foo', $folderData);
+		$this->sourceCache->put('source/foo/bar', $folderData);
+		$this->sourceCache->put('target', $folderData);
+
+		$jail = new CacheJail($this->sourceCache, 'source');
+
+		$this->sourceCache->moveFromCache($jail, 'foo', 'target/foo');
+
+		$this->assertTrue($this->sourceCache->inCache('target/foo'));
+		$this->assertTrue($this->sourceCache->inCache('target/foo/bar'));
+	}
+
+	function testMoveToJail() {
+		$folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory');
+
+		$this->sourceCache->put('source', $folderData);
+		$this->sourceCache->put('source/foo', $folderData);
+		$this->sourceCache->put('source/foo/bar', $folderData);
+		$this->sourceCache->put('target', $folderData);
+
+		$jail = new CacheJail($this->sourceCache, 'target');
+
+		$jail->moveFromCache($this->sourceCache, 'source/foo', 'foo');
+
+		$this->assertTrue($this->sourceCache->inCache('target/foo'));
+		$this->assertTrue($this->sourceCache->inCache('target/foo/bar'));
+	}
+
+	function testMoveBetweenJail() {
+		$folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory');
+
+		$this->sourceCache->put('source', $folderData);
+		$this->sourceCache->put('source/foo', $folderData);
+		$this->sourceCache->put('source/foo/bar', $folderData);
+		$this->sourceCache->put('target', $folderData);
+
+		$jail = new CacheJail($this->sourceCache, 'target');
+		$sourceJail = new CacheJail($this->sourceCache, 'source');
+
+		$jail->moveFromCache($sourceJail, 'foo', 'foo');
+
+		$this->assertTrue($this->sourceCache->inCache('target/foo'));
+		$this->assertTrue($this->sourceCache->inCache('target/foo/bar'));
+	}
 }
diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php
new file mode 100644
index 00000000000..b18758585c1
--- /dev/null
+++ b/tests/lib/Repair/RepairInvalidPathsTest.php
@@ -0,0 +1,117 @@
+<?php
+/**
+ * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Test\Repair;
+
+use OC\Files\Cache\Cache;
+use OC\Files\Storage\Temporary;
+use OC\Repair\NC13\RepairInvalidPaths;
+use OCP\IConfig;
+use OCP\Migration\IOutput;
+use Test\TestCase;
+
+/**
+ * @group DB
+ */
+class RepairInvalidPathsTest extends TestCase {
+	/** @var Temporary */
+	private $storage;
+	/** @var Cache */
+	private $cache;
+	/** @var RepairInvalidPaths */
+	private $repair;
+
+	protected function setUp() {
+		parent::setUp();
+
+		$this->storage = new Temporary();
+		$this->cache = $this->storage->getCache();
+		$config = $this->createMock(IConfig::class);
+		$config->expects($this->any())
+			->method('getSystemValue')
+			->with('version', '0.0.0')
+			->willReturn('12.0.0.0');
+		$this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), $config);
+	}
+
+	protected function tearDown() {
+		$this->cache->clear();
+
+		return parent::tearDown();
+	}
+
+	public function testRepairNonDuplicate() {
+		$this->storage->mkdir('foo/bar/asd');
+		$this->storage->mkdir('foo2');
+		$this->storage->getScanner()->scan('');
+
+		$folderId = $this->cache->getId('foo/bar');
+		$newParentFolderId = $this->cache->getId('foo2');
+		// failed rename, moved entry is updated but not it's children
+		$this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]);
+
+		$this->assertTrue($this->cache->inCache('foo2/bar'));
+		$this->assertTrue($this->cache->inCache('foo/bar/asd'));
+		$this->assertFalse($this->cache->inCache('foo2/bar/asd'));
+
+		$this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']);
+
+		$this->repair->run($this->createMock(IOutput::class));
+
+		$this->assertTrue($this->cache->inCache('foo2/bar'));
+		$this->assertTrue($this->cache->inCache('foo2/bar/asd'));
+		$this->assertFalse($this->cache->inCache('foo/bar/asd'));
+
+		$this->assertEquals($folderId, $this->cache->get('foo2/bar/asd')['parent']);
+		$this->assertEquals($folderId, $this->cache->getId('foo2/bar'));
+	}
+
+	public function testRepairDuplicate() {
+		$this->storage->mkdir('foo/bar/asd');
+		$this->storage->mkdir('foo2');
+		$this->storage->getScanner()->scan('');
+
+		$folderId = $this->cache->getId('foo/bar');
+		$newParentFolderId = $this->cache->getId('foo2');
+		// failed rename, moved entry is updated but not it's children
+		$this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]);
+		$this->storage->rename('foo/bar', 'foo2/bar');
+		$this->storage->mkdir('foo2/bar/asd/foo');
+
+		// usage causes the renamed subfolder to be scanned
+		$this->storage->getScanner()->scan('foo2/bar/asd');
+
+		$this->assertTrue($this->cache->inCache('foo2/bar'));
+		$this->assertTrue($this->cache->inCache('foo/bar/asd'));
+		$this->assertTrue($this->cache->inCache('foo2/bar/asd'));
+
+		$this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']);
+
+		$this->repair->run($this->createMock(IOutput::class));
+
+		$this->assertTrue($this->cache->inCache('foo2/bar'));
+		$this->assertTrue($this->cache->inCache('foo2/bar/asd'));
+		$this->assertFalse($this->cache->inCache('foo/bar/asd'));
+
+		$this->assertEquals($this->cache->getId('foo2/bar'), $this->cache->get('foo2/bar/asd')['parent']);
+		$this->assertEquals($this->cache->getId('foo2/bar/asd'), $this->cache->get('foo2/bar/asd/foo')['parent']);
+	}
+}