1 files changed, 453 insertions, 0 deletions
diff --git a/tests/lib/Files/Cache/ScannerTest.php b/tests/lib/Files/Cache/ScannerTest.php
new file mode 100644
index 00000000000..123c13893f7
--- /dev/null
+++ b/tests/lib/Files/Cache/ScannerTest.php
@@ -0,0 +1,453 @@
+<?php
+
+/**
+ * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace Test\Files\Cache;
+
+use OC;
+use OC\Files\Cache\Cache;
+use OC\Files\Cache\CacheEntry;
+use OC\Files\Cache\Scanner;
+use OC\Files\Storage\Storage;
+use OC\Files\Storage\Temporary;
+use OCP\Files\Cache\IScanner;
+use OCP\IDBConnection;
+use OCP\Server;
+use Test\TestCase;
+
+/**
+ * Class ScannerTest
+ *
+ * @group DB
+ *
+ * @package Test\Files\Cache
+ */
+class ScannerTest extends TestCase {
+	private Storage $storage;
+	private Scanner $scanner;
+	private Cache $cache;
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$this->storage = new Temporary([]);
+		$this->scanner = new Scanner($this->storage);
+		$this->cache = new Cache($this->storage);
+	}
+
+	protected function tearDown(): void {
+		$this->cache->clear();
+
+		parent::tearDown();
+	}
+
+	public function testFile(): void {
+		$data = "dummy file data\n";
+		$this->storage->file_put_contents('foo.txt', $data);
+		$this->scanner->scanFile('foo.txt');
+
+		$this->assertEquals($this->cache->inCache('foo.txt'), true);
+		$cachedData = $this->cache->get('foo.txt');
+		$this->assertEquals($cachedData['size'], strlen($data));
+		$this->assertEquals($cachedData['mimetype'], 'text/plain');
+		$this->assertNotEquals($cachedData['parent'], -1); //parent folders should be scanned automatically
+
+		$data = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png');
+		$this->storage->file_put_contents('foo.png', $data);
+		$this->scanner->scanFile('foo.png');
+
+		$this->assertEquals($this->cache->inCache('foo.png'), true);
+		$cachedData = $this->cache->get('foo.png');
+		$this->assertEquals($cachedData['size'], strlen($data));
+		$this->assertEquals($cachedData['mimetype'], 'image/png');
+	}
+
+	public function testFile4Byte(): void {
+		$data = "dummy file data\n";
+		$this->storage->file_put_contents('foo🙈.txt', $data);
+
+		if (Server::get(IDBConnection::class)->supports4ByteText()) {
+			$this->assertNotNull($this->scanner->scanFile('foo🙈.txt'));
+			$this->assertTrue($this->cache->inCache('foo🙈.txt'), true);
+
+			$cachedData = $this->cache->get('foo🙈.txt');
+			$this->assertEquals(strlen($data), $cachedData['size']);
+			$this->assertEquals('text/plain', $cachedData['mimetype']);
+			$this->assertNotEquals(-1, $cachedData['parent']); //parent folders should be scanned automatically
+		} else {
+			$this->assertNull($this->scanner->scanFile('foo🙈.txt'));
+			$this->assertFalse($this->cache->inCache('foo🙈.txt'), true);
+		}
+	}
+
+	public function testFileInvalidChars(): void {
+		$data = "dummy file data\n";
+		$this->storage->file_put_contents("foo\nbar.txt", $data);
+
+		$this->assertNull($this->scanner->scanFile("foo\nbar.txt"));
+		$this->assertFalse($this->cache->inCache("foo\nbar.txt"), true);
+	}
+
+	private function fillTestFolders() {
+		$textData = "dummy file data\n";
+		$imgData = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png');
+		$this->storage->mkdir('folder');
+		$this->storage->file_put_contents('foo.txt', $textData);
+		$this->storage->file_put_contents('foo.png', $imgData);
+		$this->storage->file_put_contents('folder/bar.txt', $textData);
+	}
+
+	public function testFolder(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+		$this->assertEquals($this->cache->inCache(''), true);
+		$this->assertEquals($this->cache->inCache('foo.txt'), true);
+		$this->assertEquals($this->cache->inCache('foo.png'), true);
+		$this->assertEquals($this->cache->inCache('folder'), true);
+		$this->assertEquals($this->cache->inCache('folder/bar.txt'), true);
+
+		$cachedDataText = $this->cache->get('foo.txt');
+		$cachedDataText2 = $this->cache->get('foo.txt');
+		$cachedDataImage = $this->cache->get('foo.png');
+		$cachedDataFolder = $this->cache->get('');
+		$cachedDataFolder2 = $this->cache->get('folder');
+
+		$this->assertEquals($cachedDataImage['parent'], $cachedDataText['parent']);
+		$this->assertEquals($cachedDataFolder['fileid'], $cachedDataImage['parent']);
+		$this->assertEquals($cachedDataFolder['size'], $cachedDataImage['size'] + $cachedDataText['size'] + $cachedDataText2['size']);
+		$this->assertEquals($cachedDataFolder2['size'], $cachedDataText2['size']);
+	}
+
+	public function testShallow(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW);
+		$this->assertEquals($this->cache->inCache(''), true);
+		$this->assertEquals($this->cache->inCache('foo.txt'), true);
+		$this->assertEquals($this->cache->inCache('foo.png'), true);
+		$this->assertEquals($this->cache->inCache('folder'), true);
+		$this->assertEquals($this->cache->inCache('folder/bar.txt'), false);
+
+		$cachedDataFolder = $this->cache->get('');
+		$cachedDataFolder2 = $this->cache->get('folder');
+
+		$this->assertEquals(-1, $cachedDataFolder['size']);
+		$this->assertEquals(-1, $cachedDataFolder2['size']);
+
+		$this->scanner->scan('folder', IScanner::SCAN_SHALLOW);
+
+		$cachedDataFolder2 = $this->cache->get('folder');
+
+		$this->assertNotEquals($cachedDataFolder2['size'], -1);
+
+		$this->cache->correctFolderSize('folder');
+
+		$cachedDataFolder = $this->cache->get('');
+		$this->assertNotEquals($cachedDataFolder['size'], -1);
+	}
+
+	public function testBackgroundScan(): void {
+		$this->fillTestFolders();
+		$this->storage->mkdir('folder2');
+		$this->storage->file_put_contents('folder2/bar.txt', 'foobar');
+
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW);
+		$this->assertFalse($this->cache->inCache('folder/bar.txt'));
+		$this->assertFalse($this->cache->inCache('folder/2bar.txt'));
+		$cachedData = $this->cache->get('');
+		$this->assertEquals(-1, $cachedData['size']);
+
+		$this->scanner->backgroundScan();
+
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+
+		$cachedData = $this->cache->get('');
+		$this->assertnotEquals(-1, $cachedData['size']);
+
+		$this->assertFalse($this->cache->getIncomplete());
+	}
+
+	public function testBackgroundScanOnlyRecurseIncomplete(): void {
+		$this->fillTestFolders();
+		$this->storage->mkdir('folder2');
+		$this->storage->file_put_contents('folder2/bar.txt', 'foobar');
+
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW);
+		$this->assertFalse($this->cache->inCache('folder/bar.txt'));
+		$this->assertFalse($this->cache->inCache('folder/2bar.txt'));
+		$this->assertFalse($this->cache->inCache('folder2/bar.txt'));
+		$this->cache->put('folder2', ['size' => 1]); // mark as complete
+
+		$cachedData = $this->cache->get('');
+		$this->assertEquals(-1, $cachedData['size']);
+
+		$this->scanner->scan('', IScanner::SCAN_RECURSIVE_INCOMPLETE, IScanner::REUSE_ETAG | IScanner::REUSE_SIZE);
+
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$this->assertFalse($this->cache->inCache('folder2/bar.txt'));
+
+		$cachedData = $this->cache->get('');
+		$this->assertNotEquals(-1, $cachedData['size']);
+
+		$this->assertFalse($this->cache->getIncomplete());
+	}
+
+	public function testBackgroundScanNestedIncompleteFolders(): void {
+		$this->storage->mkdir('folder');
+		$this->scanner->backgroundScan();
+
+		$this->storage->mkdir('folder/subfolder1');
+		$this->storage->mkdir('folder/subfolder2');
+
+		$this->storage->mkdir('folder/subfolder1/subfolder3');
+		$this->cache->put('folder', ['size' => -1]);
+		$this->cache->put('folder/subfolder1', ['size' => -1]);
+
+		// do a scan to get the folders into the cache.
+		$this->scanner->backgroundScan();
+
+		$this->assertTrue($this->cache->inCache('folder/subfolder1/subfolder3'));
+
+		$this->storage->file_put_contents('folder/subfolder1/bar1.txt', 'foobar');
+		$this->storage->file_put_contents('folder/subfolder1/subfolder3/bar3.txt', 'foobar');
+		$this->storage->file_put_contents('folder/subfolder2/bar2.txt', 'foobar');
+
+		//mark folders as incomplete.
+		$this->cache->put('folder/subfolder1', ['size' => -1]);
+		$this->cache->put('folder/subfolder2', ['size' => -1]);
+		$this->cache->put('folder/subfolder1/subfolder3', ['size' => -1]);
+
+		$this->scanner->backgroundScan();
+
+		$this->assertTrue($this->cache->inCache('folder/subfolder1/bar1.txt'));
+		$this->assertTrue($this->cache->inCache('folder/subfolder2/bar2.txt'));
+		$this->assertTrue($this->cache->inCache('folder/subfolder1/subfolder3/bar3.txt'));
+
+		//check if folder sizes are correct.
+		$this->assertEquals(18, $this->cache->get('folder')['size']);
+		$this->assertEquals(12, $this->cache->get('folder/subfolder1')['size']);
+		$this->assertEquals(6, $this->cache->get('folder/subfolder1/subfolder3')['size']);
+		$this->assertEquals(6, $this->cache->get('folder/subfolder2')['size']);
+	}
+
+	public function testReuseExisting(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+		$oldData = $this->cache->get('');
+		$this->storage->unlink('folder/bar.txt');
+		$this->cache->put('folder', ['mtime' => $this->storage->filemtime('folder'), 'storage_mtime' => $this->storage->filemtime('folder')]);
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_SIZE);
+		$newData = $this->cache->get('');
+		$this->assertIsString($oldData['etag']);
+		$this->assertIsString($newData['etag']);
+		$this->assertNotSame($oldData['etag'], $newData['etag']);
+		$this->assertEquals($oldData['size'], $newData['size']);
+
+		$oldData = $newData;
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
+		$newData = $this->cache->get('');
+		$this->assertSame($oldData['etag'], $newData['etag']);
+		$this->assertEquals(-1, $newData['size']);
+
+		$this->scanner->scan('', IScanner::SCAN_RECURSIVE);
+		$oldData = $this->cache->get('');
+		$this->assertNotEquals(-1, $oldData['size']);
+		$this->scanner->scanFile('', IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
+		$newData = $this->cache->get('');
+		$this->assertSame($oldData['etag'], $newData['etag']);
+		$this->assertEquals($oldData['size'], $newData['size']);
+
+		$this->scanner->scan('', IScanner::SCAN_RECURSIVE, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
+		$newData = $this->cache->get('');
+		$this->assertSame($oldData['etag'], $newData['etag']);
+		$this->assertEquals($oldData['size'], $newData['size']);
+
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
+		$newData = $this->cache->get('');
+		$this->assertSame($oldData['etag'], $newData['etag']);
+		$this->assertEquals($oldData['size'], $newData['size']);
+	}
+
+	public function testRemovedFile(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+		$this->assertTrue($this->cache->inCache('foo.txt'));
+		$this->storage->unlink('foo.txt');
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW);
+		$this->assertFalse($this->cache->inCache('foo.txt'));
+	}
+
+	public function testRemovedFolder(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$this->storage->rmdir('/folder');
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW);
+		$this->assertFalse($this->cache->inCache('folder'));
+		$this->assertFalse($this->cache->inCache('folder/bar.txt'));
+	}
+
+	public function testScanRemovedFile(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$this->storage->unlink('folder/bar.txt');
+		$this->scanner->scanFile('folder/bar.txt');
+		$this->assertFalse($this->cache->inCache('folder/bar.txt'));
+	}
+
+	public function testETagRecreation(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('folder/bar.txt');
+
+		// manipulate etag to simulate an empty etag
+		$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
+		/** @var CacheEntry $data0 */
+		$data0 = $this->cache->get('folder/bar.txt');
+		$this->assertIsString($data0['etag']);
+		$data1 = $this->cache->get('folder');
+		$this->assertIsString($data1['etag']);
+		$data2 = $this->cache->get('');
+		$this->assertIsString($data2['etag']);
+		$data0['etag'] = '';
+		$this->cache->put('folder/bar.txt', $data0->getData());
+
+		// rescan
+		$this->scanner->scan('folder/bar.txt', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
+
+		// verify cache content
+		$newData0 = $this->cache->get('folder/bar.txt');
+		$this->assertIsString($newData0['etag']);
+		$this->assertNotEmpty($newData0['etag']);
+	}
+
+	public function testRepairParent(): void {
+		$this->fillTestFolders();
+		$this->scanner->scan('');
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$oldFolderId = $this->cache->getId('folder');
+
+		// delete the folder without removing the children
+		$query = Server::get(IDBConnection::class)->getQueryBuilder();
+		$query->delete('filecache')
+			->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId)));
+		$query->execute();
+
+		$cachedData = $this->cache->get('folder/bar.txt');
+		$this->assertEquals($oldFolderId, $cachedData['parent']);
+		$this->assertFalse($this->cache->inCache('folder'));
+
+		$this->scanner->scan('');
+
+		$this->assertTrue($this->cache->inCache('folder'));
+		$newFolderId = $this->cache->getId('folder');
+		$this->assertNotEquals($oldFolderId, $newFolderId);
+
+		$cachedData = $this->cache->get('folder/bar.txt');
+		$this->assertEquals($newFolderId, $cachedData['parent']);
+	}
+
+	public function testRepairParentShallow(): void {
+		$this->fillTestFolders();
+		$this->scanner->scan('');
+		$this->assertTrue($this->cache->inCache('folder/bar.txt'));
+		$oldFolderId = $this->cache->getId('folder');
+
+		// delete the folder without removing the children
+		$query = Server::get(IDBConnection::class)->getQueryBuilder();
+		$query->delete('filecache')
+			->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId)));
+		$query->execute();
+
+		$cachedData = $this->cache->get('folder/bar.txt');
+		$this->assertEquals($oldFolderId, $cachedData['parent']);
+		$this->assertFalse($this->cache->inCache('folder'));
+
+		$this->scanner->scan('folder', IScanner::SCAN_SHALLOW);
+
+		$this->assertTrue($this->cache->inCache('folder'));
+		$newFolderId = $this->cache->getId('folder');
+		$this->assertNotEquals($oldFolderId, $newFolderId);
+
+		$cachedData = $this->cache->get('folder/bar.txt');
+		$this->assertEquals($newFolderId, $cachedData['parent']);
+	}
+
+	/**
+	 *
+	 * @param string $path
+	 * @param bool $expected
+	 */
+	#[\PHPUnit\Framework\Attributes\DataProvider('dataTestIsPartialFile')]
+	public function testIsPartialFile($path, $expected): void {
+		$this->assertSame($expected,
+			$this->scanner->isPartialFile($path)
+		);
+	}
+
+	public static function dataTestIsPartialFile(): array {
+		return [
+			['foo.txt.part', true],
+			['/sub/folder/foo.txt.part', true],
+			['/sub/folder.part/foo.txt', true],
+			['foo.txt', false],
+			['/sub/folder/foo.txt', false],
+		];
+	}
+
+	public function testNoETagUnscannedFolder(): void {
+		$this->fillTestFolders();
+
+		$this->scanner->scan('');
+
+		$oldFolderEntry = $this->cache->get('folder');
+		// create a new file in a folder by keeping the mtime unchanged, but mark the folder as unscanned
+		$this->storage->file_put_contents('folder/new.txt', 'foo');
+		$this->storage->touch('folder', $oldFolderEntry->getMTime());
+		$this->cache->update($oldFolderEntry->getId(), ['size' => -1]);
+
+		$this->scanner->scan('');
+
+		$this->cache->inCache('folder/new.txt');
+
+		$newFolderEntry = $this->cache->get('folder');
+		$this->assertNotEquals($newFolderEntry->getEtag(), $oldFolderEntry->getEtag());
+	}
+
+	public function testNoETagUnscannedSubFolder(): void {
+		$this->fillTestFolders();
+		$this->storage->mkdir('folder/sub');
+
+		$this->scanner->scan('');
+
+		$oldFolderEntry1 = $this->cache->get('folder');
+		$oldFolderEntry2 = $this->cache->get('folder/sub');
+		// create a new file in a folder by keeping the mtime unchanged, but mark the folder as unscanned
+		$this->storage->file_put_contents('folder/sub/new.txt', 'foo');
+		$this->storage->touch('folder/sub', $oldFolderEntry1->getMTime());
+
+		// we only mark the direct parent as unscanned, which is the current "notify" behavior
+		$this->cache->update($oldFolderEntry2->getId(), ['size' => -1]);
+
+		$this->scanner->scan('');
+
+		$this->cache->inCache('folder/new.txt');
+
+		$newFolderEntry1 = $this->cache->get('folder');
+		$this->assertNotEquals($newFolderEntry1->getEtag(), $oldFolderEntry1->getEtag());
+		$newFolderEntry2 = $this->cache->get('folder/sub');
+		$this->assertNotEquals($newFolderEntry2->getEtag(), $oldFolderEntry2->getEtag());
+	}
+}