Move files/ajax/scan.php to background job

The background job will now be executed in chunks of 500 users all 10 minutes.

8 files changed, 280 insertions, 145 deletions

diff --git a/apps/files/ajax/scan.php b/apps/files/ajax/scan.php
deleted file mode 100644
index 7710a28a8ca..00000000000
--- a/apps/files/ajax/scan.php
+++ /dev/null
@@ -1,94 +0,0 @@
-<?php
-/**
- * @author Bart Visscher <bartv@thisnet.nl>
- * @author Jörn Friedrich Dreyer <jfd@butonic.de>
- * @author Lukas Reschke <lukas@owncloud.com>
- * @author Robin Appelman <icewind@owncloud.com>
- * @author Vincent Petry <pvince81@owncloud.com>
- *
- * @copyright Copyright (c) 2015, ownCloud, Inc.
- * @license AGPL-3.0
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License, version 3,
- * along with this program.  If not, see <http://www.gnu.org/licenses/>
- *
- */
-set_time_limit(0); //scanning can take ages
-
-\OCP\JSON::checkLoggedIn();
-\OCP\JSON::callCheck();
-
-\OC::$server->getSession()->close();
-
-$force = (isset($_GET['force']) and ($_GET['force'] === 'true'));
-$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : '';
-if (isset($_GET['users'])) {
-	\OCP\JSON::checkAdminUser();
-	if ($_GET['users'] === 'all') {
-		$users = OC_User::getUsers();
-	} else {
-		$users = json_decode($_GET['users']);
-	}
-} else {
-	$users = array(OC_User::getUser());
-}
-
-$eventSource = \OC::$server->createEventSource();
-$listener = new ScanListener($eventSource);
-
-foreach ($users as $user) {
-	$eventSource->send('user', $user);
-	$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
-	$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
-	try {
-		if ($force) {
-			$scanner->scan($dir);
-		} else {
-			$scanner->backgroundScan($dir);
-		}
-	} catch (\Exception $e) {
-		$eventSource->send('error', get_class($e) . ': ' . $e->getMessage());
-	}
-}
-
-$eventSource->send('done', $listener->getCount());
-$eventSource->close();
-
-class ScanListener {
-
-	private $fileCount = 0;
-	private $lastCount = 0;
-
-	/**
-	 * @var \OCP\IEventSource event source to pass events to
-	 */
-	private $eventSource;
-
-	/**
-	 * @param \OCP\IEventSource $eventSource
-	 */
-	public function __construct($eventSource) {
-		$this->eventSource = $eventSource;
-	}
-
-	public function file() {
-		$this->fileCount++;
-		if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files
-			$this->lastCount = $this->fileCount;
-			$this->eventSource->send('count', $this->fileCount);
-		}
-	}
-
-	public function getCount() {
-		return $this->fileCount;
-	}
-}
diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml
index 4ab226f3968..df12b87397d 100644
--- a/apps/files/appinfo/info.xml
+++ b/apps/files/appinfo/info.xml
@@ -8,7 +8,7 @@
 	<shipped>true</shipped>
 	<standalone/>
 	<default_enable/>
-	<version>1.4.0</version>
+	<version>1.4.1</version>
 	<types>
 		<filesystem/>
 	</types>
diff --git a/apps/files/appinfo/install.php b/apps/files/appinfo/install.php
new file mode 100644
index 00000000000..b47bf6ac4b0
--- /dev/null
+++ b/apps/files/appinfo/install.php
@@ -0,0 +1,26 @@
+<?php
+/**
+ * @author Lukas Reschke <lukas@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+// Cron job for scanning user storages
+$jobList = \OC::$server->getJobList();
+$job = 'OCA\Files\BackgroundJob\ScanFiles';
+\OC::$server->getJobList()->add($job);
+
diff --git a/apps/files/appinfo/routes.php b/apps/files/appinfo/routes.php
index 2bb913c30a6..844b73b3c41 100644
--- a/apps/files/appinfo/routes.php
+++ b/apps/files/appinfo/routes.php
@@ -75,8 +75,6 @@ $this->create('files_ajax_newfolder', 'ajax/newfolder.php')
 	->actionInclude('files/ajax/newfolder.php');
 $this->create('files_ajax_rename', 'ajax/rename.php')
 	->actionInclude('files/ajax/rename.php');
-$this->create('files_ajax_scan', 'ajax/scan.php')
-	->actionInclude('files/ajax/scan.php');
 $this->create('files_ajax_upload', 'ajax/upload.php')
 	->actionInclude('files/ajax/upload.php');
 
diff --git a/apps/files/appinfo/update.php b/apps/files/appinfo/update.php
index 6084435fa5a..d181dff560b 100644
--- a/apps/files/appinfo/update.php
+++ b/apps/files/appinfo/update.php
@@ -96,6 +96,11 @@ if ($installedVersion === '1.1.9' && (
 	}
 }
 
+// Add cron job for scanning user storages
+$jobList = \OC::$server->getJobList();
+$job = 'OCA\Files\BackgroundJob\ScanFiles';
+\OC::$server->getJobList()->add($job);
+
 /**
  * migrate old constant DEBUG to new config value 'debug'
  *
diff --git a/apps/files/js/files.js b/apps/files/js/files.js
index e33b8354437..6bf4a4cfe5a 100644
--- a/apps/files/js/files.js
+++ b/apps/files/js/files.js
@@ -243,9 +243,6 @@
 					e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone
 				});
 
-			//do a background scan if needed
-			scanFiles();
-
 			// display storage warnings
 			setTimeout(Files.displayStorageWarnings, 100);
 
@@ -323,51 +320,6 @@
 	OCA.Files.Files = Files;
 })();
 
-function scanFiles(force, dir, users) {
-	if (!OC.currentUser) {
-		return;
-	}
-
-	if (!dir) {
-		dir = '';
-	}
-	force = !!force; //cast to bool
-	scanFiles.scanning = true;
-	var scannerEventSource;
-	if (users) {
-		var usersString;
-		if (users === 'all') {
-			usersString = users;
-		} else {
-			usersString = JSON.stringify(users);
-		}
-		scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString});
-	} else {
-		scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir});
-	}
-	scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource);
-	scannerEventSource.listen('count',function(count) {
-		console.log(count + ' files scanned');
-	});
-	scannerEventSource.listen('folder',function(path) {
-		console.log('now scanning ' + path);
-	});
-	scannerEventSource.listen('error',function(message) {
-		console.error('Scanner error: ', message);
-	});
-	scannerEventSource.listen('done',function(count) {
-		scanFiles.scanning=false;
-		console.log('done after ' + count + ' files');
-		if (OCA.Files.App) {
-			OCA.Files.App.fileList.updateStorageStatistics(true);
-		}
-	});
-	scannerEventSource.listen('user',function(user) {
-		console.log('scanning files for ' + user);
-	});
-}
-scanFiles.scanning=false;
-
 // TODO: move to FileList
 var createDragShadow = function(event) {
 	// FIXME: inject file list instance somehow
diff --git a/apps/files/lib/backgroundjob/scanfiles.php b/apps/files/lib/backgroundjob/scanfiles.php
new file mode 100644
index 00000000000..7371429a268
--- /dev/null
+++ b/apps/files/lib/backgroundjob/scanfiles.php
@@ -0,0 +1,114 @@
+<?php
+/**
+ * @author Lukas Reschke <lukas@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\Files\BackgroundJob;
+
+use OC\Files\Utils\Scanner;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\ILogger;
+use OCP\IUser;
+use OCP\IUserManager;
+
+/**
+ * Class ScanFiles is a background job used to run the file scanner over the user
+ * accounts to ensure integrity of the file cache.
+ *
+ * @package OCA\Files\BackgroundJob
+ */
+class ScanFiles extends \OC\BackgroundJob\TimedJob {
+	/** @var IConfig */
+	private $config;
+	/** @var IUserManager */
+	private $userManager;
+	/** @var IDBConnection */
+	private $dbConnection;
+	/** @var ILogger */
+	private $logger;
+	/** Amount of users that should get scanned per execution */
+	const USERS_PER_SESSION = 500;
+
+	/**
+	 * @param IConfig|null $config
+	 * @param IUserManager|null $userManager
+	 * @param IDBConnection|null $dbConnection
+	 * @param ILogger|null $logger
+	 */
+	public function __construct(IConfig $config = null,
+								IUserManager $userManager = null,
+								IDBConnection $dbConnection = null,
+								ILogger $logger = null) {
+		// Run once per 10 minutes
+		$this->setInterval(60 * 10);
+
+		if (is_null($userManager) || is_null($config)) {
+			$this->fixDIForJobs();
+		} else {
+			$this->config = $config;
+			$this->userManager = $userManager;
+			$this->logger = $logger;
+		}
+	}
+
+	protected function fixDIForJobs() {
+		$this->config = \OC::$server->getConfig();
+		$this->userManager = \OC::$server->getUserManager();
+		$this->logger = \OC::$server->getLogger();
+	}
+
+	/**
+	 * @param IUser $user
+	 */
+	protected function runScanner(IUser $user) {
+		try {
+			$scanner = new Scanner(
+					$user->getUID(),
+					$this->dbConnection,
+					$this->logger
+			);
+			$scanner->backgroundScan('');
+		} catch (\Exception $e) {
+			$this->logger->logException($e, ['app' => 'files']);
+		}
+		\OC_Util::tearDownFS();
+	}
+
+	/**
+	 * @param $argument
+	 * @throws \Exception
+	 */
+	protected function run($argument) {
+		$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
+		$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
+		if (!count($users)) {
+			// No users found, reset offset and retry
+			$offset = 0;
+			$users = $this->userManager->search('', self::USERS_PER_SESSION);
+		}
+
+		$offset += self::USERS_PER_SESSION;
+		$this->config->setAppValue('files', 'cronjob_scan_files', $offset);
+
+		foreach ($users as $user) {
+			$this->runScanner($user);
+		}
+	}
+}
diff --git a/apps/files/tests/backgroundjob/ScanFilesTest.php b/apps/files/tests/backgroundjob/ScanFilesTest.php
new file mode 100644
index 00000000000..907cad64f9e
--- /dev/null
+++ b/apps/files/tests/backgroundjob/ScanFilesTest.php
@@ -0,0 +1,134 @@
+<?php
+/**
+ * @author Lukas Reschke <lukas@owncloud.com>
+ *
+ * @copyright Copyright (c) 2015, ownCloud, Inc.
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+namespace OCA\Files\Tests\BackgroundJob;
+
+use Test\TestCase;
+use OCP\IConfig;
+use OCP\IUserManager;
+use OCA\Files\BackgroundJob\ScanFiles;
+use OCP\ILogger;
+
+/**
+ * Class ScanFilesTest
+ *
+ * @package OCA\Files\Tests\BackgroundJob
+ */
+class ScanFilesTest extends TestCase {
+	/** @var IConfig */
+	private $config;
+	/** @var IUserManager */
+	private $userManager;
+	/** @var ScanFiles */
+	private $scanFiles;
+
+	public function setUp() {
+		parent::setUp();
+
+		$this->config = $this->getMock('\OCP\IConfig');
+		$this->userManager = $this->getMock('\OCP\IUserManager');
+
+		$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles')
+				->setConstructorArgs([
+						$this->config,
+						$this->userManager,
+				])
+				->setMethods(['runScanner'])
+				->getMock();
+	}
+
+	public function testRunWithoutUsers() {
+		$this->config
+				->expects($this->at(0))
+				->method('getAppValue')
+				->with('files', 'cronjob_scan_files', 0)
+				->will($this->returnValue(50));
+		$this->userManager
+				->expects($this->at(0))
+				->method('search')
+				->with('', 500, 50)
+				->will($this->returnValue([]));
+		$this->userManager
+				->expects($this->at(1))
+				->method('search')
+				->with('', 500)
+				->will($this->returnValue([]));
+		$this->config
+				->expects($this->at(1))
+				->method('setAppValue')
+				->with('files', 'cronjob_scan_files', 500);
+
+		$this->invokePrivate($this->scanFiles, 'run', [[]]);
+	}
+
+	public function testRunWithUsers() {
+		$fakeUser = $this->getMock('\OCP\IUser');
+		$this->config
+				->expects($this->at(0))
+				->method('getAppValue')
+				->with('files', 'cronjob_scan_files', 0)
+				->will($this->returnValue(50));
+		$this->userManager
+				->expects($this->at(0))
+				->method('search')
+				->with('', 500, 50)
+				->will($this->returnValue([
+						$fakeUser
+				]));
+		$this->config
+				->expects($this->at(1))
+				->method('setAppValue')
+				->with('files', 'cronjob_scan_files', 550);
+		$this->scanFiles
+				->expects($this->once())
+				->method('runScanner')
+				->with($fakeUser);
+
+		$this->invokePrivate($this->scanFiles, 'run', [[]]);
+	}
+
+	public function testRunWithUsersAndOffsetAtEndOfUserList() {
+		$this->config
+				->expects($this->at(0))
+				->method('getAppValue')
+				->with('files', 'cronjob_scan_files', 0)
+				->will($this->returnValue(50));
+		$this->userManager
+				->expects($this->at(0))
+				->method('search')
+				->with('', 500, 50)
+				->will($this->returnValue([]));
+		$this->userManager
+				->expects($this->at(1))
+				->method('search')
+				->with('', 500)
+				->will($this->returnValue([]));
+		$this->config
+				->expects($this->at(1))
+				->method('setAppValue')
+				->with('files', 'cronjob_scan_files', 500);
+		$this->scanFiles
+				->expects($this->never())
+				->method('runScanner');
+
+		$this->invokePrivate($this->scanFiles, 'run', [[]]);
+	}
+
+}