Move files/ajax/scan.php to background jobtags/v9.0beta1
<?php | |||||
/** | |||||
* @author Bart Visscher <bartv@thisnet.nl> | |||||
* @author Jörn Friedrich Dreyer <jfd@butonic.de> | |||||
* @author Lukas Reschke <lukas@owncloud.com> | |||||
* @author Robin Appelman <icewind@owncloud.com> | |||||
* @author Vincent Petry <pvince81@owncloud.com> | |||||
* | |||||
* @copyright Copyright (c) 2015, ownCloud, Inc. | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
set_time_limit(0); //scanning can take ages | |||||
\OCP\JSON::checkLoggedIn(); | |||||
\OCP\JSON::callCheck(); | |||||
\OC::$server->getSession()->close(); | |||||
$force = (isset($_GET['force']) and ($_GET['force'] === 'true')); | |||||
$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : ''; | |||||
if (isset($_GET['users'])) { | |||||
\OCP\JSON::checkAdminUser(); | |||||
if ($_GET['users'] === 'all') { | |||||
$users = OC_User::getUsers(); | |||||
} else { | |||||
$users = json_decode($_GET['users']); | |||||
} | |||||
} else { | |||||
$users = array(OC_User::getUser()); | |||||
} | |||||
$eventSource = \OC::$server->createEventSource(); | |||||
$listener = new ScanListener($eventSource); | |||||
foreach ($users as $user) { | |||||
$eventSource->send('user', $user); | |||||
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger()); | |||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file')); | |||||
try { | |||||
if ($force) { | |||||
$scanner->scan($dir); | |||||
} else { | |||||
$scanner->backgroundScan($dir); | |||||
} | |||||
} catch (\Exception $e) { | |||||
$eventSource->send('error', get_class($e) . ': ' . $e->getMessage()); | |||||
} | |||||
} | |||||
$eventSource->send('done', $listener->getCount()); | |||||
$eventSource->close(); | |||||
class ScanListener { | |||||
private $fileCount = 0; | |||||
private $lastCount = 0; | |||||
/** | |||||
* @var \OCP\IEventSource event source to pass events to | |||||
*/ | |||||
private $eventSource; | |||||
/** | |||||
* @param \OCP\IEventSource $eventSource | |||||
*/ | |||||
public function __construct($eventSource) { | |||||
$this->eventSource = $eventSource; | |||||
} | |||||
public function file() { | |||||
$this->fileCount++; | |||||
if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files | |||||
$this->lastCount = $this->fileCount; | |||||
$this->eventSource->send('count', $this->fileCount); | |||||
} | |||||
} | |||||
public function getCount() { | |||||
return $this->fileCount; | |||||
} | |||||
} |
<shipped>true</shipped> | <shipped>true</shipped> | ||||
<standalone/> | <standalone/> | ||||
<default_enable/> | <default_enable/> | ||||
<version>1.4.0</version> | |||||
<version>1.4.1</version> | |||||
<types> | <types> | ||||
<filesystem/> | <filesystem/> | ||||
</types> | </types> |
<?php | |||||
/** | |||||
* @author Lukas Reschke <lukas@owncloud.com> | |||||
* | |||||
* @copyright Copyright (c) 2015, ownCloud, Inc. | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
// Cron job for scanning user storages | |||||
$jobList = \OC::$server->getJobList(); | |||||
$job = 'OCA\Files\BackgroundJob\ScanFiles'; | |||||
\OC::$server->getJobList()->add($job); | |||||
->actionInclude('files/ajax/newfolder.php'); | ->actionInclude('files/ajax/newfolder.php'); | ||||
$this->create('files_ajax_rename', 'ajax/rename.php') | $this->create('files_ajax_rename', 'ajax/rename.php') | ||||
->actionInclude('files/ajax/rename.php'); | ->actionInclude('files/ajax/rename.php'); | ||||
$this->create('files_ajax_scan', 'ajax/scan.php') | |||||
->actionInclude('files/ajax/scan.php'); | |||||
$this->create('files_ajax_upload', 'ajax/upload.php') | $this->create('files_ajax_upload', 'ajax/upload.php') | ||||
->actionInclude('files/ajax/upload.php'); | ->actionInclude('files/ajax/upload.php'); | ||||
} | } | ||||
} | } | ||||
// Add cron job for scanning user storages | |||||
$jobList = \OC::$server->getJobList(); | |||||
$job = 'OCA\Files\BackgroundJob\ScanFiles'; | |||||
\OC::$server->getJobList()->add($job); | |||||
/** | /** | ||||
* migrate old constant DEBUG to new config value 'debug' | * migrate old constant DEBUG to new config value 'debug' | ||||
* | * |
e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone | e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone | ||||
}); | }); | ||||
//do a background scan if needed | |||||
scanFiles(); | |||||
// display storage warnings | // display storage warnings | ||||
setTimeout(Files.displayStorageWarnings, 100); | setTimeout(Files.displayStorageWarnings, 100); | ||||
OCA.Files.Files = Files; | OCA.Files.Files = Files; | ||||
})(); | })(); | ||||
function scanFiles(force, dir, users) { | |||||
if (!OC.currentUser) { | |||||
return; | |||||
} | |||||
if (!dir) { | |||||
dir = ''; | |||||
} | |||||
force = !!force; //cast to bool | |||||
scanFiles.scanning = true; | |||||
var scannerEventSource; | |||||
if (users) { | |||||
var usersString; | |||||
if (users === 'all') { | |||||
usersString = users; | |||||
} else { | |||||
usersString = JSON.stringify(users); | |||||
} | |||||
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString}); | |||||
} else { | |||||
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir}); | |||||
} | |||||
scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource); | |||||
scannerEventSource.listen('count',function(count) { | |||||
console.log(count + ' files scanned'); | |||||
}); | |||||
scannerEventSource.listen('folder',function(path) { | |||||
console.log('now scanning ' + path); | |||||
}); | |||||
scannerEventSource.listen('error',function(message) { | |||||
console.error('Scanner error: ', message); | |||||
}); | |||||
scannerEventSource.listen('done',function(count) { | |||||
scanFiles.scanning=false; | |||||
console.log('done after ' + count + ' files'); | |||||
if (OCA.Files.App) { | |||||
OCA.Files.App.fileList.updateStorageStatistics(true); | |||||
} | |||||
}); | |||||
scannerEventSource.listen('user',function(user) { | |||||
console.log('scanning files for ' + user); | |||||
}); | |||||
} | |||||
scanFiles.scanning=false; | |||||
// TODO: move to FileList | // TODO: move to FileList | ||||
var createDragShadow = function(event) { | var createDragShadow = function(event) { | ||||
// FIXME: inject file list instance somehow | // FIXME: inject file list instance somehow |
<?php | |||||
/** | |||||
* @author Lukas Reschke <lukas@owncloud.com> | |||||
* | |||||
* @copyright Copyright (c) 2015, ownCloud, Inc. | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
namespace OCA\Files\BackgroundJob; | |||||
use OC\Files\Utils\Scanner; | |||||
use OCP\IConfig; | |||||
use OCP\IDBConnection; | |||||
use OCP\ILogger; | |||||
use OCP\IUser; | |||||
use OCP\IUserManager; | |||||
/** | |||||
* Class ScanFiles is a background job used to run the file scanner over the user | |||||
* accounts to ensure integrity of the file cache. | |||||
* | |||||
* @package OCA\Files\BackgroundJob | |||||
*/ | |||||
class ScanFiles extends \OC\BackgroundJob\TimedJob { | |||||
/** @var IConfig */ | |||||
private $config; | |||||
/** @var IUserManager */ | |||||
private $userManager; | |||||
/** @var IDBConnection */ | |||||
private $dbConnection; | |||||
/** @var ILogger */ | |||||
private $logger; | |||||
/** Amount of users that should get scanned per execution */ | |||||
const USERS_PER_SESSION = 500; | |||||
/** | |||||
* @param IConfig|null $config | |||||
* @param IUserManager|null $userManager | |||||
* @param IDBConnection|null $dbConnection | |||||
* @param ILogger|null $logger | |||||
*/ | |||||
public function __construct(IConfig $config = null, | |||||
IUserManager $userManager = null, | |||||
IDBConnection $dbConnection = null, | |||||
ILogger $logger = null) { | |||||
// Run once per 10 minutes | |||||
$this->setInterval(60 * 10); | |||||
if (is_null($userManager) || is_null($config)) { | |||||
$this->fixDIForJobs(); | |||||
} else { | |||||
$this->config = $config; | |||||
$this->userManager = $userManager; | |||||
$this->logger = $logger; | |||||
} | |||||
} | |||||
protected function fixDIForJobs() { | |||||
$this->config = \OC::$server->getConfig(); | |||||
$this->userManager = \OC::$server->getUserManager(); | |||||
$this->logger = \OC::$server->getLogger(); | |||||
} | |||||
/** | |||||
* @param IUser $user | |||||
*/ | |||||
protected function runScanner(IUser $user) { | |||||
try { | |||||
$scanner = new Scanner( | |||||
$user->getUID(), | |||||
$this->dbConnection, | |||||
$this->logger | |||||
); | |||||
$scanner->backgroundScan(''); | |||||
} catch (\Exception $e) { | |||||
$this->logger->logException($e, ['app' => 'files']); | |||||
} | |||||
\OC_Util::tearDownFS(); | |||||
} | |||||
/** | |||||
* @param $argument | |||||
* @throws \Exception | |||||
*/ | |||||
protected function run($argument) { | |||||
$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0); | |||||
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset); | |||||
if (!count($users)) { | |||||
// No users found, reset offset and retry | |||||
$offset = 0; | |||||
$users = $this->userManager->search('', self::USERS_PER_SESSION); | |||||
} | |||||
$offset += self::USERS_PER_SESSION; | |||||
$this->config->setAppValue('files', 'cronjob_scan_files', $offset); | |||||
foreach ($users as $user) { | |||||
$this->runScanner($user); | |||||
} | |||||
} | |||||
} |
<?php | |||||
/** | |||||
* @author Lukas Reschke <lukas@owncloud.com> | |||||
* | |||||
* @copyright Copyright (c) 2015, ownCloud, Inc. | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
namespace OCA\Files\Tests\BackgroundJob; | |||||
use Test\TestCase; | |||||
use OCP\IConfig; | |||||
use OCP\IUserManager; | |||||
use OCA\Files\BackgroundJob\ScanFiles; | |||||
use OCP\ILogger; | |||||
/** | |||||
* Class ScanFilesTest | |||||
* | |||||
* @package OCA\Files\Tests\BackgroundJob | |||||
*/ | |||||
class ScanFilesTest extends TestCase { | |||||
/** @var IConfig */ | |||||
private $config; | |||||
/** @var IUserManager */ | |||||
private $userManager; | |||||
/** @var ScanFiles */ | |||||
private $scanFiles; | |||||
public function setUp() { | |||||
parent::setUp(); | |||||
$this->config = $this->getMock('\OCP\IConfig'); | |||||
$this->userManager = $this->getMock('\OCP\IUserManager'); | |||||
$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles') | |||||
->setConstructorArgs([ | |||||
$this->config, | |||||
$this->userManager, | |||||
]) | |||||
->setMethods(['runScanner']) | |||||
->getMock(); | |||||
} | |||||
public function testRunWithoutUsers() { | |||||
$this->config | |||||
->expects($this->at(0)) | |||||
->method('getAppValue') | |||||
->with('files', 'cronjob_scan_files', 0) | |||||
->will($this->returnValue(50)); | |||||
$this->userManager | |||||
->expects($this->at(0)) | |||||
->method('search') | |||||
->with('', 500, 50) | |||||
->will($this->returnValue([])); | |||||
$this->userManager | |||||
->expects($this->at(1)) | |||||
->method('search') | |||||
->with('', 500) | |||||
->will($this->returnValue([])); | |||||
$this->config | |||||
->expects($this->at(1)) | |||||
->method('setAppValue') | |||||
->with('files', 'cronjob_scan_files', 500); | |||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); | |||||
} | |||||
public function testRunWithUsers() { | |||||
$fakeUser = $this->getMock('\OCP\IUser'); | |||||
$this->config | |||||
->expects($this->at(0)) | |||||
->method('getAppValue') | |||||
->with('files', 'cronjob_scan_files', 0) | |||||
->will($this->returnValue(50)); | |||||
$this->userManager | |||||
->expects($this->at(0)) | |||||
->method('search') | |||||
->with('', 500, 50) | |||||
->will($this->returnValue([ | |||||
$fakeUser | |||||
])); | |||||
$this->config | |||||
->expects($this->at(1)) | |||||
->method('setAppValue') | |||||
->with('files', 'cronjob_scan_files', 550); | |||||
$this->scanFiles | |||||
->expects($this->once()) | |||||
->method('runScanner') | |||||
->with($fakeUser); | |||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); | |||||
} | |||||
public function testRunWithUsersAndOffsetAtEndOfUserList() { | |||||
$this->config | |||||
->expects($this->at(0)) | |||||
->method('getAppValue') | |||||
->with('files', 'cronjob_scan_files', 0) | |||||
->will($this->returnValue(50)); | |||||
$this->userManager | |||||
->expects($this->at(0)) | |||||
->method('search') | |||||
->with('', 500, 50) | |||||
->will($this->returnValue([])); | |||||
$this->userManager | |||||
->expects($this->at(1)) | |||||
->method('search') | |||||
->with('', 500) | |||||
->will($this->returnValue([])); | |||||
$this->config | |||||
->expects($this->at(1)) | |||||
->method('setAppValue') | |||||
->with('files', 'cronjob_scan_files', 500); | |||||
$this->scanFiles | |||||
->expects($this->never()) | |||||
->method('runScanner'); | |||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); | |||||
} | |||||
} |
* | * | ||||
* @param \OCP\BackgroundJob\IJob|string $job | * @param \OCP\BackgroundJob\IJob|string $job | ||||
* @param mixed $argument The argument to be passed to $job->run() when the job is exectured | * @param mixed $argument The argument to be passed to $job->run() when the job is exectured | ||||
* @param string $job | |||||
* @return void | * @return void | ||||
* @since 7.0.0 | * @since 7.0.0 | ||||
*/ | */ |