mirrors
/
nextcloud
镜像来自 https://github.com/nextcloud/server.git
關注 1
收藏 0
複製 0
程式碼 問題 0 版本發佈 993 Wiki 活動
瀏覽代碼

Move files/ajax/scan.php to background job 

The background job will now be executed in chunks of 500 users all 10 minutes.
tags/v9.0beta1
Lukas Reschke 8 年之前
父節點
d305412a35
當前提交
391bc49dab
共有 9 個檔案被更改，包括 280 行新增 和 146 行删除
分割檢視 顯示文件統計
  1. 0
    94
      apps/files/ajax/scan.php
  2. 1
    1
      apps/files/appinfo/info.xml
  3. 26
    0
      apps/files/appinfo/install.php
  4. 0
    2
      apps/files/appinfo/routes.php
  5. 5
    0
      apps/files/appinfo/update.php
  6. 0
    48
      apps/files/js/files.js
  7. 114
    0
      apps/files/lib/backgroundjob/scanfiles.php
  8. 134
    0
      apps/files/tests/backgroundjob/ScanFilesTest.php
  9. 0
    1
      lib/public/backgroundjob/ijoblist.php

+ 0
- 94
apps/files/ajax/scan.php 查看文件

@@ -1,94 +0,0 @@
<?php
/**
* @author Bart Visscher <bartv@thisnet.nl>
* @author Jörn Friedrich Dreyer <jfd@butonic.de>
* @author Lukas Reschke <lukas@owncloud.com>
* @author Robin Appelman <icewind@owncloud.com>
* @author Vincent Petry <pvince81@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
set_time_limit(0); //scanning can take ages

\OCP\JSON::checkLoggedIn();
\OCP\JSON::callCheck();

\OC::$server->getSession()->close();

$force = (isset($_GET['force']) and ($_GET['force'] === 'true'));
$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : '';
if (isset($_GET['users'])) {
\OCP\JSON::checkAdminUser();
if ($_GET['users'] === 'all') {
$users = OC_User::getUsers();
} else {
$users = json_decode($_GET['users']);
}
} else {
$users = array(OC_User::getUser());
}

$eventSource = \OC::$server->createEventSource();
$listener = new ScanListener($eventSource);

foreach ($users as $user) {
$eventSource->send('user', $user);
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
try {
if ($force) {
$scanner->scan($dir);
} else {
$scanner->backgroundScan($dir);
}
} catch (\Exception $e) {
$eventSource->send('error', get_class($e) . ': ' . $e->getMessage());
}
}

$eventSource->send('done', $listener->getCount());
$eventSource->close();

class ScanListener {

private $fileCount = 0;
private $lastCount = 0;

/**
* @var \OCP\IEventSource event source to pass events to
*/
private $eventSource;

/**
* @param \OCP\IEventSource $eventSource
*/
public function __construct($eventSource) {
$this->eventSource = $eventSource;
}

public function file() {
$this->fileCount++;
if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files
$this->lastCount = $this->fileCount;
$this->eventSource->send('count', $this->fileCount);
}
}

public function getCount() {
return $this->fileCount;
}
}

+ 1
- 1
apps/files/appinfo/info.xml 查看文件

@@ -8,7 +8,7 @@
<shipped>true</shipped>
<standalone/>
<default_enable/>
<version>1.4.0</version>
<version>1.4.1</version>
<types>
<filesystem/>
</types>

+ 26
- 0
apps/files/appinfo/install.php 查看文件

@@ -0,0 +1,26 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/

// Cron job for scanning user storages
$jobList = \OC::$server->getJobList();
$job = 'OCA\Files\BackgroundJob\ScanFiles';
\OC::$server->getJobList()->add($job);


+ 0
- 2
apps/files/appinfo/routes.php 查看文件

@@ -75,8 +75,6 @@ $this->create('files_ajax_newfolder', 'ajax/newfolder.php')
->actionInclude('files/ajax/newfolder.php');
$this->create('files_ajax_rename', 'ajax/rename.php')
->actionInclude('files/ajax/rename.php');
$this->create('files_ajax_scan', 'ajax/scan.php')
->actionInclude('files/ajax/scan.php');
$this->create('files_ajax_upload', 'ajax/upload.php')
->actionInclude('files/ajax/upload.php');


+ 5
- 0
apps/files/appinfo/update.php 查看文件

@@ -96,6 +96,11 @@ if ($installedVersion === '1.1.9' && (
}
}

// Add cron job for scanning user storages
$jobList = \OC::$server->getJobList();
$job = 'OCA\Files\BackgroundJob\ScanFiles';
\OC::$server->getJobList()->add($job);

/**
* migrate old constant DEBUG to new config value 'debug'
*

+ 0
- 48
apps/files/js/files.js 查看文件

@@ -243,9 +243,6 @@
e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone
});

//do a background scan if needed
scanFiles();

// display storage warnings
setTimeout(Files.displayStorageWarnings, 100);

@@ -323,51 +320,6 @@
OCA.Files.Files = Files;
})();

function scanFiles(force, dir, users) {
if (!OC.currentUser) {
return;
}

if (!dir) {
dir = '';
}
force = !!force; //cast to bool
scanFiles.scanning = true;
var scannerEventSource;
if (users) {
var usersString;
if (users === 'all') {
usersString = users;
} else {
usersString = JSON.stringify(users);
}
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString});
} else {
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir});
}
scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource);
scannerEventSource.listen('count',function(count) {
console.log(count + ' files scanned');
});
scannerEventSource.listen('folder',function(path) {
console.log('now scanning ' + path);
});
scannerEventSource.listen('error',function(message) {
console.error('Scanner error: ', message);
});
scannerEventSource.listen('done',function(count) {
scanFiles.scanning=false;
console.log('done after ' + count + ' files');
if (OCA.Files.App) {
OCA.Files.App.fileList.updateStorageStatistics(true);
}
});
scannerEventSource.listen('user',function(user) {
console.log('scanning files for ' + user);
});
}
scanFiles.scanning=false;

// TODO: move to FileList
var createDragShadow = function(event) {
// FIXME: inject file list instance somehow

+ 114
- 0
apps/files/lib/backgroundjob/scanfiles.php 查看文件

@@ -0,0 +1,114 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/

namespace OCA\Files\BackgroundJob;

use OC\Files\Utils\Scanner;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\IUser;
use OCP\IUserManager;

/**
* Class ScanFiles is a background job used to run the file scanner over the user
* accounts to ensure integrity of the file cache.
*
* @package OCA\Files\BackgroundJob
*/
class ScanFiles extends \OC\BackgroundJob\TimedJob {
/** @var IConfig */
private $config;
/** @var IUserManager */
private $userManager;
/** @var IDBConnection */
private $dbConnection;
/** @var ILogger */
private $logger;
/** Amount of users that should get scanned per execution */
const USERS_PER_SESSION = 500;

/**
* @param IConfig|null $config
* @param IUserManager|null $userManager
* @param IDBConnection|null $dbConnection
* @param ILogger|null $logger
*/
public function __construct(IConfig $config = null,
IUserManager $userManager = null,
IDBConnection $dbConnection = null,
ILogger $logger = null) {
// Run once per 10 minutes
$this->setInterval(60 * 10);

if (is_null($userManager) || is_null($config)) {
$this->fixDIForJobs();
} else {
$this->config = $config;
$this->userManager = $userManager;
$this->logger = $logger;
}
}

protected function fixDIForJobs() {
$this->config = \OC::$server->getConfig();
$this->userManager = \OC::$server->getUserManager();
$this->logger = \OC::$server->getLogger();
}

/**
* @param IUser $user
*/
protected function runScanner(IUser $user) {
try {
$scanner = new Scanner(
$user->getUID(),
$this->dbConnection,
$this->logger
);
$scanner->backgroundScan('');
} catch (\Exception $e) {
$this->logger->logException($e, ['app' => 'files']);
}
\OC_Util::tearDownFS();
}

/**
* @param $argument
* @throws \Exception
*/
protected function run($argument) {
$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
if (!count($users)) {
// No users found, reset offset and retry
$offset = 0;
$users = $this->userManager->search('', self::USERS_PER_SESSION);
}

$offset += self::USERS_PER_SESSION;
$this->config->setAppValue('files', 'cronjob_scan_files', $offset);

foreach ($users as $user) {
$this->runScanner($user);
}
}
}

+ 134
- 0
apps/files/tests/backgroundjob/ScanFilesTest.php 查看文件

@@ -0,0 +1,134 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OCA\Files\Tests\BackgroundJob;

use Test\TestCase;
use OCP\IConfig;
use OCP\IUserManager;
use OCA\Files\BackgroundJob\ScanFiles;
use OCP\ILogger;

/**
* Class ScanFilesTest
*
* @package OCA\Files\Tests\BackgroundJob
*/
class ScanFilesTest extends TestCase {
/** @var IConfig */
private $config;
/** @var IUserManager */
private $userManager;
/** @var ScanFiles */
private $scanFiles;

public function setUp() {
parent::setUp();

$this->config = $this->getMock('\OCP\IConfig');
$this->userManager = $this->getMock('\OCP\IUserManager');

$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles')
->setConstructorArgs([
$this->config,
$this->userManager,
])
->setMethods(['runScanner'])
->getMock();
}

public function testRunWithoutUsers() {
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([]));
$this->userManager
->expects($this->at(1))
->method('search')
->with('', 500)
->will($this->returnValue([]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 500);

$this->invokePrivate($this->scanFiles, 'run', [[]]);
}

public function testRunWithUsers() {
$fakeUser = $this->getMock('\OCP\IUser');
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([
$fakeUser
]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 550);
$this->scanFiles
->expects($this->once())
->method('runScanner')
->with($fakeUser);

$this->invokePrivate($this->scanFiles, 'run', [[]]);
}

public function testRunWithUsersAndOffsetAtEndOfUserList() {
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([]));
$this->userManager
->expects($this->at(1))
->method('search')
->with('', 500)
->will($this->returnValue([]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 500);
$this->scanFiles
->expects($this->never())
->method('runScanner');

$this->invokePrivate($this->scanFiles, 'run', [[]]);
}

}

+ 0
- 1
lib/public/backgroundjob/ijoblist.php 查看文件

@@ -36,7 +36,6 @@ interface IJobList {
*
* @param \OCP\BackgroundJob\IJob|string $job
* @param mixed $argument The argument to be passed to $job->run() when the job is exectured
* @param string $job
* @return void
* @since 7.0.0
*/

Write Preview
Loading…
取消
儲存