diff options
-rw-r--r-- | apps/files_external/3rdparty/composer.json | 2 | ||||
-rw-r--r-- | apps/files_external/3rdparty/composer.lock | 14 | ||||
-rw-r--r-- | apps/files_external/3rdparty/composer/LICENSE | 2 | ||||
-rw-r--r-- | apps/files_external/3rdparty/composer/installed.json | 12 | ||||
-rw-r--r-- | apps/files_external/3rdparty/icewind/smb/src/Connection.php | 52 | ||||
-rw-r--r-- | apps/files_external/3rdparty/icewind/smb/src/IShare.php | 18 | ||||
-rw-r--r-- | apps/files_external/3rdparty/icewind/smb/src/NativeShare.php | 12 | ||||
-rw-r--r-- | apps/files_external/3rdparty/icewind/smb/src/Share.php | 51 | ||||
-rw-r--r-- | lib/private/App/InfoParser.php | 6 | ||||
-rw-r--r-- | lib/private/AppFramework/Http/Request.php | 3 | ||||
-rw-r--r-- | lib/private/Migration/BackgroundRepair.php | 116 | ||||
-rw-r--r-- | lib/private/app.php | 14 | ||||
-rw-r--r-- | tests/data/app/expected-info.json | 3 | ||||
-rw-r--r-- | tests/lib/migration/BackgroundRepairTest.php | 120 |
14 files changed, 383 insertions, 42 deletions
diff --git a/apps/files_external/3rdparty/composer.json b/apps/files_external/3rdparty/composer.json index b0267ba3438..72335c8d891 100644 --- a/apps/files_external/3rdparty/composer.json +++ b/apps/files_external/3rdparty/composer.json @@ -8,7 +8,7 @@ "classmap-authoritative": true }, "require": { - "icewind/smb": "1.0.8", + "icewind/smb": "1.1.0", "icewind/streams": "0.4" } } diff --git a/apps/files_external/3rdparty/composer.lock b/apps/files_external/3rdparty/composer.lock index 13931ad757d..7161ae19a2c 100644 --- a/apps/files_external/3rdparty/composer.lock +++ b/apps/files_external/3rdparty/composer.lock @@ -4,21 +4,21 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "1671a5ec7bef407432d42775f898dc34", - "content-hash": "9d995f0d55bee8a3b344a3c685e7b4a4", + "hash": "8de0823d3d0a167ee24450a111cb67b9", + "content-hash": "6733058865c1765823b31cfbb24552e1", "packages": [ { "name": "icewind/smb", - "version": "v1.0.8", + "version": "v1.1.0", "source": { "type": "git", "url": "https://github.com/icewind1991/SMB.git", - "reference": "764f3fc793a904eb937d619ad097fb076ff199cd" + "reference": "822f924967c68228555cea84ea44765f8e85c601" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/icewind1991/SMB/zipball/764f3fc793a904eb937d619ad097fb076ff199cd", - "reference": "764f3fc793a904eb937d619ad097fb076ff199cd", + "url": "https://api.github.com/repos/icewind1991/SMB/zipball/822f924967c68228555cea84ea44765f8e85c601", + "reference": "822f924967c68228555cea84ea44765f8e85c601", "shasum": "" }, "require": { @@ -47,7 +47,7 @@ } ], "description": "php wrapper for smbclient and libsmbclient-php", - "time": "2016-03-17 13:29:58" + "time": "2016-04-26 13:26:39" }, { "name": "icewind/streams", diff --git a/apps/files_external/3rdparty/composer/LICENSE b/apps/files_external/3rdparty/composer/LICENSE index c8d57af8b27..1a28124886d 100644 --- a/apps/files_external/3rdparty/composer/LICENSE +++ b/apps/files_external/3rdparty/composer/LICENSE @@ -1,5 +1,5 @@ -Copyright (c) 2015 Nils Adermann, Jordi Boggiano +Copyright (c) 2016 Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/apps/files_external/3rdparty/composer/installed.json b/apps/files_external/3rdparty/composer/installed.json index 48f8c555c34..31c602de3ca 100644 --- a/apps/files_external/3rdparty/composer/installed.json +++ b/apps/files_external/3rdparty/composer/installed.json @@ -44,17 +44,17 @@ }, { "name": "icewind/smb", - "version": "v1.0.8", - "version_normalized": "1.0.8.0", + "version": "v1.1.0", + "version_normalized": "1.1.0.0", "source": { "type": "git", "url": "https://github.com/icewind1991/SMB.git", - "reference": "764f3fc793a904eb937d619ad097fb076ff199cd" + "reference": "822f924967c68228555cea84ea44765f8e85c601" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/icewind1991/SMB/zipball/764f3fc793a904eb937d619ad097fb076ff199cd", - "reference": "764f3fc793a904eb937d619ad097fb076ff199cd", + "url": "https://api.github.com/repos/icewind1991/SMB/zipball/822f924967c68228555cea84ea44765f8e85c601", + "reference": "822f924967c68228555cea84ea44765f8e85c601", "shasum": "" }, "require": { @@ -65,7 +65,7 @@ "phpunit/phpunit": "^4.8", "satooshi/php-coveralls": "v1.0.0" }, - "time": "2016-03-17 13:29:58", + "time": "2016-04-26 13:26:39", "type": "library", "installation-source": "source", "autoload": { diff --git a/apps/files_external/3rdparty/icewind/smb/src/Connection.php b/apps/files_external/3rdparty/icewind/smb/src/Connection.php index f48dcb766e4..d24cdc1f6d0 100644 --- a/apps/files_external/3rdparty/icewind/smb/src/Connection.php +++ b/apps/files_external/3rdparty/icewind/smb/src/Connection.php @@ -15,6 +15,7 @@ use Icewind\SMB\Exception\NoLoginServerException; class Connection extends RawConnection { const DELIMITER = 'smb:'; + const DELIMITER_LENGTH = 4; /** * send input to smbclient @@ -28,6 +29,7 @@ class Connection extends RawConnection { /** * get all unprocessed output from smbclient until the next prompt * + * @param callable $callback (optional) callback to call for every line read * @return string * @throws AuthenticationException * @throws ConnectException @@ -35,7 +37,7 @@ class Connection extends RawConnection { * @throws InvalidHostException * @throws NoLoginServerException */ - public function read() { + public function read(callable $callback = null) { if (!$this->isValid()) { throw new ConnectionException('Connection not valid'); } @@ -45,26 +47,50 @@ class Connection extends RawConnection { $output = array(); $line = $this->readLine(); if ($line === false) { - if ($promptLine) { //maybe we have some error we missed on the previous line - throw new ConnectException('Unknown error (' . $promptLine . ')'); - } else { - $error = $this->readError(); // maybe something on stderr - if ($error) { - throw new ConnectException('Unknown error (' . $error . ')'); - } else { - throw new ConnectException('Unknown error'); + $this->unknownError($promptLine); + } + while (!$this->isPrompt($line)) { //next prompt functions as delimiter + if (is_callable($callback)) { + $result = $callback($line); + if ($result === false) { // allow the callback to close the connection for infinite running commands + $this->close(true); } + } else { + $output[] .= $line; } - } - $length = mb_strlen(self::DELIMITER); - while (mb_substr($line, 0, $length) !== self::DELIMITER) { //next prompt functions as delimiter - $output[] .= $line; $line = $this->readLine(); } return $output; } /** + * Check + * + * @param $line + * @return bool + */ + private function isPrompt($line) { + return mb_substr($line, 0, self::DELIMITER_LENGTH) === self::DELIMITER || $line === false; + } + + /** + * @param string $promptLine (optional) prompt line that might contain some info about the error + * @throws ConnectException + */ + private function unknownError($promptLine = '') { + if ($promptLine) { //maybe we have some error we missed on the previous line + throw new ConnectException('Unknown error (' . $promptLine . ')'); + } else { + $error = $this->readError(); // maybe something on stderr + if ($error) { + throw new ConnectException('Unknown error (' . $error . ')'); + } else { + throw new ConnectException('Unknown error'); + } + } + } + + /** * check if the first line holds a connection failure * * @param $line diff --git a/apps/files_external/3rdparty/icewind/smb/src/IShare.php b/apps/files_external/3rdparty/icewind/smb/src/IShare.php index 4851e9de053..40423151332 100644 --- a/apps/files_external/3rdparty/icewind/smb/src/IShare.php +++ b/apps/files_external/3rdparty/icewind/smb/src/IShare.php @@ -8,6 +8,17 @@ namespace Icewind\SMB; interface IShare { + // https://msdn.microsoft.com/en-us/library/dn392331.aspx + const NOTIFY_ADDED = 1; + const NOTIFY_REMOVED = 2; + const NOTIFY_MODIFIED = 3; + const NOTIFY_RENAMED_OLD = 4; + const NOTIFY_RENAMED_NEW = 5; + const NOTIFY_ADDED_STREAM = 6; + const NOTIFY_REMOVED_STREAM = 7; + const NOTIFY_MODIFIED_STREAM = 8; + const NOTIFY_REMOVED_BY_DELETE = 9; + /** * Get the name of the share * @@ -131,4 +142,11 @@ interface IShare { * @return mixed */ public function setMode($path, $mode); + + /** + * @param string $path + * @param callable $callback callable which will be called for each received change + * @return mixed + */ + public function notify($path, callable $callback); } diff --git a/apps/files_external/3rdparty/icewind/smb/src/NativeShare.php b/apps/files_external/3rdparty/icewind/smb/src/NativeShare.php index 27d975514a3..51e16d1841f 100644 --- a/apps/files_external/3rdparty/icewind/smb/src/NativeShare.php +++ b/apps/files_external/3rdparty/icewind/smb/src/NativeShare.php @@ -301,6 +301,18 @@ class NativeShare extends AbstractShare { return $this->setAttribute($path, 'system.dos_attr.mode', $mode); } + /** + * @param string $path + * @param callable $callback callable which will be called for each received change + * @return mixed + */ + public function notify($path, callable $callback) { + // php-smbclient does support notify (https://github.com/eduardok/libsmbclient-php/issues/29) + // so we use the smbclient based backend for this + $share = new Share($this->server, $this->getName()); + $share->notify($path, $callback); + } + public function __destruct() { unset($this->state); } diff --git a/apps/files_external/3rdparty/icewind/smb/src/Share.php b/apps/files_external/3rdparty/icewind/smb/src/Share.php index 694bd30bd0d..21f8fe5b139 100644 --- a/apps/files_external/3rdparty/icewind/smb/src/Share.php +++ b/apps/files_external/3rdparty/icewind/smb/src/Share.php @@ -51,6 +51,22 @@ class Share extends AbstractShare { $this->parser = new Parser(new TimeZoneProvider($this->server->getHost(), $this->system)); } + protected function getConnection() { + $workgroupArgument = ($this->server->getWorkgroup()) ? ' -W ' . escapeshellarg($this->server->getWorkgroup()) : ''; + $command = sprintf('stdbuf -o0 %s %s --authentication-file=%s %s', + $this->system->getSmbclientPath(), + $workgroupArgument, + System::getFD(3), + escapeshellarg('//' . $this->server->getHost() . '/' . $this->name) + ); + $connection = new Connection($command); + $connection->writeAuthentication($this->server->getUser(), $this->server->getPassword()); + if (!$connection->isValid()) { + throw new ConnectionException(); + } + return $connection; + } + /** * @throws \Icewind\SMB\Exception\ConnectionException * @throws \Icewind\SMB\Exception\AuthenticationException @@ -60,18 +76,7 @@ class Share extends AbstractShare { if ($this->connection and $this->connection->isValid()) { return; } - $workgroupArgument = ($this->server->getWorkgroup()) ? ' -W ' . escapeshellarg($this->server->getWorkgroup()) : ''; - $command = sprintf('%s %s --authentication-file=%s %s', - $this->system->getSmbclientPath(), - $workgroupArgument, - System::getFD(3), - escapeshellarg('//' . $this->server->getHost() . '/' . $this->name) - ); - $this->connection = new Connection($command); - $this->connection->writeAuthentication($this->server->getUser(), $this->server->getPassword()); - if (!$this->connection->isValid()) { - throw new ConnectionException(); - } + $this->connection = $this->getConnection(); } protected function reconnect() { @@ -345,6 +350,26 @@ class Share extends AbstractShare { } /** + * @param string $path + * @param callable $callback callable which will be called for each received change + * @return mixed + */ + public function notify($path, callable $callback) { + $connection = $this->getConnection(); // use a fresh connection since the notify command blocks the process + $command = 'notify ' . $this->escapePath($path); + $connection->write($command . PHP_EOL); + $connection->read(function ($line) use ($callback, $path) { + $code = (int)substr($line, 0, 4); + $subPath = substr($line, 5); + if ($path === '') { + return $callback($code, $subPath); + } else { + return $callback($code, $path . '/' . $subPath); + } + }); + } + + /** * @param string $command * @return array */ @@ -370,7 +395,7 @@ class Share extends AbstractShare { * @return bool */ protected function parseOutput($lines, $path = '') { - $this->parser->checkForError($lines, $path); + return $this->parser->checkForError($lines, $path); } /** diff --git a/lib/private/App/InfoParser.php b/lib/private/App/InfoParser.php index e763364e148..b7540c04248 100644 --- a/lib/private/App/InfoParser.php +++ b/lib/private/App/InfoParser.php @@ -80,6 +80,9 @@ class InfoParser { if (!array_key_exists('post-migration', $array['repair-steps'])) { $array['repair-steps']['post-migration'] = []; } + if (!array_key_exists('live-migration', $array['repair-steps'])) { + $array['repair-steps']['live-migration'] = []; + } if (array_key_exists('documentation', $array) && is_array($array['documentation'])) { foreach ($array['documentation'] as $key => $url) { @@ -110,6 +113,9 @@ class InfoParser { if (isset($array['repair-steps']['post-migration']['step']) && is_array($array['repair-steps']['post-migration']['step'])) { $array['repair-steps']['post-migration'] = $array['repair-steps']['post-migration']['step']; } + if (isset($array['repair-steps']['live-migration']['step']) && is_array($array['repair-steps']['live-migration']['step'])) { + $array['repair-steps']['live-migration'] = $array['repair-steps']['live-migration']['step']; + } return $array; } diff --git a/lib/private/AppFramework/Http/Request.php b/lib/private/AppFramework/Http/Request.php index 7cd8cedcfdd..fb6f1415fe9 100644 --- a/lib/private/AppFramework/Http/Request.php +++ b/lib/private/AppFramework/Http/Request.php @@ -271,6 +271,9 @@ class Request implements \ArrayAccess, \Countable, IRequest { * @return bool */ public function __isset($name) { + if (in_array($name, $this->allowedKeys, true)) { + return true; + } return isset($this->items['parameters'][$name]); } diff --git a/lib/private/Migration/BackgroundRepair.php b/lib/private/Migration/BackgroundRepair.php new file mode 100644 index 00000000000..d85c8550d5d --- /dev/null +++ b/lib/private/Migration/BackgroundRepair.php @@ -0,0 +1,116 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ +namespace OC\Migration; + +use OC\BackgroundJob\JobList; +use OC\BackgroundJob\TimedJob; +use OC\NeedsUpdateException; +use OC\Repair; +use OC_App; +use OCP\BackgroundJob\IJobList; +use OCP\ILogger; +use Symfony\Component\EventDispatcher\EventDispatcher; + +/** + * Class BackgroundRepair + * + * @package OC\Migration + */ +class BackgroundRepair extends TimedJob { + + /** @var IJobList */ + private $jobList; + + /** @var ILogger */ + private $logger; + + /** @var EventDispatcher */ + private $dispatcher; + + public function setDispatcher(EventDispatcher $dispatcher) { + $this->dispatcher = $dispatcher; + } + /** + * run the job, then remove it from the job list + * + * @param JobList $jobList + * @param ILogger $logger + */ + public function execute($jobList, ILogger $logger = null) { + // add an interval of 15 mins + $this->setInterval(15*60); + + $this->jobList = $jobList; + $this->logger = $logger; + parent::execute($jobList, $logger); + } + + /** + * @param array $argument + * @throws \Exception + * @throws \OC\NeedsUpdateException + */ + protected function run($argument) { + if (!isset($argument['app']) || !isset($argument['step'])) { + // remove the job - we can never execute it + $this->jobList->remove($this, $this->argument); + return; + } + $app = $argument['app']; + + try { + $this->loadApp($app); + } catch (NeedsUpdateException $ex) { + // as long as the app is not yet done with it's offline migration + // we better not start with the live migration + return; + } + + $step = $argument['step']; + $repair = new Repair([], $this->dispatcher); + try { + $repair->addStep($step); + } catch (\Exception $ex) { + $this->logger->logException($ex,[ + 'app' => 'migration' + ]); + + // remove the job - we can never execute it + $this->jobList->remove($this, $this->argument); + return; + } + + // execute the repair step + $repair->run(); + + // remove the job once executed successfully + $this->jobList->remove($this, $this->argument); + } + + /** + * @codeCoverageIgnore + * @param $app + * @throws NeedsUpdateException + */ + protected function loadApp($app) { + OC_App::loadApp($app); + } +} diff --git a/lib/private/app.php b/lib/private/app.php index 7bcbef32531..246bf97ee91 100644 --- a/lib/private/app.php +++ b/lib/private/app.php @@ -1153,6 +1153,7 @@ class OC_App { OC_DB::updateDbFromStructure($appPath . '/appinfo/database.xml'); } self::executeRepairSteps($appId, $appData['repair-steps']['post-migration']); + self::setupLiveMigrations($appId, $appData['repair-steps']['live-migration']); unset(self::$appVersion[$appId]); // run upgrade code if (file_exists($appPath . '/appinfo/update.php')) { @@ -1211,6 +1212,19 @@ class OC_App { /** * @param string $appId + * @param string[] $steps + */ + private static function setupLiveMigrations($appId, array $steps) { + $queue = \OC::$server->getJobList(); + foreach ($steps as $step) { + $queue->add('OC\Migration\BackgroundRepair', [ + 'app' => $appId, + 'step' => $step]); + } + } + + /** + * @param string $appId * @return \OC\Files\View|false */ public static function getStorage($appId) { diff --git a/tests/data/app/expected-info.json b/tests/data/app/expected-info.json index e05d02f7641..51d0c00ccef 100644 --- a/tests/data/app/expected-info.json +++ b/tests/data/app/expected-info.json @@ -70,6 +70,7 @@ }, "repair-steps": { "pre-migration": [], - "post-migration": [] + "post-migration": [], + "live-migration": [] } } diff --git a/tests/lib/migration/BackgroundRepairTest.php b/tests/lib/migration/BackgroundRepairTest.php new file mode 100644 index 00000000000..e092f6c2e8b --- /dev/null +++ b/tests/lib/migration/BackgroundRepairTest.php @@ -0,0 +1,120 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace Test\Migration; + + +use OC\Migration\BackgroundRepair; +use OC\NeedsUpdateException; +use OCP\ILogger; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Symfony\Component\EventDispatcher\EventDispatcher; +use Symfony\Component\EventDispatcher\GenericEvent; +use Test\TestCase; + +class TestRepairStep implements IRepairStep { + + /** + * Returns the step's name + * + * @return string + * @since 9.1.0 + */ + public function getName() { + return 'A test repair step'; + } + + /** + * Run repair step. + * Must throw exception on error. + * + * @since 9.1.0 + * @throws \Exception in case of failure + */ + public function run(IOutput $output) { + // TODO: Implement run() method. + } +} + +class BackgroundRepairTest extends TestCase { + + /** @var \OC\BackgroundJob\JobList | \PHPUnit_Framework_MockObject_MockObject */ + private $jobList; + + /** @var BackgroundRepair | \PHPUnit_Framework_MockObject_MockObject */ + private $job; + + /** @var ILogger | \PHPUnit_Framework_MockObject_MockObject */ + private $logger; + + public function setUp() { + parent::setUp(); + + $this->jobList = $this->getMockBuilder('OC\BackgroundJob\JobList') + ->disableOriginalConstructor() + ->getMock(); + $this->logger = $this->getMockBuilder('OCP\ILogger') + ->disableOriginalConstructor() + ->getMock(); + $this->job = $this->getMock('OC\Migration\BackgroundRepair', ['loadApp']); + } + + public function testNoArguments() { + $this->jobList->expects($this->once())->method('remove'); + $this->job->execute($this->jobList); + } + + public function testAppUpgrading() { + $this->jobList->expects($this->never())->method('remove'); + $this->job->expects($this->once())->method('loadApp')->with('test')->willThrowException(new NeedsUpdateException()); + $this->job->setArgument([ + 'app' => 'test', + 'step' => 'j' + ]); + $this->job->execute($this->jobList); + } + + public function testUnknownStep() { + $this->jobList->expects($this->once())->method('remove'); + $this->logger->expects($this->once())->method('logException'); + $this->job->setArgument([ + 'app' => 'test', + 'step' => 'j' + ]); + $this->job->execute($this->jobList, $this->logger); + } + + public function testWorkingStep() { + /** @var EventDispatcher | \PHPUnit_Framework_MockObject_MockObject $dispatcher */ + $dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcher', []); + $dispatcher->expects($this->once())->method('dispatch') + ->with('\OC\Repair::step', new GenericEvent('\OC\Repair::step', ['A test repair step'])); + + $this->jobList->expects($this->once())->method('remove'); + $this->job->setDispatcher($dispatcher); + $this->job->setArgument([ + 'app' => 'test', + 'step' => '\Test\Migration\TestRepairStep' + ]); + $this->job->execute($this->jobList, $this->logger); + } +} |