diff options
-rw-r--r-- | core/Command/Db/Migrations/ExecuteCommand.php | 67 | ||||
-rw-r--r-- | core/Command/Db/Migrations/GenerateCommand.php | 166 | ||||
-rw-r--r-- | core/Command/Db/Migrations/MigrateCommand.php | 64 | ||||
-rw-r--r-- | core/Command/Db/Migrations/StatusCommand.php | 115 | ||||
-rw-r--r-- | core/register_command.php | 4 | ||||
-rw-r--r-- | lib/private/DB/Connection.php | 24 | ||||
-rw-r--r-- | lib/private/DB/MigrationService.php | 401 | ||||
-rw-r--r-- | lib/private/DB/Migrator.php | 17 | ||||
-rw-r--r-- | lib/private/DB/OracleConnection.php | 23 | ||||
-rw-r--r-- | lib/private/DB/OracleMigrator.php | 56 | ||||
-rw-r--r-- | lib/private/Migration/SimpleOutput.php | 84 | ||||
-rw-r--r-- | lib/private/Setup.php | 2 | ||||
-rw-r--r-- | lib/private/Setup/AbstractDatabase.php | 9 | ||||
-rw-r--r-- | lib/private/Updater.php | 8 | ||||
-rw-r--r-- | lib/private/legacy/app.php | 9 | ||||
-rw-r--r-- | lib/public/IDBConnection.php | 17 | ||||
-rw-r--r-- | lib/public/Migration/IMigrationStep.php | 49 | ||||
-rw-r--r-- | tests/lib/DB/MigrationsTest.php | 162 |
18 files changed, 1260 insertions, 17 deletions
diff --git a/core/Command/Db/Migrations/ExecuteCommand.php b/core/Command/Db/Migrations/ExecuteCommand.php new file mode 100644 index 00000000000..6aad4f4973f --- /dev/null +++ b/core/Command/Db/Migrations/ExecuteCommand.php @@ -0,0 +1,67 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Core\Command\Db\Migrations; + + +use OC\DB\MigrationService; +use OC\Migration\ConsoleOutput; +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class ExecuteCommand extends Command { + + /** @var IDBConnection */ + private $connection; + + /** + * ExecuteCommand constructor. + * + * @param IDBConnection $connection + */ + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + + parent::__construct(); + } + + protected function configure() { + $this + ->setName('migrations:execute') + ->setDescription('Execute a single migration version manually.') + ->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on') + ->addArgument('version', InputArgument::REQUIRED, 'The version to execute.', null); + + parent::configure(); + } + + public function execute(InputInterface $input, OutputInterface $output) { + $appName = $input->getArgument('app'); + $ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output)); + $version = $input->getArgument('version'); + + $ms->executeStep($version); + } + +} diff --git a/core/Command/Db/Migrations/GenerateCommand.php b/core/Command/Db/Migrations/GenerateCommand.php new file mode 100644 index 00000000000..307989c845a --- /dev/null +++ b/core/Command/Db/Migrations/GenerateCommand.php @@ -0,0 +1,166 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Core\Command\Db\Migrations; + + +use OC\DB\MigrationService; +use OC\Migration\ConsoleOutput; +use OCP\IConfig; +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Exception\RuntimeException; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class GenerateCommand extends Command { + + private static $_templateSimple = + '<?php +namespace <namespace>; + +use OCP\Migration\ISimpleMigration; +use OCP\Migration\IOutput; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version<version> implements ISimpleMigration { + + /** + * @param IOutput $out + */ + public function run(IOutput $out) { + // auto-generated - please modify it to your needs + } +} +'; + + private static $_templateSchema = + '<?php +namespace <namespace>; + +use Doctrine\DBAL\Schema\Schema; +use OCP\Migration\ISchemaMigration; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version<version> implements ISchemaMigration { + + public function changeSchema(Schema $schema, array $options) { + // auto-generated - please modify it to your needs + } +} +'; + + private static $_templateSql = + '<?php +namespace <namespace>; + +use OCP\IDBConnection; +use OCP\Migration\ISqlMigration; + +/** + * Auto-generated migration step: Please modify to your needs! + */ +class Version<version> implements ISqlMigration { + + public function sql(IDBConnection $connection) { + // auto-generated - please modify it to your needs + } +} +'; + + /** @var IDBConnection */ + private $connection; + + /** + * @param IDBConnection $connection + */ + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + + parent::__construct(); + } + + protected function configure() { + $this + ->setName('migrations:generate') + ->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on') + ->addArgument('kind', InputArgument::REQUIRED, 'simple, schema or sql - defines the kind of migration to be generated'); + + parent::configure(); + } + + public function execute(InputInterface $input, OutputInterface $output) { + $appName = $input->getArgument('app'); + $ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output)); + + $kind = $input->getArgument('kind'); + $version = date('YmdHis'); + $path = $this->generateMigration($ms, $version, $kind); + + $output->writeln("New migration class has been generated to <info>$path</info>"); + + } + + /** + * @param MigrationService $ms + * @param string $version + * @param string $kind + * @return string + */ + private function generateMigration(MigrationService $ms, $version, $kind) { + $placeHolders = [ + '<namespace>', + '<version>', + ]; + $replacements = [ + $ms->getMigrationsNamespace(), + $version, + ]; + $code = str_replace($placeHolders, $replacements, $this->getTemplate($kind)); + $dir = $ms->getMigrationsDirectory(); + $path = $dir . '/Version' . $version . '.php'; + + if (file_put_contents($path, $code) === false) { + throw new RuntimeException('Failed to generate new migration step.'); + } + + return $path; + } + + private function getTemplate($kind) { + if ($kind === 'simple') { + return self::$_templateSimple; + } + if ($kind === 'schema') { + return self::$_templateSchema; + } + if ($kind === 'sql') { + return self::$_templateSql; + } + throw new \InvalidArgumentException('Kind can only be one of the following: simple, schema or sql'); + } + +} diff --git a/core/Command/Db/Migrations/MigrateCommand.php b/core/Command/Db/Migrations/MigrateCommand.php new file mode 100644 index 00000000000..2b0e082acaa --- /dev/null +++ b/core/Command/Db/Migrations/MigrateCommand.php @@ -0,0 +1,64 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Core\Command\Db\Migrations; + + +use OC\DB\MigrationService; +use OC\Migration\ConsoleOutput; +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class MigrateCommand extends Command { + + /** @var IDBConnection */ + private $connection; + + /** + * @param IDBConnection $connection + */ + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + parent::__construct(); + } + + protected function configure() { + $this + ->setName('migrations:migrate') + ->setDescription('Execute a migration to a specified version or the latest available version.') + ->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on') + ->addArgument('version', InputArgument::OPTIONAL, 'The version number (YYYYMMDDHHMMSS) or alias (first, prev, next, latest) to migrate to.', 'latest'); + + parent::configure(); + } + + public function execute(InputInterface $input, OutputInterface $output) { + $appName = $input->getArgument('app'); + $ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output)); + $version = $input->getArgument('version'); + + $ms->migrate($version); + } + +} diff --git a/core/Command/Db/Migrations/StatusCommand.php b/core/Command/Db/Migrations/StatusCommand.php new file mode 100644 index 00000000000..20172000ee3 --- /dev/null +++ b/core/Command/Db/Migrations/StatusCommand.php @@ -0,0 +1,115 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\Core\Command\Db\Migrations; + +use OC\DB\MigrationService; +use OC\Migration\ConsoleOutput; +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class StatusCommand extends Command { + + /** @var IDBConnection */ + private $connection; + + /** + * @param IDBConnection $connection + */ + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + parent::__construct(); + } + + protected function configure() { + $this + ->setName('migrations:status') + ->setDescription('View the status of a set of migrations.') + ->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on'); + } + + public function execute(InputInterface $input, OutputInterface $output) { + $appName = $input->getArgument('app'); + $ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output)); + + $infos = $this->getMigrationsInfos($ms); + foreach ($infos as $key => $value) { + $output->writeln(" <comment>>></comment> $key: " . str_repeat(' ', 50 - strlen($key)) . $value); + } + } + + /** + * @param MigrationService $ms + * @return array associative array of human readable info name as key and the actual information as value + */ + public function getMigrationsInfos(MigrationService $ms) { + + $executedMigrations = $ms->getMigratedVersions(); + $availableMigrations = $ms->getAvailableVersions(); + $executedUnavailableMigrations = array_diff($executedMigrations, array_keys($availableMigrations)); + + $numExecutedUnavailableMigrations = count($executedUnavailableMigrations); + $numNewMigrations = count(array_diff(array_keys($availableMigrations), $executedMigrations)); + + $infos = [ + 'App' => $ms->getApp(), + 'Version Table Name' => $ms->getMigrationsTableName(), + 'Migrations Namespace' => $ms->getMigrationsNamespace(), + 'Migrations Directory' => $ms->getMigrationsDirectory(), + 'Previous Version' => $this->getFormattedVersionAlias($ms, 'prev'), + 'Current Version' => $this->getFormattedVersionAlias($ms, 'current'), + 'Next Version' => $this->getFormattedVersionAlias($ms, 'next'), + 'Latest Version' => $this->getFormattedVersionAlias($ms, 'latest'), + 'Executed Migrations' => count($executedMigrations), + 'Executed Unavailable Migrations' => $numExecutedUnavailableMigrations, + 'Available Migrations' => count($availableMigrations), + 'New Migrations' => $numNewMigrations, + ]; + + return $infos; + } + + /** + * @param MigrationService $migrationService + * @param string $alias + * @return mixed|null|string + */ + private function getFormattedVersionAlias(MigrationService $migrationService, $alias) { + $migration = $migrationService->getMigration($alias); + //No version found + if ($migration === null) { + if ($alias === 'next') { + return 'Already at latest migration step'; + } + + if ($alias === 'prev') { + return 'Already at first migration step'; + } + } + + return $migration; + } + + +} diff --git a/core/register_command.php b/core/register_command.php index 59fc65edbc8..924da6fc94f 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -85,6 +85,10 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { $application->add(new OC\Core\Command\Db\GenerateChangeScript()); $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); $application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger())); + $application->add(new OC\Core\Command\Db\Migrations\StatusCommand(\OC::$server->getDatabaseConnection())); + $application->add(new OC\Core\Command\Db\Migrations\MigrateCommand(\OC::$server->getDatabaseConnection())); + $application->add(new OC\Core\Command\Db\Migrations\GenerateCommand(\OC::$server->getDatabaseConnection())); + $application->add(new OC\Core\Command\Db\Migrations\ExecuteCommand(\OC::$server->getDatabaseConnection())); $application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 6b56ae8ad5c..563c077b04a 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -35,6 +35,7 @@ use Doctrine\DBAL\Cache\QueryCacheProfile; use Doctrine\Common\EventManager; use Doctrine\DBAL\Platforms\MySqlPlatform; use Doctrine\DBAL\Exception\ConstraintViolationException; +use Doctrine\DBAL\Schema\Schema; use OC\DB\QueryBuilder\QueryBuilder; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; @@ -418,4 +419,27 @@ class Connection extends \Doctrine\DBAL\Connection implements IDBConnection { } return $this->getParams()['charset'] === 'utf8mb4'; } + + + /** + * Create the schema of the connected database + * + * @return Schema + */ + public function createSchema() { + $schemaManager = new MDB2SchemaManager($this); + $migrator = $schemaManager->getMigrator(); + return $migrator->createSchema(); + } + + /** + * Migrate the database to the given schema + * + * @param Schema $toSchema + */ + public function migrateToSchema(Schema $toSchema) { + $schemaManager = new MDB2SchemaManager($this); + $migrator = $schemaManager->getMigrator(); + $migrator->migrate($toSchema); + } } diff --git a/lib/private/DB/MigrationService.php b/lib/private/DB/MigrationService.php new file mode 100644 index 00000000000..8a4980d1118 --- /dev/null +++ b/lib/private/DB/MigrationService.php @@ -0,0 +1,401 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * @author Vincent Petry <pvince81@owncloud.com> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + +namespace OC\DB; + +use OC\IntegrityCheck\Helpers\AppLocator; +use OC\Migration\SimpleOutput; +use OCP\AppFramework\QueryException; +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\ISchemaMigration; +use OCP\Migration\ISimpleMigration; +use OCP\Migration\ISqlMigration; +use Doctrine\DBAL\Schema\Column; +use Doctrine\DBAL\Schema\Table; +use Doctrine\DBAL\Types\Type; + +class MigrationService { + + /** @var boolean */ + private $migrationTableCreated; + /** @var array */ + private $migrations; + /** @var IOutput */ + private $output; + /** @var Connection */ + private $connection; + /** @var string */ + private $appName; + + /** + * MigrationService constructor. + * + * @param $appName + * @param IDBConnection $connection + * @param AppLocator $appLocator + * @param IOutput|null $output + * @throws \Exception + */ + function __construct($appName, IDBConnection $connection, IOutput $output = null, AppLocator $appLocator = null) { + $this->appName = $appName; + $this->connection = $connection; + $this->output = $output; + if (is_null($this->output)) { + $this->output = new SimpleOutput(\OC::$server->getLogger(), $appName); + } + + if ($appName === 'core') { + $this->migrationsPath = \OC::$SERVERROOT . '/core/Migrations'; + $this->migrationsNamespace = 'OC\\Migrations'; + } else { + if (is_null($appLocator)) { + $appLocator = new AppLocator(); + } + $appPath = $appLocator->getAppPath($appName); + $this->migrationsPath = "$appPath/appinfo/Migrations"; + $this->migrationsNamespace = "OCA\\$appName\\Migrations"; + } + + if (!is_dir($this->migrationsPath)) { + if (!mkdir($this->migrationsPath)) { + throw new \Exception("Could not create migration folder \"{$this->migrationsPath}\""); + }; + } + } + + private static function requireOnce($file) { + require_once $file; + } + + /** + * Returns the name of the app for which this migration is executed + * + * @return string + */ + public function getApp() { + return $this->appName; + } + + /** + * @return bool + * @codeCoverageIgnore - this will implicitly tested on installation + */ + private function createMigrationTable() { + if ($this->migrationTableCreated) { + return false; + } + + if ($this->connection->tableExists('migrations')) { + $this->migrationTableCreated = true; + return false; + } + + $tableName = $this->connection->getPrefix() . 'migrations'; + $tableName = $this->connection->getDatabasePlatform()->quoteIdentifier($tableName); + + $columns = [ + 'app' => new Column($this->connection->getDatabasePlatform()->quoteIdentifier('app'), Type::getType('string'), ['length' => 255]), + 'version' => new Column($this->connection->getDatabasePlatform()->quoteIdentifier('version'), Type::getType('string'), ['length' => 255]), + ]; + $table = new Table($tableName, $columns); + $table->setPrimaryKey([ + $this->connection->getDatabasePlatform()->quoteIdentifier('app'), + $this->connection->getDatabasePlatform()->quoteIdentifier('version')]); + $this->connection->getSchemaManager()->createTable($table); + + $this->migrationTableCreated = true; + + return true; + } + + /** + * Returns all versions which have already been applied + * + * @return string[] + * @codeCoverageIgnore - no need to test this + */ + public function getMigratedVersions() { + $this->createMigrationTable(); + $qb = $this->connection->getQueryBuilder(); + + $qb->select('version') + ->from('migrations') + ->where($qb->expr()->eq('app', $qb->createNamedParameter($this->getApp()))) + ->orderBy('version'); + + $result = $qb->execute(); + $rows = $result->fetchAll(\PDO::FETCH_COLUMN); + $result->closeCursor(); + + return $rows; + } + + /** + * Returns all versions which are available in the migration folder + * + * @return array + */ + public function getAvailableVersions() { + $this->ensureMigrationsAreLoaded(); + return array_keys($this->migrations); + } + + protected function findMigrations() { + $directory = realpath($this->migrationsPath); + $iterator = new \RegexIterator( + new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::LEAVES_ONLY + ), + '#^.+\\/Version[^\\/]{1,255}\\.php$#i', + \RegexIterator::GET_MATCH); + + $files = array_keys(iterator_to_array($iterator)); + uasort($files, function ($a, $b) { + return (basename($a) < basename($b)) ? -1 : 1; + }); + + $migrations = []; + + foreach ($files as $file) { + static::requireOnce($file); + $className = basename($file, '.php'); + $version = (string) substr($className, 7); + if ($version === '0') { + throw new \InvalidArgumentException( + "Cannot load a migrations with the name '$version' because it is a reserved number" + ); + } + $migrations[$version] = sprintf('%s\\%s', $this->migrationsNamespace, $className); + } + + return $migrations; + } + + /** + * @param string $to + */ + private function getMigrationsToExecute($to) { + $knownMigrations = $this->getMigratedVersions(); + $availableMigrations = $this->getAvailableVersions(); + + $toBeExecuted = []; + foreach ($availableMigrations as $v) { + if ($to !== 'latest' && $v > $to) { + continue; + } + if ($this->shallBeExecuted($v, $knownMigrations)) { + $toBeExecuted[] = $v; + } + } + + return $toBeExecuted; + } + + /** + * @param string[] $knownMigrations + */ + private function shallBeExecuted($m, $knownMigrations) { + if (in_array($m, $knownMigrations)) { + return false; + } + + return true; + } + + /** + * @param string $version + */ + private function markAsExecuted($version) { + $this->connection->insertIfNotExist('*PREFIX*migrations', [ + 'app' => $this->appName, + 'version' => $version + ]); + } + + /** + * Returns the name of the table which holds the already applied versions + * + * @return string + */ + public function getMigrationsTableName() { + return $this->connection->getPrefix() . 'migrations'; + } + + /** + * Returns the namespace of the version classes + * + * @return string + */ + public function getMigrationsNamespace() { + return $this->migrationsNamespace; + } + + /** + * Returns the directory which holds the versions + * + * @return string + */ + public function getMigrationsDirectory() { + return $this->migrationsPath; + } + + /** + * Return the explicit version for the aliases; current, next, prev, latest + * + * @param string $alias + * @return mixed|null|string + */ + public function getMigration($alias) { + switch($alias) { + case 'current': + return $this->getCurrentVersion(); + case 'next': + return $this->getRelativeVersion($this->getCurrentVersion(), 1); + case 'prev': + return $this->getRelativeVersion($this->getCurrentVersion(), -1); + case 'latest': + $this->ensureMigrationsAreLoaded(); + + return @end($this->getAvailableVersions()); + } + return '0'; + } + + /** + * @param string $version + * @param int $delta + * @return null|string + */ + private function getRelativeVersion($version, $delta) { + $this->ensureMigrationsAreLoaded(); + + $versions = $this->getAvailableVersions(); + array_unshift($versions, 0); + $offset = array_search($version, $versions); + if ($offset === false || !isset($versions[$offset + $delta])) { + // Unknown version or delta out of bounds. + return null; + } + + return (string) $versions[$offset + $delta]; + } + + /** + * @return string + */ + private function getCurrentVersion() { + $m = $this->getMigratedVersions(); + if (count($m) === 0) { + return '0'; + } + return @end(array_values($m)); + } + + /** + * @return string + */ + private function getClass($version) { + $this->ensureMigrationsAreLoaded(); + + if (isset($this->migrations[$version])) { + return $this->migrations[$version]; + } + + throw new \InvalidArgumentException("Version $version is unknown."); + } + + /** + * Allows to set an IOutput implementation which is used for logging progress and messages + * + * @param IOutput $output + */ + public function setOutput(IOutput $output) { + $this->output = $output; + } + + /** + * Applies all not yet applied versions up to $to + * + * @param string $to + */ + public function migrate($to = 'latest') { + // read known migrations + $toBeExecuted = $this->getMigrationsToExecute($to); + foreach ($toBeExecuted as $version) { + $this->executeStep($version); + } + } + + /** + * @param string $version + * @return mixed + * @throws \Exception + */ + protected function createInstance($version) { + $class = $this->getClass($version); + try { + $s = \OC::$server->query($class); + } catch (QueryException $e) { + if (class_exists($class)) { + $s = new $class(); + } else { + throw new \Exception("Migration step '$class' is unknown"); + } + } + + return $s; + } + + /** + * Executes one explicit version + * + * @param string $version + */ + public function executeStep($version) { + + // FIXME our interface + $instance = $this->createInstance($version); + if ($instance instanceof ISimpleMigration) { + $instance->run($this->output); + } + if ($instance instanceof ISqlMigration) { + $sqls = $instance->sql($this->connection); + foreach ($sqls as $s) { + $this->connection->executeQuery($s); + } + } + if ($instance instanceof ISchemaMigration) { + $toSchema = $this->connection->createSchema(); + $instance->changeSchema($toSchema, ['tablePrefix' => $this->connection->getPrefix()]); + $this->connection->migrateToSchema($toSchema); + } + $this->markAsExecuted($version); + } + + private function ensureMigrationsAreLoaded() { + if (empty($this->migrations)) { + $this->migrations = $this->findMigrations(); + } + } +} diff --git a/lib/private/DB/Migrator.php b/lib/private/DB/Migrator.php index 1d00d9a1b45..da381ba0284 100644 --- a/lib/private/DB/Migrator.php +++ b/lib/private/DB/Migrator.php @@ -43,14 +43,10 @@ use Symfony\Component\EventDispatcher\GenericEvent; class Migrator { - /** - * @var \Doctrine\DBAL\Connection $connection - */ + /** @var \Doctrine\DBAL\Connection */ protected $connection; - /** - * @var ISecureRandom - */ + /** @var ISecureRandom */ private $random; /** @var IConfig */ @@ -197,6 +193,12 @@ class Migrator { return new Table($newName, $table->getColumns(), $newIndexes, array(), 0, $table->getOptions()); } + public function createSchema() { + $filterExpression = $this->getFilterExpression(); + $this->connection->getConfiguration()->setFilterSchemaAssetsExpression($filterExpression); + return $this->connection->getSchemaManager()->createSchema(); + } + /** * @param Schema $targetSchema * @param \Doctrine\DBAL\Connection $connection @@ -217,8 +219,7 @@ class Migrator { } $filterExpression = $this->getFilterExpression(); - $this->connection->getConfiguration()-> - setFilterSchemaAssetsExpression($filterExpression); + $this->connection->getConfiguration()->setFilterSchemaAssetsExpression($filterExpression); $sourceSchema = $connection->getSchemaManager()->createSchema(); // remove tables we don't know about diff --git a/lib/private/DB/OracleConnection.php b/lib/private/DB/OracleConnection.php index 08d71365172..51faf21970c 100644 --- a/lib/private/DB/OracleConnection.php +++ b/lib/private/DB/OracleConnection.php @@ -30,9 +30,14 @@ class OracleConnection extends Connection { * Quote the keys of the array */ private function quoteKeys(array $data) { - $return = array(); + $return = []; + $c = $this->getDatabasePlatform()->getIdentifierQuoteCharacter(); foreach($data as $key => $value) { - $return[$this->quoteIdentifier($key)] = $value; + if ($key[0] !== $c) { + $return[$this->quoteIdentifier($key)] = $value; + } else { + $return[$key] = $value; + } } return $return; } @@ -41,7 +46,9 @@ class OracleConnection extends Connection { * {@inheritDoc} */ public function insert($tableName, array $data, array $types = array()) { - $tableName = $this->quoteIdentifier($tableName); + if ($tableName[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) { + $tableName = $this->quoteIdentifier($tableName); + } $data = $this->quoteKeys($data); return parent::insert($tableName, $data, $types); } @@ -50,7 +57,9 @@ class OracleConnection extends Connection { * {@inheritDoc} */ public function update($tableName, array $data, array $identifier, array $types = array()) { - $tableName = $this->quoteIdentifier($tableName); + if ($tableName[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) { + $tableName = $this->quoteIdentifier($tableName); + } $data = $this->quoteKeys($data); $identifier = $this->quoteKeys($identifier); return parent::update($tableName, $data, $identifier, $types); @@ -60,9 +69,11 @@ class OracleConnection extends Connection { * {@inheritDoc} */ public function delete($tableExpression, array $identifier, array $types = array()) { - $tableName = $this->quoteIdentifier($tableExpression); + if ($tableExpression[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) { + $tableExpression = $this->quoteIdentifier($tableExpression); + } $identifier = $this->quoteKeys($identifier); - return parent::delete($tableName, $identifier); + return parent::delete($tableExpression, $identifier); } /** diff --git a/lib/private/DB/OracleMigrator.php b/lib/private/DB/OracleMigrator.php index 908b2dedf03..2735529b5e2 100644 --- a/lib/private/DB/OracleMigrator.php +++ b/lib/private/DB/OracleMigrator.php @@ -24,19 +24,75 @@ namespace OC\DB; +use Doctrine\DBAL\DBALException; +use Doctrine\DBAL\Schema\Column; use Doctrine\DBAL\Schema\ColumnDiff; +use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Schema; +use Doctrine\DBAL\Schema\Table; class OracleMigrator extends NoCheckMigrator { /** * @param Schema $targetSchema * @param \Doctrine\DBAL\Connection $connection * @return \Doctrine\DBAL\Schema\SchemaDiff + * @throws DBALException */ protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { $schemaDiff = parent::getDiff($targetSchema, $connection); // oracle forces us to quote the identifiers + $schemaDiff->newTables = array_map(function(Table $table) { + return new Table( + $this->connection->quoteIdentifier($table->getName()), + array_map(function(Column $column) { + $newColumn = new Column( + $this->connection->quoteIdentifier($column->getName()), + $column->getType() + ); + $newColumn->setAutoincrement($column->getAutoincrement()); + $newColumn->setColumnDefinition($column->getColumnDefinition()); + $newColumn->setComment($column->getComment()); + $newColumn->setDefault($column->getDefault()); + $newColumn->setFixed($column->getFixed()); + $newColumn->setLength($column->getLength()); + $newColumn->setNotnull($column->getNotnull()); + $newColumn->setPrecision($column->getPrecision()); + $newColumn->setScale($column->getScale()); + $newColumn->setUnsigned($column->getUnsigned()); + $newColumn->setPlatformOptions($column->getPlatformOptions()); + $newColumn->setCustomSchemaOptions($column->getPlatformOptions()); + return $newColumn; + }, $table->getColumns()), + array_map(function(Index $index) { + return new Index( + $this->connection->quoteIdentifier($index->getName()), + array_map(function($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $index->getColumns()), + $index->isUnique(), + $index->isPrimary(), + $index->getFlags(), + $index->getOptions() + ); + }, $table->getIndexes()), + $table->getForeignKeys(), + 0, + $table->getOptions() + ); + }, $schemaDiff->newTables); + + $schemaDiff->removedTables = array_map(function(Table $table) { + return new Table( + $this->connection->quoteIdentifier($table->getName()), + $table->getColumns(), + $table->getIndexes(), + $table->getForeignKeys(), + 0, + $table->getOptions() + ); + }, $schemaDiff->removedTables); + foreach ($schemaDiff->changedTables as $tableDiff) { $tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name); foreach ($tableDiff->changedColumns as $column) { diff --git a/lib/private/Migration/SimpleOutput.php b/lib/private/Migration/SimpleOutput.php new file mode 100644 index 00000000000..b28fcbd7628 --- /dev/null +++ b/lib/private/Migration/SimpleOutput.php @@ -0,0 +1,84 @@ +<?php +/** + * @author Thomas Müller <thomas.mueller@tmit.eu> + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see <http://www.gnu.org/licenses/> + * + */ + + +namespace OC\Migration; + + +use OCP\ILogger; +use OCP\Migration\IOutput; + +/** + * Class SimpleOutput + * + * Just a simple IOutput implementation with writes messages to the log file. + * Alternative implementations will write to the console or to the web ui (web update case) + * + * @package OC\Migration + */ +class SimpleOutput implements IOutput { + + /** @var ILogger */ + private $logger; + private $appName; + + public function __construct(ILogger $logger, $appName) { + $this->logger = $logger; + $this->appName = $appName; + } + + /** + * @param string $message + * @since 9.1.0 + */ + public function info($message) { + $this->logger->info($message, ['app' => $this->appName]); + } + + /** + * @param string $message + * @since 9.1.0 + */ + public function warning($message) { + $this->logger->warning($message, ['app' => $this->appName]); + } + + /** + * @param int $max + * @since 9.1.0 + */ + public function startProgress($max = 0) { + } + + /** + * @param int $step + * @param string $description + * @since 9.1.0 + */ + public function advance($step = 1, $description = '') { + } + + /** + * @since 9.1.0 + */ + public function finishProgress() { + } +} diff --git a/lib/private/Setup.php b/lib/private/Setup.php index b8a861fd296..5cd3c84ce92 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -332,6 +332,8 @@ class Setup { try { $dbSetup->initialize($options); $dbSetup->setupDatabase($username); + // apply necessary migrations + $dbSetup->runMigrations(); } catch (\OC\DatabaseSetupException $e) { $error[] = array( 'error' => $e->getMessage(), diff --git a/lib/private/Setup/AbstractDatabase.php b/lib/private/Setup/AbstractDatabase.php index d5c34291e60..2fbec326a5d 100644 --- a/lib/private/Setup/AbstractDatabase.php +++ b/lib/private/Setup/AbstractDatabase.php @@ -27,6 +27,7 @@ namespace OC\Setup; use OC\DB\ConnectionFactory; +use OC\DB\MigrationService; use OC\SystemConfig; use OCP\IL10N; use OCP\ILogger; @@ -143,4 +144,12 @@ abstract class AbstractDatabase { * @param string $userName */ abstract public function setupDatabase($userName); + + public function runMigrations() { + if (!is_dir(\OC::$SERVERROOT."/core/Migrations")) { + return; + } + $ms = new MigrationService('core', \OC::$server->getDatabaseConnection()); + $ms->migrate(); + } } diff --git a/lib/private/Updater.php b/lib/private/Updater.php index 6d08e5d4cc0..464344d2209 100644 --- a/lib/private/Updater.php +++ b/lib/private/Updater.php @@ -32,6 +32,7 @@ namespace OC; +use OC\DB\MigrationService; use OC\Hooks\BasicEmitter; use OC\IntegrityCheck\Checker; use OC_App; @@ -300,8 +301,11 @@ class Updater extends BasicEmitter { protected function doCoreUpgrade() { $this->emit('\OC\Updater', 'dbUpgradeBefore'); - // do the real upgrade - \OC_DB::updateDbFromStructure(\OC::$SERVERROOT . '/db_structure.xml'); + // execute core migrations + if (is_dir(\OC::$SERVERROOT . '/core/Migrations')) { + $ms = new MigrationService('core', \OC::$server->getDatabaseConnection()); + $ms->migrate(); + } $this->emit('\OC\Updater', 'dbUpgrade'); } diff --git a/lib/private/legacy/app.php b/lib/private/legacy/app.php index 1bdbd1e2a83..631738c726b 100644 --- a/lib/private/legacy/app.php +++ b/lib/private/legacy/app.php @@ -50,6 +50,7 @@ use OC\App\DependencyAnalyzer; use OC\App\InfoParser; use OC\App\Platform; +use OC\DB\MigrationService; use OC\Installer; use OC\Repair; use OCP\App\ManagerEvent; @@ -1043,12 +1044,18 @@ class OC_App { } $appData = self::getAppInfo($appId); self::executeRepairSteps($appId, $appData['repair-steps']['pre-migration']); - if (file_exists($appPath . '/appinfo/database.xml')) { + + if (isset($appData['use-migrations']) && $appData['use-migrations'] === 'true') { + $ms = new MigrationService($appId, \OC::$server->getDatabaseConnection()); + $ms->migrate(); + } else if (file_exists($appPath . '/appinfo/database.xml')) { OC_DB::updateDbFromStructure($appPath . '/appinfo/database.xml'); } + self::executeRepairSteps($appId, $appData['repair-steps']['post-migration']); self::setupLiveMigrations($appId, $appData['repair-steps']['live-migration']); unset(self::$appVersion[$appId]); + // run upgrade code if (file_exists($appPath . '/appinfo/update.php')) { self::loadApp($appId); diff --git a/lib/public/IDBConnection.php b/lib/public/IDBConnection.php index efd65d55f7e..56cf50c5fb3 100644 --- a/lib/public/IDBConnection.php +++ b/lib/public/IDBConnection.php @@ -34,6 +34,7 @@ // use OCP namespace for all classes that are considered public. // This means that they should be used by apps instead of the internal ownCloud classes namespace OCP; +use Doctrine\DBAL\Schema\Schema; use OCP\DB\QueryBuilder\IQueryBuilder; /** @@ -259,4 +260,20 @@ interface IDBConnection { * @since 11.0.0 */ public function supports4ByteText(); + + /** + * Create the schema of the connected database + * + * @return Schema + * @since 13.0.0 + */ + public function createSchema(); + + /** + * Migrate the database to the given schema + * + * @param Schema $toSchema + * @since 13.0.0 + */ + public function migrateToSchema(Schema $toSchema); } diff --git a/lib/public/Migration/IMigrationStep.php b/lib/public/Migration/IMigrationStep.php new file mode 100644 index 00000000000..3f95eed7598 --- /dev/null +++ b/lib/public/Migration/IMigrationStep.php @@ -0,0 +1,49 @@ +<?php +/** + * @copyright Copyright (c) 2017 Joas Schilling <coding@schilljs.com> + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +namespace OCP\Migration; + +use Doctrine\DBAL\Schema\Schema; + +/** + * @since 13.0.0 + */ +interface IMigrationStep { + + /** + * @param IOutput $output + * @since 13.0.0 + */ + public function preSchemaChange(IOutput $output); + + /** + * @param Schema $schema + * @param array $options + * @since 13.0.0 + */ + public function changeSchema(Schema $schema, array $options); + + /** + * @param IOutput $output + * @since 13.0.0 + */ + public function postSchemaChange(IOutput $output); +} diff --git a/tests/lib/DB/MigrationsTest.php b/tests/lib/DB/MigrationsTest.php new file mode 100644 index 00000000000..fbb54bf9a1b --- /dev/null +++ b/tests/lib/DB/MigrationsTest.php @@ -0,0 +1,162 @@ +<?php + +/** + * Copyright (c) 2016 Thomas Müller <thomas.mueller@tmit.eu> + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + + +namespace Test\DB; + +use Doctrine\DBAL\Schema\Schema; +use OC\DB\Connection; +use OC\DB\MigrationService; +use OCP\IDBConnection; +use OCP\Migration\ISchemaMigration; +use OCP\Migration\ISqlMigration; + +/** + * Class MigrationsTest + * + * @package Test\DB + */ +class MigrationsTest extends \Test\TestCase { + + /** @var MigrationService | \PHPUnit_Framework_MockObject_MockObject */ + private $migrationService; + /** @var \PHPUnit_Framework_MockObject_MockObject | IDBConnection $db */ + private $db; + + public function setUp() { + parent::setUp(); + + $this->db = $this->createMock(Connection::class); + $this->db->expects($this->any())->method('getPrefix')->willReturn('test_oc_'); + $this->migrationService = new MigrationService('testing', $this->db); + } + + public function testGetters() { + $this->assertEquals('testing', $this->migrationService->getApp()); + $this->assertEquals(\OC::$SERVERROOT . '/apps/testing/appinfo/Migrations', $this->migrationService->getMigrationsDirectory()); + $this->assertEquals('OCA\testing\Migrations', $this->migrationService->getMigrationsNamespace()); + $this->assertEquals('test_oc_migrations', $this->migrationService->getMigrationsTableName()); + } + + public function testCore() { + $this->migrationService = new MigrationService('core', $this->db); + + $this->assertEquals('core', $this->migrationService->getApp()); + $this->assertEquals(\OC::$SERVERROOT . '/core/Migrations', $this->migrationService->getMigrationsDirectory()); + $this->assertEquals('OC\Migrations', $this->migrationService->getMigrationsNamespace()); + $this->assertEquals('test_oc_migrations', $this->migrationService->getMigrationsTableName()); + } + + /** + * @expectedException \InvalidArgumentException + * @expectedExceptionMessage Version 20170130180000 is unknown. + */ + public function testExecuteUnknownStep() { + $this->migrationService->executeStep('20170130180000'); + } + + /** + * @expectedException \Exception + * @expectedExceptionMessage App not found + */ + public function testUnknownApp() { + $migrationService = new MigrationService('unknown-bloody-app', $this->db); + } + + /** + * @expectedException \Exception + * @expectedExceptionMessage Migration step 'X' is unknown + */ + public function testExecuteStepWithUnknownClass() { + $this->migrationService = $this->getMockBuilder(MigrationService::class) + ->setMethods(['findMigrations']) + ->setConstructorArgs(['testing', $this->db]) + ->getMock(); + $this->migrationService->expects($this->any())->method('findMigrations')->willReturn( + ['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A'] + ); + $this->migrationService->executeStep('20170130180000'); + } + + public function testExecuteStepWithSchemaMigrationStep() { + + $schema = $this->createMock(Schema::class); + $this->db->expects($this->any())->method('createSchema')->willReturn($schema); + + $step = $this->createMock(ISchemaMigration::class); + $step->expects($this->once())->method('changeSchema'); + $this->migrationService = $this->getMockBuilder(MigrationService::class) + ->setMethods(['createInstance']) + ->setConstructorArgs(['testing', $this->db]) + ->getMock(); + $this->migrationService->expects($this->any())->method('createInstance')->with('20170130180000')->willReturn($step); + $this->migrationService->executeStep('20170130180000'); + } + + public function testExecuteStepWithSqlMigrationStep() { + + $this->db->expects($this->exactly(3))->method('executeQuery')->withConsecutive(['1'], ['2'], ['3']); + + $step = $this->createMock(ISqlMigration::class); + $step->expects($this->once())->method('sql')->willReturn(['1', '2', '3']); + $this->migrationService = $this->getMockBuilder(MigrationService::class) + ->setMethods(['createInstance']) + ->setConstructorArgs(['testing', $this->db]) + ->getMock(); + $this->migrationService->expects($this->any())->method('createInstance')->with('20170130180000')->willReturn($step); + $this->migrationService->executeStep('20170130180000'); + } + + public function testGetMigration() { + $this->migrationService = $this->getMockBuilder(MigrationService::class) + ->setMethods(['getMigratedVersions', 'findMigrations']) + ->setConstructorArgs(['testing', $this->db]) + ->getMock(); + $this->migrationService->expects($this->any())->method('getMigratedVersions')->willReturn( + ['20170130180000', '20170130180001'] + ); + $this->migrationService->expects($this->any())->method('findMigrations')->willReturn( + ['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A'] + ); + + $this->assertEquals( + ['20170130180000', '20170130180001', '20170130180002', '20170130180003'], + $this->migrationService->getAvailableVersions()); + + $migration = $this->migrationService->getMigration('current'); + $this->assertEquals('20170130180001', $migration); + $migration = $this->migrationService->getMigration('prev'); + $this->assertEquals('20170130180000', $migration); + $migration = $this->migrationService->getMigration('next'); + $this->assertEquals('20170130180002', $migration); + $migration = $this->migrationService->getMigration('latest'); + $this->assertEquals('20170130180003', $migration); + } + + public function testMigrate() { + $this->migrationService = $this->getMockBuilder(MigrationService::class) + ->setMethods(['getMigratedVersions', 'findMigrations', 'executeStep']) + ->setConstructorArgs(['testing', $this->db]) + ->getMock(); + $this->migrationService->expects($this->any())->method('getMigratedVersions')->willReturn( + ['20170130180000', '20170130180001'] + ); + $this->migrationService->expects($this->any())->method('findMigrations')->willReturn( + ['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A'] + ); + + $this->assertEquals( + ['20170130180000', '20170130180001', '20170130180002', '20170130180003'], + $this->migrationService->getAvailableVersions()); + + $this->migrationService->expects($this->exactly(2))->method('executeStep') + ->withConsecutive(['20170130180002'], ['20170130180003']); + $this->migrationService->migrate(); + } +} |