* success on SQLite and Postgres * failure on MySQL due to the limited charset that only supports up to 3 bytes Add config option to update charset of mysql to utf8mb4 * fully optional * requires additional options set in the database only disable unicode test on mysql Fixing ctor call Adding docker based unit test execution for mysql utf8mb4 Add mysqlmb4 test configuration to Jenkinsfile fix collation on utf8mb4 Properly setup charset and collation in the doctrine connection Allow files containing 4-byte chars in case the database supports it During setup of a mysql database we try to detect if charset 'utf8mb4' can be used Fix mysql settings Add console command to migrate the charset Set ROW_FORMAT before setting collation to mb4 Also select tables with wrong collation Faster MySQL docker Signed-off-by: Morris Jobke <hey@morrisjobke.de>tags/v12.0.0beta1
<?php | |||||
/** | |||||
* @author Thomas Müller <thomas.mueller@tmit.eu> | |||||
* | |||||
* @copyright Copyright (c) 2017, ownCloud GmbH | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
namespace OC\Core\Command\Db; | |||||
use Doctrine\DBAL\Platforms\MySqlPlatform; | |||||
use OC\Migration\ConsoleOutput; | |||||
use OC\Repair\Collation; | |||||
use OCP\IConfig; | |||||
use OCP\IDBConnection; | |||||
use OCP\ILogger; | |||||
use OCP\IURLGenerator; | |||||
use Symfony\Component\Console\Command\Command; | |||||
use Symfony\Component\Console\Input\InputInterface; | |||||
use Symfony\Component\Console\Output\OutputInterface; | |||||
class ConvertMysqlToMB4 extends Command { | |||||
/** @var IConfig */ | |||||
private $config; | |||||
/** @var IDBConnection */ | |||||
private $connection; | |||||
/** @var IURLGenerator */ | |||||
private $urlGenerator; | |||||
/** @var ILogger */ | |||||
private $logger; | |||||
/** | |||||
* @param IConfig $config | |||||
* @param IDBConnection $connection | |||||
* @param IURLGenerator $urlGenerator | |||||
* @param ILogger $logger | |||||
*/ | |||||
public function __construct(IConfig $config, IDBConnection $connection, IURLGenerator $urlGenerator, ILogger $logger) { | |||||
$this->config = $config; | |||||
$this->connection = $connection; | |||||
$this->urlGenerator = $urlGenerator; | |||||
$this->logger = $logger; | |||||
parent::__construct(); | |||||
} | |||||
protected function configure() { | |||||
$this | |||||
->setName('db:convert-mysql-charset') | |||||
->setDescription('Convert charset of MySQL/MariaDB to use utf8mb4'); | |||||
} | |||||
protected function execute(InputInterface $input, OutputInterface $output) { | |||||
if (!$this->connection->getDatabasePlatform() instanceof MySqlPlatform) { | |||||
$output->writeln("This command is only valid for MySQL/MariaDB databases."); | |||||
return 1; | |||||
} | |||||
if (!$this->connection->supports4ByteText()) { | |||||
$url = $this->urlGenerator->linkToDocs('admin-db-conversion'); | |||||
$output->writeln("The database is not properly setup to use the charset utf8mb4."); | |||||
$output->writeln("Also check that the setting 'mysql.utf8mb4' is set to true in the config.php."); | |||||
$output->writeln("For more information please read the documentation at $url"); | |||||
return 1; | |||||
} | |||||
// enable charset | |||||
$this->config->setSystemValue('mysql.utf8mb4', true); | |||||
// run conversion | |||||
$coll = new Collation($this->config, $this->logger, $this->connection, false); | |||||
$coll->run(new ConsoleOutput($output)); | |||||
return 0; | |||||
} | |||||
} |
$application->add(new OC\Core\Command\Db\GenerateChangeScript()); | $application->add(new OC\Core\Command\Db\GenerateChangeScript()); | ||||
$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); | $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); | ||||
$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger())); | |||||
$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); | $application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); | ||||
$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); | $application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); |
* @since 11.0.0 | * @since 11.0.0 | ||||
*/ | */ | ||||
public function supports4ByteText() { | public function supports4ByteText() { | ||||
return ! ($this->getDatabasePlatform() instanceof MySqlPlatform && $this->getParams()['charset'] !== 'utf8mb4'); | |||||
if (!$this->getDatabasePlatform() instanceof MySqlPlatform) { | |||||
return true; | |||||
} | |||||
if ($this->getParams()['charset'] === 'utf8mb4') { | |||||
return true; | |||||
} | |||||
return false; | |||||
} | } | ||||
} | } |
$connectionParams['driverOptions'] = $driverOptions; | $connectionParams['driverOptions'] = $driverOptions; | ||||
} | } | ||||
// set default table creation options | |||||
$connectionParams['defaultTableOptions'] = [ | |||||
'collate' => 'utf8_bin', | |||||
'tablePrefix' => $connectionParams['tablePrefix'] | |||||
]; | |||||
if($this->config->getValue('mysql.utf8mb4', false)) { | |||||
$connectionParams['defaultTableOptions'] = [ | |||||
'collate' => 'utf8mb4_bin', | |||||
'charset' => 'utf8mb4', | |||||
'row_format' => 'compressed', | |||||
'tablePrefix' => $connectionParams['tablePrefix'] | |||||
]; | |||||
} | |||||
return $connectionParams; | return $connectionParams; | ||||
} | } | ||||
} | } |
use Doctrine\DBAL\Platforms\OraclePlatform; | use Doctrine\DBAL\Platforms\OraclePlatform; | ||||
use Doctrine\DBAL\Platforms\PostgreSqlPlatform; | use Doctrine\DBAL\Platforms\PostgreSqlPlatform; | ||||
use Doctrine\DBAL\Platforms\SqlitePlatform; | use Doctrine\DBAL\Platforms\SqlitePlatform; | ||||
use Doctrine\DBAL\Schema\Schema; | |||||
use OCP\IDBConnection; | use OCP\IDBConnection; | ||||
class MDB2SchemaManager { | class MDB2SchemaManager { | ||||
*/ | */ | ||||
public function createDbFromStructure($file) { | public function createDbFromStructure($file) { | ||||
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); | $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); | ||||
$toSchema = $schemaReader->loadSchemaFromFile($file); | |||||
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); | |||||
$toSchema = $schemaReader->loadSchemaFromFile($file, $toSchema); | |||||
return $this->executeSchemaChange($toSchema); | return $this->executeSchemaChange($toSchema); | ||||
} | } | ||||
private function readSchemaFromFile($file) { | private function readSchemaFromFile($file) { | ||||
$platform = $this->conn->getDatabasePlatform(); | $platform = $this->conn->getDatabasePlatform(); | ||||
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform); | $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform); | ||||
return $schemaReader->loadSchemaFromFile($file); | |||||
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); | |||||
return $schemaReader->loadSchemaFromFile($file, $toSchema); | |||||
} | } | ||||
/** | /** | ||||
*/ | */ | ||||
public function removeDBStructure($file) { | public function removeDBStructure($file) { | ||||
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); | $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); | ||||
$fromSchema = $schemaReader->loadSchemaFromFile($file); | |||||
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); | |||||
$fromSchema = $schemaReader->loadSchemaFromFile($file, $toSchema); | |||||
$toSchema = clone $fromSchema; | $toSchema = clone $fromSchema; | ||||
/** @var $table \Doctrine\DBAL\Schema\Table */ | /** @var $table \Doctrine\DBAL\Schema\Table */ | ||||
foreach ($toSchema->getTables() as $table) { | foreach ($toSchema->getTables() as $table) { |
$this->config = $config; | $this->config = $config; | ||||
$this->DBNAME = $config->getSystemValue('dbname', 'owncloud'); | $this->DBNAME = $config->getSystemValue('dbname', 'owncloud'); | ||||
$this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_'); | $this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_'); | ||||
// Oracle does not support longer index names then 30 characters. | |||||
// We use this limit for all DBs to make sure it does not cause a | |||||
// problem. | |||||
$this->schemaConfig = new SchemaConfig(); | |||||
$this->schemaConfig->setMaxIdentifierLength(30); | |||||
} | } | ||||
/** | /** | ||||
* @param string $file | * @param string $file | ||||
* @param Schema $schema | |||||
* @return Schema | * @return Schema | ||||
* @throws \DomainException | * @throws \DomainException | ||||
*/ | */ | ||||
public function loadSchemaFromFile($file) { | |||||
$schema = new \Doctrine\DBAL\Schema\Schema(); | |||||
public function loadSchemaFromFile($file, Schema $schema) { | |||||
$loadEntities = libxml_disable_entity_loader(false); | $loadEntities = libxml_disable_entity_loader(false); | ||||
$xml = simplexml_load_file($file); | $xml = simplexml_load_file($file); | ||||
libxml_disable_entity_loader($loadEntities); | libxml_disable_entity_loader($loadEntities); |
<?php | |||||
/** | |||||
* @author Thomas Müller <thomas.mueller@tmit.eu> | |||||
* | |||||
* @copyright Copyright (c) 2017, ownCloud GmbH | |||||
* @license AGPL-3.0 | |||||
* | |||||
* This code is free software: you can redistribute it and/or modify | |||||
* it under the terms of the GNU Affero General Public License, version 3, | |||||
* as published by the Free Software Foundation. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||||
* GNU Affero General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Affero General Public License, version 3, | |||||
* along with this program. If not, see <http://www.gnu.org/licenses/> | |||||
* | |||||
*/ | |||||
namespace OC\Migration; | |||||
use OCP\Migration\IOutput; | |||||
use Symfony\Component\Console\Helper\ProgressBar; | |||||
use Symfony\Component\Console\Output\OutputInterface; | |||||
/** | |||||
* Class SimpleOutput | |||||
* | |||||
* Just a simple IOutput implementation with writes messages to the log file. | |||||
* Alternative implementations will write to the console or to the web ui (web update case) | |||||
* | |||||
* @package OC\Migration | |||||
*/ | |||||
class ConsoleOutput implements IOutput { | |||||
/** @var OutputInterface */ | |||||
private $output; | |||||
/** @var ProgressBar */ | |||||
private $progressBar; | |||||
public function __construct(OutputInterface $output) { | |||||
$this->output = $output; | |||||
} | |||||
/** | |||||
* @param string $message | |||||
*/ | |||||
public function info($message) { | |||||
$this->output->writeln("<info>$message</info>"); | |||||
} | |||||
/** | |||||
* @param string $message | |||||
*/ | |||||
public function warning($message) { | |||||
$this->output->writeln("<comment>$message</comment>"); | |||||
} | |||||
/** | |||||
* @param int $max | |||||
*/ | |||||
public function startProgress($max = 0) { | |||||
if (!is_null($this->progressBar)) { | |||||
$this->progressBar->finish(); | |||||
} | |||||
$this->progressBar = new ProgressBar($this->output); | |||||
$this->progressBar->start($max); | |||||
} | |||||
/** | |||||
* @param int $step | |||||
* @param string $description | |||||
*/ | |||||
public function advance($step = 1, $description = '') { | |||||
if (!is_null($this->progressBar)) { | |||||
$this->progressBar = new ProgressBar($this->output); | |||||
$this->progressBar->start(); | |||||
} | |||||
$this->progressBar->advance($step); | |||||
} | |||||
public function finishProgress() { | |||||
if (is_null($this->progressBar)) { | |||||
return; | |||||
} | |||||
$this->progressBar->finish(); | |||||
} | |||||
} |
} | } | ||||
$output->info("Change collation for $table ..."); | $output->info("Change collation for $table ..."); | ||||
if ($characterSet === 'utf8mb4') { | |||||
// need to set row compression first | |||||
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` ROW_FORMAT=COMPRESSED;'); | |||||
$query->execute(); | |||||
} | |||||
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;'); | $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;'); | ||||
try { | try { | ||||
$query->execute(); | $query->execute(); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
if (empty($tables)) { | |||||
$output->info('All tables already have the correct collation -> nothing to do'); | |||||
} | |||||
} | } | ||||
/** | /** | ||||
* @param \Doctrine\DBAL\Connection $connection | |||||
* @param IDBConnection $connection | |||||
* @return string[] | * @return string[] | ||||
*/ | */ | ||||
protected function getAllNonUTF8BinTables($connection) { | |||||
protected function getAllNonUTF8BinTables(IDBConnection $connection) { | |||||
$dbName = $this->config->getSystemValue("dbname"); | $dbName = $this->config->getSystemValue("dbname"); | ||||
$characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; | $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; | ||||
$rows = $connection->fetchAll( | |||||
// fetch tables by columns | |||||
$statement = $connection->executeQuery( | |||||
"SELECT DISTINCT(TABLE_NAME) AS `table`" . | "SELECT DISTINCT(TABLE_NAME) AS `table`" . | ||||
" FROM INFORMATION_SCHEMA . COLUMNS" . | " FROM INFORMATION_SCHEMA . COLUMNS" . | ||||
" WHERE TABLE_SCHEMA = ?" . | " WHERE TABLE_SCHEMA = ?" . | ||||
" AND TABLE_NAME LIKE \"*PREFIX*%\"", | " AND TABLE_NAME LIKE \"*PREFIX*%\"", | ||||
array($dbName) | array($dbName) | ||||
); | ); | ||||
$result = array(); | |||||
$rows = $statement->fetchAll(); | |||||
$result = []; | |||||
foreach ($rows as $row) { | foreach ($rows as $row) { | ||||
$result[] = $row['table']; | |||||
$result[$row['table']] = true; | |||||
} | } | ||||
return $result; | |||||
// fetch tables by collation | |||||
$statement = $connection->executeQuery( | |||||
"SELECT DISTINCT(TABLE_NAME) AS `table`" . | |||||
" FROM INFORMATION_SCHEMA . TABLES" . | |||||
" WHERE TABLE_SCHEMA = ?" . | |||||
" AND TABLE_COLLATION <> '" . $characterSet . "_bin'" . | |||||
" AND TABLE_NAME LIKE \"*PREFIX*%\"", | |||||
[$dbName] | |||||
); | |||||
$rows = $statement->fetchAll(); | |||||
foreach ($rows as $row) { | |||||
$result[$row['table']] = true; | |||||
} | |||||
return array_keys($result); | |||||
} | } | ||||
} | } | ||||
public function testRead() { | public function testRead() { | ||||
$reader = new MDB2SchemaReader($this->getConfig(), new MySqlPlatform()); | $reader = new MDB2SchemaReader($this->getConfig(), new MySqlPlatform()); | ||||
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml'); | |||||
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml', new Schema()); | |||||
$this->assertCount(1, $schema->getTables()); | $this->assertCount(1, $schema->getTables()); | ||||
$table = $schema->getTable('test_table'); | $table = $schema->getTable('test_table'); |
namespace Test\DB; | namespace Test\DB; | ||||
use Doctrine\DBAL\Schema\Schema; | |||||
use Doctrine\DBAL\Schema\SchemaDiff; | use Doctrine\DBAL\Schema\SchemaDiff; | ||||
use OC\DB\MDB2SchemaManager; | use OC\DB\MDB2SchemaManager; | ||||
use OC\DB\MDB2SchemaReader; | use OC\DB\MDB2SchemaReader; | ||||
$this->manager->createDbFromStructure($schemaFile); | $this->manager->createDbFromStructure($schemaFile); | ||||
$schemaReader = new MDB2SchemaReader($this->config, $this->connection->getDatabasePlatform()); | $schemaReader = new MDB2SchemaReader($this->config, $this->connection->getDatabasePlatform()); | ||||
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile); | |||||
$toSchema = new Schema([], [], $this->connection->getSchemaManager()->createSchemaConfig()); | |||||
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile, $toSchema); | |||||
// get the diff | // get the diff | ||||
/** @var SchemaDiff $diff */ | /** @var SchemaDiff $diff */ |