Adding tests for 4 byte unicode characters

* success on SQLite and Postgres
* failure on MySQL due to the limited charset that only supports up to 3 bytes

Add config option to update charset of mysql to utf8mb4

* fully optional
* requires additional options set in the database

only disable unicode test on mysql

Fixing ctor call

Adding docker based unit test execution for mysql utf8mb4

Add mysqlmb4 test configuration to Jenkinsfile

fix collation on utf8mb4

Properly setup charset and collation in the doctrine connection

Allow files containing 4-byte chars in case the database supports it

During setup of a mysql database we try to detect if charset 'utf8mb4' can be used

Fix mysql settings

Add console command to migrate the charset

Set ROW_FORMAT before setting collation to mb4

Also select tables with wrong collation

Faster MySQL docker

Signed-off-by: Morris Jobke <hey@morrisjobke.de>

2 files changed, 92 insertions, 0 deletions

diff --git a/core/Command/Db/ConvertMysqlToMB4.php b/core/Command/Db/ConvertMysqlToMB4.php
new file mode 100644
index 00000000000..d456db13b11
--- /dev/null
+++ b/core/Command/Db/ConvertMysqlToMB4.php
@@ -0,0 +1,91 @@
+<?php
+/**
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Core\Command\Db;
+
+use Doctrine\DBAL\Platforms\MySqlPlatform;
+use OC\Migration\ConsoleOutput;
+use OC\Repair\Collation;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\ILogger;
+use OCP\IURLGenerator;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ConvertMysqlToMB4 extends Command {
+	/** @var IConfig */
+	private $config;
+
+	/** @var IDBConnection */
+	private $connection;
+
+	/** @var IURLGenerator */
+	private $urlGenerator;
+
+	/** @var ILogger */
+	private $logger;
+
+	/**
+	 * @param IConfig $config
+	 * @param IDBConnection $connection
+	 * @param IURLGenerator $urlGenerator
+	 * @param ILogger $logger
+	 */
+	public function __construct(IConfig $config, IDBConnection $connection, IURLGenerator $urlGenerator, ILogger $logger) {
+		$this->config = $config;
+		$this->connection = $connection;
+		$this->urlGenerator = $urlGenerator;
+		$this->logger = $logger;
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('db:convert-mysql-charset')
+			->setDescription('Convert charset of MySQL/MariaDB to use utf8mb4');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		if (!$this->connection->getDatabasePlatform() instanceof MySqlPlatform) {
+			$output->writeln("This command is only valid for MySQL/MariaDB databases.");
+			return 1;
+		}
+
+		if (!$this->connection->supports4ByteText()) {
+			$url = $this->urlGenerator->linkToDocs('admin-db-conversion');
+			$output->writeln("The database is not properly setup to use the charset utf8mb4.");
+			$output->writeln("Also check that the setting 'mysql.utf8mb4' is set to true in the config.php.");
+			$output->writeln("For more information please read the documentation at $url");
+			return 1;
+		}
+
+		// enable charset
+		$this->config->setSystemValue('mysql.utf8mb4', true);
+
+		// run conversion
+		$coll = new Collation($this->config, $this->logger, $this->connection, false);
+		$coll->run(new ConsoleOutput($output));
+
+		return 0;
+	}
+}
diff --git a/core/register_command.php b/core/register_command.php
index 6a8ab2bebe2..629fd183b06 100644
--- a/core/register_command.php
+++ b/core/register_command.php
@@ -84,6 +84,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
 
 	$application->add(new OC\Core\Command\Db\GenerateChangeScript());
 	$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
+	$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger()));
 
 	$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig()));
 	$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager()));