Adding tests for 4 byte unicode characters

* success on SQLite and Postgres
* failure on MySQL due to the limited charset that only supports up to 3 bytes

Add config option to update charset of mysql to utf8mb4

* fully optional
* requires additional options set in the database

only disable unicode test on mysql

Fixing ctor call

Adding docker based unit test execution for mysql utf8mb4

Add mysqlmb4 test configuration to Jenkinsfile

fix collation on utf8mb4

Properly setup charset and collation in the doctrine connection

Allow files containing 4-byte chars in case the database supports it

During setup of a mysql database we try to detect if charset 'utf8mb4' can be used

Fix mysql settings

Add console command to migrate the charset

Set ROW_FORMAT before setting collation to mb4

Also select tables with wrong collation

Faster MySQL docker

Signed-off-by: Morris Jobke <hey@morrisjobke.de>

6 files changed, 156 insertions, 18 deletions

diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php
index 4b1c560c5ca..0ee58a54ece 100644
--- a/lib/private/DB/Connection.php
+++ b/lib/private/DB/Connection.php
@@ -416,6 +416,12 @@ class Connection extends \Doctrine\DBAL\Connection implements IDBConnection {
 	 * @since 11.0.0
 	 */
 	public function supports4ByteText() {
-		return ! ($this->getDatabasePlatform() instanceof MySqlPlatform && $this->getParams()['charset'] !== 'utf8mb4');
+		if (!$this->getDatabasePlatform() instanceof MySqlPlatform) {
+			return true;
+		}
+		if ($this->getParams()['charset'] === 'utf8mb4') {
+			return true;
+		}
+		return false;
 	}
 }
diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php
index d8f1fb2480d..39f15ff4a63 100644
--- a/lib/private/DB/ConnectionFactory.php
+++ b/lib/private/DB/ConnectionFactory.php
@@ -201,6 +201,21 @@ class ConnectionFactory {
 			$connectionParams['driverOptions'] = $driverOptions;
 		}
 
+		// set default table creation options
+		$connectionParams['defaultTableOptions'] = [
+			'collate' => 'utf8_bin',
+			'tablePrefix' => $connectionParams['tablePrefix']
+		];
+
+		if($this->config->getValue('mysql.utf8mb4', false)) {
+			$connectionParams['defaultTableOptions'] = [
+				'collate' => 'utf8mb4_bin',
+				'charset' => 'utf8mb4',
+				'row_format' => 'compressed',
+				'tablePrefix' => $connectionParams['tablePrefix']
+			];
+		}
+
 		return $connectionParams;
 	}
 }
diff --git a/lib/private/DB/MDB2SchemaManager.php b/lib/private/DB/MDB2SchemaManager.php
index f209991eb84..89b0d153212 100644
--- a/lib/private/DB/MDB2SchemaManager.php
+++ b/lib/private/DB/MDB2SchemaManager.php
@@ -33,6 +33,7 @@ use Doctrine\DBAL\Platforms\MySqlPlatform;
 use Doctrine\DBAL\Platforms\OraclePlatform;
 use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
 use Doctrine\DBAL\Platforms\SqlitePlatform;
+use Doctrine\DBAL\Schema\Schema;
 use OCP\IDBConnection;
 
 class MDB2SchemaManager {
@@ -66,7 +67,8 @@ class MDB2SchemaManager {
 	 */
 	public function createDbFromStructure($file) {
 		$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
-		$toSchema = $schemaReader->loadSchemaFromFile($file);
+		$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
+		$toSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
 		return $this->executeSchemaChange($toSchema);
 	}
 
@@ -100,7 +102,8 @@ class MDB2SchemaManager {
 	private function readSchemaFromFile($file) {
 		$platform = $this->conn->getDatabasePlatform();
 		$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform);
-		return $schemaReader->loadSchemaFromFile($file);
+		$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
+		return $schemaReader->loadSchemaFromFile($file, $toSchema);
 	}
 
 	/**
@@ -137,7 +140,8 @@ class MDB2SchemaManager {
 	 */
 	public function removeDBStructure($file) {
 		$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
-		$fromSchema = $schemaReader->loadSchemaFromFile($file);
+		$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
+		$fromSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
 		$toSchema = clone $fromSchema;
 		/** @var $table \Doctrine\DBAL\Schema\Table */
 		foreach ($toSchema->getTables() as $table) {
diff --git a/lib/private/DB/MDB2SchemaReader.php b/lib/private/DB/MDB2SchemaReader.php
index 0a51f1b48f2..dc067b43c4a 100644
--- a/lib/private/DB/MDB2SchemaReader.php
+++ b/lib/private/DB/MDB2SchemaReader.php
@@ -68,21 +68,15 @@ class MDB2SchemaReader {
 		$this->config = $config;
 		$this->DBNAME = $config->getSystemValue('dbname', 'owncloud');
 		$this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_');
-
-		// Oracle does not support longer index names then 30 characters.
-		// We use this limit for all DBs to make sure it does not cause a
-		// problem.
-		$this->schemaConfig = new SchemaConfig();
-		$this->schemaConfig->setMaxIdentifierLength(30);
 	}
 
 	/**
 	 * @param string $file
+	 * @param Schema $schema
 	 * @return Schema
 	 * @throws \DomainException
 	 */
-	public function loadSchemaFromFile($file) {
-		$schema = new \Doctrine\DBAL\Schema\Schema();
+	public function loadSchemaFromFile($file, Schema $schema) {
 		$loadEntities = libxml_disable_entity_loader(false);
 		$xml = simplexml_load_file($file);
 		libxml_disable_entity_loader($loadEntities);
diff --git a/lib/private/Migration/ConsoleOutput.php b/lib/private/Migration/ConsoleOutput.php
new file mode 100644
index 00000000000..892a2f43419
--- /dev/null
+++ b/lib/private/Migration/ConsoleOutput.php
@@ -0,0 +1,93 @@
+<?php
+/**
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+
+namespace OC\Migration;
+
+
+use OCP\Migration\IOutput;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Class SimpleOutput
+ *
+ * Just a simple IOutput implementation with writes messages to the log file.
+ * Alternative implementations will write to the console or to the web ui (web update case)
+ *
+ * @package OC\Migration
+ */
+class ConsoleOutput implements IOutput {
+
+	/** @var OutputInterface */
+	private $output;
+
+	/** @var ProgressBar */
+	private $progressBar;
+
+	public function __construct(OutputInterface $output) {
+		$this->output = $output;
+	}
+
+	/**
+	 * @param string $message
+	 */
+	public function info($message) {
+		$this->output->writeln("<info>$message</info>");
+	}
+
+	/**
+	 * @param string $message
+	 */
+	public function warning($message) {
+		$this->output->writeln("<comment>$message</comment>");
+	}
+
+	/**
+	 * @param int $max
+	 */
+	public function startProgress($max = 0) {
+		if (!is_null($this->progressBar)) {
+			$this->progressBar->finish();
+		}
+		$this->progressBar = new ProgressBar($this->output);
+		$this->progressBar->start($max);
+	}
+
+	/**
+	 * @param int $step
+	 * @param string $description
+	 */
+	public function advance($step = 1, $description = '') {
+		if (!is_null($this->progressBar)) {
+			$this->progressBar = new ProgressBar($this->output);
+			$this->progressBar->start();
+		}
+		$this->progressBar->advance($step);
+	}
+
+	public function finishProgress() {
+		if (is_null($this->progressBar)) {
+			return;
+		}
+		$this->progressBar->finish();
+	}
+}
diff --git a/lib/private/Repair/Collation.php b/lib/private/Repair/Collation.php
index 12e83c2b981..a01700e047e 100644
--- a/lib/private/Repair/Collation.php
+++ b/lib/private/Repair/Collation.php
@@ -88,6 +88,11 @@ class Collation implements IRepairStep {
 			}
 
 			$output->info("Change collation for $table ...");
+			if ($characterSet === 'utf8mb4') {
+				// need to set row compression first
+				$query = $this->connection->prepare('ALTER TABLE `' . $table . '` ROW_FORMAT=COMPRESSED;');
+				$query->execute();
+			}
 			$query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;');
 			try {
 				$query->execute();
@@ -99,16 +104,21 @@ class Collation implements IRepairStep {
 				}
 			}
 		}
+		if (empty($tables)) {
+			$output->info('All tables already have the correct collation -> nothing to do');
+		}
 	}
 
 	/**
-	 * @param \Doctrine\DBAL\Connection $connection
+	 * @param IDBConnection $connection
 	 * @return string[]
 	 */
-	protected function getAllNonUTF8BinTables($connection) {
+	protected function getAllNonUTF8BinTables(IDBConnection $connection) {
 		$dbName = $this->config->getSystemValue("dbname");
 		$characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
-		$rows = $connection->fetchAll(
+
+		// fetch tables by columns
+		$statement = $connection->executeQuery(
 			"SELECT DISTINCT(TABLE_NAME) AS `table`" .
 			"	FROM INFORMATION_SCHEMA . COLUMNS" .
 			"	WHERE TABLE_SCHEMA = ?" .
@@ -116,11 +126,27 @@ class Collation implements IRepairStep {
 			"	AND TABLE_NAME LIKE \"*PREFIX*%\"",
 			array($dbName)
 		);
-		$result = array();
+		$rows = $statement->fetchAll();
+		$result = [];
 		foreach ($rows as $row) {
-			$result[] = $row['table'];
+			$result[$row['table']] = true;
 		}
-		return $result;
+
+		// fetch tables by collation
+		$statement = $connection->executeQuery(
+			"SELECT DISTINCT(TABLE_NAME) AS `table`" .
+			"	FROM INFORMATION_SCHEMA . TABLES" .
+			"	WHERE TABLE_SCHEMA = ?" .
+			"	AND TABLE_COLLATION <> '" . $characterSet . "_bin'" .
+			"	AND TABLE_NAME LIKE \"*PREFIX*%\"",
+			[$dbName]
+		);
+		$rows = $statement->fetchAll();
+		foreach ($rows as $row) {
+			$result[$row['table']] = true;
+		}
+
+		return array_keys($result);
 	}
 }