Merge pull request #6563 from nextcloud/bigint-ids

Migrate ID columns to bigint

12 files changed, 349 insertions, 3 deletions

diff --git a/apps/dav/appinfo/info.xml b/apps/dav/appinfo/info.xml
index b104cb7fd69..9b500c9f300 100644
--- a/apps/dav/appinfo/info.xml
+++ b/apps/dav/appinfo/info.xml
@@ -5,7 +5,7 @@
 	<description>WebDAV endpoint</description>
 	<licence>AGPL</licence>
 	<author>owncloud.org</author>
-	<version>1.4.3</version>
+	<version>1.4.4</version>
 	<default_enable/>
 	<types>
 		<filesystem/>
diff --git a/apps/dav/lib/Migration/Version1004Date20170926103422.php b/apps/dav/lib/Migration/Version1004Date20170926103422.php
new file mode 100644
index 00000000000..76adc8fe6a1
--- /dev/null
+++ b/apps/dav/lib/Migration/Version1004Date20170926103422.php
@@ -0,0 +1,35 @@
+<?php
+namespace OCA\DAV\Migration;
+
+use Doctrine\DBAL\Schema\Schema;
+use OCP\Migration\BigIntMigration;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Migration\IOutput;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version1004Date20170926103422 extends BigIntMigration {
+
+	/**
+	 * @return array Returns an array with the following structure
+	 * ['table1' => ['column1', 'column2'], ...]
+	 * @since 13.0.0
+	 */
+	protected function getColumnsByTable() {
+		return [
+			'addressbooks' => ['id'],
+			'addressbookchanges' => ['id', 'addressbookid'],
+			'calendars' => ['id'],
+			'calendarchanges' => ['id', 'calendarid'],
+			'calendarobjects' => ['id', 'calendarid'],
+			'calendarobjects_props' => ['id', 'calendarid', 'objectid'],
+			'calendarsubscriptions' => ['id'],
+			'cards' => ['id', 'addressbookid'],
+			'cards_properties' => ['id', 'addressbookid', 'cardid'],
+			'dav_shares' => ['id', 'resourceid'],
+			'schedulingobjects' => ['id'],
+		];
+	}
+
+}
diff --git a/apps/twofactor_backupcodes/appinfo/info.xml b/apps/twofactor_backupcodes/appinfo/info.xml
index 0002d0dcf0c..18728a7047a 100644
--- a/apps/twofactor_backupcodes/appinfo/info.xml
+++ b/apps/twofactor_backupcodes/appinfo/info.xml
@@ -5,7 +5,7 @@
 	<description>A two-factor auth backup codes provider</description>
 	<licence>agpl</licence>
 	<author>Christoph Wurst</author>
-	<version>1.2.2</version>
+	<version>1.2.3</version>
 	<namespace>TwoFactorBackupCodes</namespace>
 	<category>other</category>
 
diff --git a/apps/twofactor_backupcodes/lib/Migration/Version1002Date20170926101419.php b/apps/twofactor_backupcodes/lib/Migration/Version1002Date20170926101419.php
new file mode 100644
index 00000000000..8072ebd6810
--- /dev/null
+++ b/apps/twofactor_backupcodes/lib/Migration/Version1002Date20170926101419.php
@@ -0,0 +1,25 @@
+<?php
+namespace OCA\TwoFactorBackupCodes\Migration;
+
+use Doctrine\DBAL\Schema\Schema;
+use OCP\Migration\BigIntMigration;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Migration\IOutput;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version1002Date20170926101419 extends BigIntMigration {
+
+	/**
+	 * @return array Returns an array with the following structure
+	 * ['table1' => ['column1', 'column2'], ...]
+	 * @since 13.0.0
+	 */
+	protected function getColumnsByTable() {
+		return [
+			'twofactor_backupcodes' => ['id'],
+		];
+	}
+
+}
diff --git a/core/Migrations/Version13000Date20170926101637.php b/core/Migrations/Version13000Date20170926101637.php
new file mode 100644
index 00000000000..3eea8a8a3b9
--- /dev/null
+++ b/core/Migrations/Version13000Date20170926101637.php
@@ -0,0 +1,42 @@
+<?php
+namespace OC\Core\Migrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use OCP\Migration\BigIntMigration;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Migration\IOutput;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version13000Date20170926101637 extends BigIntMigration {
+
+	/**
+	 * @return array Returns an array with the following structure
+	 * ['table1' => ['column1', 'column2'], ...]
+	 * @since 13.0.0
+	 */
+	protected function getColumnsByTable() {
+		return [
+			'admin_settings' => ['id'],
+			'authtoken' => ['id'],
+			'bruteforce_attempts' => ['id'],
+			'comments' => ['id', 'parent_id', 'topmost_parent_id'],
+			'filecache' => ['fileid', 'storage', 'parent', 'mimetype', 'mimepart'],
+			'file_locks' => ['id'],
+			'jobs' => ['id'],
+			'mimetypes' => ['id'],
+			'mounts' => ['id'],
+			'personal_settings' => ['id'],
+			'properties' => ['id'],
+			'share' => ['id', 'parent', 'file_source'],
+			'storages' => ['numeric_id'],
+			'systemtag' => ['id'],
+			'systemtag_group' => ['systemtagid'],
+			'systemtag_object_mapping' => ['systemtagid'],
+			'vcategory' => ['id'],
+			'vcategory_to_object' => ['objid', 'categoryid'],
+		];
+	}
+
+}
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 29bca415ed6..50bd6d33274 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -222,6 +222,7 @@ return array(
     'OCP\\Lockdown\\ILockdownManager' => $baseDir . '/lib/public/Lockdown/ILockdownManager.php',
     'OCP\\Mail\\IEMailTemplate' => $baseDir . '/lib/public/Mail/IEMailTemplate.php',
     'OCP\\Mail\\IMailer' => $baseDir . '/lib/public/Mail/IMailer.php',
+    'OCP\\Migration\\BigIntMigration' => $baseDir . '/lib/public/Migration/BigIntMigration.php',
     'OCP\\Migration\\IMigrationStep' => $baseDir . '/lib/public/Migration/IMigrationStep.php',
     'OCP\\Migration\\IOutput' => $baseDir . '/lib/public/Migration/IOutput.php',
     'OCP\\Migration\\IRepairStep' => $baseDir . '/lib/public/Migration/IRepairStep.php',
@@ -498,6 +499,7 @@ return array(
     'OC\\Core\\Migrations\\Version13000Date20170718121200' => $baseDir . '/core/Migrations/Version13000Date20170718121200.php',
     'OC\\Core\\Migrations\\Version13000Date20170814074715' => $baseDir . '/core/Migrations/Version13000Date20170814074715.php',
     'OC\\Core\\Migrations\\Version13000Date20170919121250' => $baseDir . '/core/Migrations/Version13000Date20170919121250.php',
+    'OC\\Core\\Migrations\\Version13000Date20170926101637' => $baseDir . '/core/Migrations/Version13000Date20170926101637.php',
     'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php',
     'OC\\DB\\AdapterMySQL' => $baseDir . '/lib/private/DB/AdapterMySQL.php',
     'OC\\DB\\AdapterOCI8' => $baseDir . '/lib/private/DB/AdapterOCI8.php',
@@ -514,6 +516,7 @@ return array(
     'OC\\DB\\MySQLMigrator' => $baseDir . '/lib/private/DB/MySQLMigrator.php',
     'OC\\DB\\MySqlTools' => $baseDir . '/lib/private/DB/MySqlTools.php',
     'OC\\DB\\NoCheckMigrator' => $baseDir . '/lib/private/DB/NoCheckMigrator.php',
+    'OC\\DB\\OCPostgreSqlPlatform' => $baseDir . '/lib/private/DB/OCPostgreSqlPlatform.php',
     'OC\\DB\\OCSqlitePlatform' => $baseDir . '/lib/private/DB/OCSqlitePlatform.php',
     'OC\\DB\\OracleConnection' => $baseDir . '/lib/private/DB/OracleConnection.php',
     'OC\\DB\\OracleMigrator' => $baseDir . '/lib/private/DB/OracleMigrator.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 4cfbbcc2e39..1828957b32a 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -252,6 +252,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OCP\\Lockdown\\ILockdownManager' => __DIR__ . '/../../..' . '/lib/public/Lockdown/ILockdownManager.php',
         'OCP\\Mail\\IEMailTemplate' => __DIR__ . '/../../..' . '/lib/public/Mail/IEMailTemplate.php',
         'OCP\\Mail\\IMailer' => __DIR__ . '/../../..' . '/lib/public/Mail/IMailer.php',
+        'OCP\\Migration\\BigIntMigration' => __DIR__ . '/../../..' . '/lib/public/Migration/BigIntMigration.php',
         'OCP\\Migration\\IMigrationStep' => __DIR__ . '/../../..' . '/lib/public/Migration/IMigrationStep.php',
         'OCP\\Migration\\IOutput' => __DIR__ . '/../../..' . '/lib/public/Migration/IOutput.php',
         'OCP\\Migration\\IRepairStep' => __DIR__ . '/../../..' . '/lib/public/Migration/IRepairStep.php',
@@ -528,6 +529,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Core\\Migrations\\Version13000Date20170718121200' => __DIR__ . '/../../..' . '/core/Migrations/Version13000Date20170718121200.php',
         'OC\\Core\\Migrations\\Version13000Date20170814074715' => __DIR__ . '/../../..' . '/core/Migrations/Version13000Date20170814074715.php',
         'OC\\Core\\Migrations\\Version13000Date20170919121250' => __DIR__ . '/../../..' . '/core/Migrations/Version13000Date20170919121250.php',
+        'OC\\Core\\Migrations\\Version13000Date20170926101637' => __DIR__ . '/../../..' . '/core/Migrations/Version13000Date20170926101637.php',
         'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php',
         'OC\\DB\\AdapterMySQL' => __DIR__ . '/../../..' . '/lib/private/DB/AdapterMySQL.php',
         'OC\\DB\\AdapterOCI8' => __DIR__ . '/../../..' . '/lib/private/DB/AdapterOCI8.php',
@@ -544,6 +546,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\DB\\MySQLMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/MySQLMigrator.php',
         'OC\\DB\\MySqlTools' => __DIR__ . '/../../..' . '/lib/private/DB/MySqlTools.php',
         'OC\\DB\\NoCheckMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/NoCheckMigrator.php',
+        'OC\\DB\\OCPostgreSqlPlatform' => __DIR__ . '/../../..' . '/lib/private/DB/OCPostgreSqlPlatform.php',
         'OC\\DB\\OCSqlitePlatform' => __DIR__ . '/../../..' . '/lib/private/DB/OCSqlitePlatform.php',
         'OC\\DB\\OracleConnection' => __DIR__ . '/../../..' . '/lib/private/DB/OracleConnection.php',
         'OC\\DB\\OracleMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/OracleMigrator.php',
diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php
index d90d7737d40..8848f6db3b4 100644
--- a/lib/private/DB/ConnectionFactory.php
+++ b/lib/private/DB/ConnectionFactory.php
@@ -137,6 +137,10 @@ class ConnectionFactory {
 				}
 				unset($additionalConnectionParams['host']);
 				break;
+
+			case 'pgsql':
+				$additionalConnectionParams['platform'] = new OCPostgreSqlPlatform();
+				break;
 			case 'sqlite3':
 				$journalMode = $additionalConnectionParams['sqlite.journal_mode'];
 				$additionalConnectionParams['platform'] = new OCSqlitePlatform();
diff --git a/lib/private/DB/OCPostgreSqlPlatform.php b/lib/private/DB/OCPostgreSqlPlatform.php
new file mode 100644
index 00000000000..524a62c2beb
--- /dev/null
+++ b/lib/private/DB/OCPostgreSqlPlatform.php
@@ -0,0 +1,94 @@
+<?php
+/**
+ * @author Victor Dubiniuk <dubiniuk@owncloud.com>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\DB;
+
+use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
+use Doctrine\DBAL\Schema\ColumnDiff;
+use Doctrine\DBAL\Schema\TableDiff;
+use Doctrine\DBAL\Types\Type;
+
+class OCPostgreSqlPlatform extends PostgreSqlPlatform {
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function getAlterTableSQL(TableDiff $diff){
+		$queries = parent::getAlterTableSQL($diff);
+		foreach ($queries as $index => $sql){
+			// BIGSERIAL could not be used in statements altering column type
+			// That's why we replace it with BIGINT 
+			// see https://github.com/owncloud/core/pull/28364#issuecomment-315006853
+			if (preg_match('|(ALTER TABLE\s+\S+\s+ALTER\s+\S+\s+TYPE\s+)(BIGSERIAL)|i', $sql, $matches)) {
+				$alterTable = $matches[1];
+				$queries[$index] = $alterTable . 'BIGINT';
+			}
+
+			// Changing integer to bigint kills next autoincrement value
+			// see https://github.com/owncloud/core/pull/28364#issuecomment-315006853
+			if (preg_match('|ALTER TABLE\s+(\S+)\s+ALTER\s+(\S+)\s+DROP DEFAULT|i', $sql, $matches)) {
+				$queryColumnName = $matches[2];
+				$columnDiff = $this->findColumnDiffByName($diff, $queryColumnName);
+				if ($columnDiff && $this->shouldSkipDropDefault($columnDiff)) {
+					unset($queries[$index]);
+					continue;
+				}
+			}
+		}
+		
+		return $queries;
+	}
+
+	/**
+	 * We should NOT drop next sequence value if
+	 * - type was changed from INTEGER to BIGINT
+	 * - column keeps an autoincrement
+	 * - default value is kept NULL
+	 *
+	 * @param ColumnDiff $columnDiff
+	 * @return bool
+	 */
+	private function shouldSkipDropDefault(ColumnDiff $columnDiff) {
+		$column = $columnDiff->column;
+		$fromColumn = $columnDiff->fromColumn;
+		return $fromColumn->getType()->getName() === Type::INTEGER
+				&& $column->getType()->getName() === Type::BIGINT
+				&& $fromColumn->getDefault() === null
+				&& $column->getDefault() === null
+				&& $fromColumn->getAutoincrement()
+				&& $column->getAutoincrement();
+	}
+
+	/**
+	 * @param TableDiff $diff
+	 * @param string $name
+	 * @return  ColumnDiff | false
+	 */
+	private function findColumnDiffByName(TableDiff $diff, $name) {
+		foreach ($diff->changedColumns as $columnDiff) {
+			$oldColumnName = $columnDiff->getOldColumnName()->getQuotedName($this);
+			if ($oldColumnName === $name) {
+				return $columnDiff;
+			}
+		}
+		return false;
+	}
+}
diff --git a/lib/public/Migration/BigIntMigration.php b/lib/public/Migration/BigIntMigration.php
new file mode 100644
index 00000000000..91722671860
--- /dev/null
+++ b/lib/public/Migration/BigIntMigration.php
@@ -0,0 +1,66 @@
+<?php
+/**
+ * @copyright Copyright (c) 2017 Joas Schilling <coding@schilljs.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCP\Migration;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Type;
+
+/**
+ * @since 13.0.0
+ */
+abstract class BigIntMigration extends SimpleMigrationStep {
+
+	/**
+	 * @return array Returns an array with the following structure
+	 * ['table1' => ['column1', 'column2'], ...]
+	 * @since 13.0.0
+	 */
+	abstract protected function getColumnsByTable();
+
+	/**
+	 * @param IOutput $output
+	 * @param \Closure $schemaClosure The `\Closure` returns a `Schema`
+	 * @param array $options
+	 * @return null|Schema
+	 * @since 13.0.0
+	 */
+	public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
+		/** @var Schema $schema */
+		$schema = $schemaClosure();
+
+		$tables = $this->getColumnsByTable();
+
+		foreach ($tables as $tableName => $columns) {
+			$table = $schema->getTable($tableName);
+
+			foreach ($columns as $columnName) {
+				$column = $table->getColumn($columnName);
+				if ($column->getType()->getName() !== Type::BIGINT) {
+					$column->setType(Type::getType(Type::BIGINT));
+					$column->setOptions(['length' => 20]);
+				}
+			}
+		}
+
+		return $schema;
+	}
+}
diff --git a/tests/lib/DB/OCPostgreSqlPlatformTest.php b/tests/lib/DB/OCPostgreSqlPlatformTest.php
new file mode 100644
index 00000000000..56fab621cfc
--- /dev/null
+++ b/tests/lib/DB/OCPostgreSqlPlatformTest.php
@@ -0,0 +1,74 @@
+<?php
+/**
+ * @author Victor Dubiniuk <dubiniuk@owncloud.com>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+ 
+namespace Test\DB;
+
+use Doctrine\DBAL\Schema\Comparator;
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Schema\TableDiff;
+use Doctrine\DBAL\Types\Type;
+use OC\DB\OCPostgreSqlPlatform;
+ 
+ /**
+ * Class OCPostgreSqlPlatformTest
+ *
+ * @group DB
+ *
+ * @package Test\DB
+ */
+ 
+class OCPostgreSqlPlatformTest extends \Test\TestCase {
+
+	public function testAlterBigint(){
+		$platform = new OCPostgreSqlPlatform();
+		$sourceSchema = new Schema();
+		$targetSchema = new Schema();
+		
+		$this->createTableAndColumn($sourceSchema, Type::INTEGER);
+		$this->createTableAndColumn($targetSchema, Type::BIGINT);
+		
+		$comparator = new Comparator();
+		$diff = $comparator->compare($sourceSchema, $targetSchema);
+		$sqlStatements = $diff->toSql($platform);
+		$this->assertContains(
+			'ALTER TABLE poor_yorick ALTER id TYPE BIGINT',
+			$sqlStatements,
+			true
+		);
+		
+		$this->assertNotContains(
+			'ALTER TABLE poor_yorick ALTER id DROP DEFAULT',
+			$sqlStatements,
+			true
+		);
+	}
+	
+	protected function createTableAndColumn($schema, $type){
+		$table = $schema->createTable("poor_yorick");
+		$table->addColumn('id', $type, [
+			'autoincrement' => true,
+				'unsigned' => true,
+				'notnull' => true,
+				'length' => 11,
+		]);
+	}
+	
+}
diff --git a/version.php b/version.php
index 7e0fba9f6aa..1ae5b87effe 100644
--- a/version.php
+++ b/version.php
@@ -26,7 +26,7 @@
 // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel
 // when updating major/minor version number.
 
-$OC_Version = array(13, 0, 0, 4);
+$OC_Version = array(13, 0, 0, 5);
 
 // The human readable string
 $OC_VersionString = '13.0.0 alpha';