Create primary keys on all tables and add a command to create the afterwards

Signed-off-by: Joas Schilling <coding@schilljs.com>

8 files changed, 280 insertions, 6 deletions

diff --git a/core/Application.php b/core/Application.php
index a452ca9677f..d9d6b92a2ad 100644
--- a/core/Application.php
+++ b/core/Application.php
@@ -42,6 +42,7 @@ use OC\Authentication\Notifications\Notifier as AuthenticationNotifier;
 use OC\Core\Notification\RemoveLinkSharesNotifier;
 use OC\DB\MissingColumnInformation;
 use OC\DB\MissingIndexInformation;
+use OC\DB\MissingPrimaryKeyInformation;
 use OC\DB\SchemaWrapper;
 use OCP\AppFramework\App;
 use OCP\EventDispatcher\IEventDispatcher;
@@ -181,6 +182,63 @@ class Application extends App {
 			}
 		);
 
+		$oldEventDispatcher->addListener(IDBConnection::CHECK_MISSING_PRIMARY_KEYS_EVENT,
+			function (GenericEvent $event) use ($container) {
+				/** @var MissingPrimaryKeyInformation $subject */
+				$subject = $event->getSubject();
+
+				$schema = new SchemaWrapper($container->query(IDBConnection::class));
+
+				if ($schema->hasTable('federated_reshares')) {
+					$table = $schema->getTable('federated_reshares');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+
+				if ($schema->hasTable('systemtag_object_mapping')) {
+					$table = $schema->getTable('systemtag_object_mapping');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+
+				if ($schema->hasTable('comments_read_markers')) {
+					$table = $schema->getTable('comments_read_markers');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+
+				if ($schema->hasTable('collres_resources')) {
+					$table = $schema->getTable('collres_resources');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+
+				if ($schema->hasTable('collres_accesscache')) {
+					$table = $schema->getTable('collres_accesscache');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+
+				if ($schema->hasTable('filecache_extended')) {
+					$table = $schema->getTable('filecache_extended');
+
+					if (!$table->hasPrimaryKey()) {
+						$subject->addHintForMissingSubject($table->getName());
+					}
+				}
+			}
+		);
+
 		$oldEventDispatcher->addListener(IDBConnection::CHECK_MISSING_COLUMNS_EVENT,
 			function (GenericEvent $event) use ($container) {
 				/** @var MissingColumnInformation $subject */
diff --git a/core/Command/Db/AddMissingPrimaryKeys.php b/core/Command/Db/AddMissingPrimaryKeys.php
new file mode 100644
index 00000000000..a51249b1b8b
--- /dev/null
+++ b/core/Command/Db/AddMissingPrimaryKeys.php
@@ -0,0 +1,181 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2017 Bjoern Schiessle <bjoern@schiessle.org>
+ *
+ * @author Bjoern Schiessle <bjoern@schiessle.org>
+ * @author Joas Schilling <coding@schilljs.com>
+ * @author Mario Danic <mario@lovelyhq.com>
+ * @author Morris Jobke <hey@morrisjobke.de>
+ * @author Robin Appelman <robin@icewind.nl>
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ * @author Thomas Citharel <nextcloud@tcit.fr>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\Db;
+
+use OC\DB\SchemaWrapper;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\EventDispatcher\GenericEvent;
+
+/**
+ * Class AddMissingPrimaryKeys
+ *
+ * if you added primary keys to the database, this is the right place to add
+ * your update routine for existing instances
+ *
+ * @package OC\Core\Command\Db
+ */
+class AddMissingPrimaryKeys extends Command {
+
+	/** @var IDBConnection */
+	private $connection;
+
+	/** @var EventDispatcherInterface */
+	private $dispatcher;
+
+	public function __construct(IDBConnection $connection, EventDispatcherInterface $dispatcher) {
+		parent::__construct();
+
+		$this->connection = $connection;
+		$this->dispatcher = $dispatcher;
+	}
+
+	protected function configure() {
+		$this
+			->setName('db:add-missing-primary-keys')
+			->setDescription('Add missing primary keys to the database tables');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$this->addCorePrimaryKeys($output);
+
+		// Dispatch event so apps can also update indexes if needed
+		$event = new GenericEvent($output);
+		$this->dispatcher->dispatch(IDBConnection::ADD_MISSING_PRIMARY_KEYS_EVENT, $event);
+		return 0;
+	}
+
+	/**
+	 * add missing indices to the share table
+	 *
+	 * @param OutputInterface $output
+	 * @throws \Doctrine\DBAL\Schema\SchemaException
+	 */
+	private function addCorePrimaryKeys(OutputInterface $output) {
+		$output->writeln('<info>Check primary keys.</info>');
+
+		$schema = new SchemaWrapper($this->connection);
+		$updated = false;
+
+		if ($schema->hasTable('federated_reshares')) {
+			$table = $schema->getTable('federated_reshares');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the federated_reshares table, this can take some time...</info>');
+				$table->setPrimaryKey(['share_id'], 'federated_res_pk');
+				if ($table->hasIndex('share_id_index')) {
+					$table->dropIndex('share_id_index');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>federated_reshares table updated successfully.</info>');
+			}
+		}
+
+		if ($schema->hasTable('systemtag_object_mapping')) {
+			$table = $schema->getTable('systemtag_object_mapping');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the systemtag_object_mapping table, this can take some time...</info>');
+				$table->setPrimaryKey(['objecttype', 'objectid', 'systemtagid'], 'som_pk');
+				if ($table->hasIndex('mapping')) {
+					$table->dropIndex('mapping');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>systemtag_object_mapping table updated successfully.</info>');
+			}
+		}
+
+		if ($schema->hasTable('comments_read_markers')) {
+			$table = $schema->getTable('comments_read_markers');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the comments_read_markers table, this can take some time...</info>');
+				$table->setPrimaryKey(['user_id', 'object_type', 'object_id'], 'crm_pk');
+				if ($table->hasIndex('comments_marker_index')) {
+					$table->dropIndex('comments_marker_index');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>comments_read_markers table updated successfully.</info>');
+			}
+		}
+
+		if ($schema->hasTable('collres_resources')) {
+			$table = $schema->getTable('collres_resources');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the collres_resources table, this can take some time...</info>');
+				$table->setPrimaryKey(['collection_id', 'resource_type', 'resource_id'], 'crr_pk');
+				if ($table->hasIndex('collres_unique_res')) {
+					$table->dropIndex('collres_unique_res');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>collres_resources table updated successfully.</info>');
+			}
+		}
+
+		if ($schema->hasTable('collres_accesscache')) {
+			$table = $schema->getTable('collres_accesscache');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the collres_accesscache table, this can take some time...</info>');
+				$table->setPrimaryKey(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'cra_pk');
+				if ($table->hasIndex('collres_unique_user')) {
+					$table->dropIndex('collres_unique_user');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>collres_accesscache table updated successfully.</info>');
+			}
+		}
+
+		if ($schema->hasTable('filecache_extended')) {
+			$table = $schema->getTable('filecache_extended');
+			if (!$table->hasPrimaryKey()) {
+				$output->writeln('<info>Adding primary key to the filecache_extended table, this can take some time...</info>');
+				$table->setPrimaryKey(['fileid'], 'fce_pk');
+				if ($table->hasIndex('fce_fileid_idx')) {
+					$table->dropIndex('fce_fileid_idx');
+				}
+				$this->connection->migrateToSchema($schema->getWrappedSchema());
+				$updated = true;
+				$output->writeln('<info>filecache_extended table updated successfully.</info>');
+			}
+		}
+
+		if (!$updated) {
+			$output->writeln('<info>Done.</info>');
+		}
+	}
+}
diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php
index ba5f72588a6..2a47bb52cfc 100644
--- a/core/Migrations/Version13000Date20170718121200.php
+++ b/core/Migrations/Version13000Date20170718121200.php
@@ -673,7 +673,8 @@ class Version13000Date20170718121200 extends SimpleMigrationStep {
 				'default' => 0,
 				'unsigned' => true,
 			]);
-			$table->addUniqueIndex(['objecttype', 'objectid', 'systemtagid'], 'mapping');
+			$table->setPrimaryKey(['objecttype', 'objectid', 'systemtagid'], 'som_pk');
+//			$table->addUniqueIndex(['objecttype', 'objectid', 'systemtagid'], 'mapping');
 		}
 
 		if (!$schema->hasTable('systemtag_group')) {
@@ -808,7 +809,8 @@ class Version13000Date20170718121200 extends SimpleMigrationStep {
 				'default' => '',
 			]);
 			$table->addIndex(['object_type', 'object_id'], 'comments_marker_object_index');
-			$table->addUniqueIndex(['user_id', 'object_type', 'object_id'], 'comments_marker_index');
+			$table->setPrimaryKey(['user_id', 'object_type', 'object_id'], 'crm_pk');
+//			$table->addUniqueIndex(['user_id', 'object_type', 'object_id'], 'comments_marker_index');
 		}
 
 //		if (!$schema->hasTable('credentials')) {
diff --git a/core/Migrations/Version16000Date20190207141427.php b/core/Migrations/Version16000Date20190207141427.php
index 63c6c871eae..77a41dc3c4f 100644
--- a/core/Migrations/Version16000Date20190207141427.php
+++ b/core/Migrations/Version16000Date20190207141427.php
@@ -77,7 +77,8 @@ class Version16000Date20190207141427 extends SimpleMigrationStep {
 				'length' => 64,
 			]);
 
-			$table->addUniqueIndex(['collection_id', 'resource_type', 'resource_id'], 'collres_unique_res');
+			$table->setPrimaryKey(['collection_id', 'resource_type', 'resource_id'], 'crr_pk');
+//			$table->addUniqueIndex(['collection_id', 'resource_type', 'resource_id'], 'collres_unique_res');
 		}
 
 		if (!$schema->hasTable('collres_accesscache')) {
@@ -106,7 +107,8 @@ class Version16000Date20190207141427 extends SimpleMigrationStep {
 				'default' => 0,
 			]);
 
-			$table->addUniqueIndex(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'collres_unique_user');
+			$table->setPrimaryKey(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'cra_pk');
+//			$table->addUniqueIndex(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'collres_unique_user');
 			$table->addIndex(['user_id', 'resource_type', 'resource_id'], 'collres_user_res');
 			$table->addIndex(['user_id', 'collection_id'], 'collres_user_coll');
 		}
diff --git a/core/Migrations/Version17000Date20190514105811.php b/core/Migrations/Version17000Date20190514105811.php
index 910d8216070..bd6ab1bc8a5 100644
--- a/core/Migrations/Version17000Date20190514105811.php
+++ b/core/Migrations/Version17000Date20190514105811.php
@@ -67,7 +67,8 @@ class Version17000Date20190514105811 extends SimpleMigrationStep {
 				'length' => 20,
 				'default' => 0,
 			]);
-			$table->addUniqueIndex(['fileid'], 'fce_fileid_idx');
+			$table->setPrimaryKey(['fileid'], 'fce_pk');
+//			$table->addUniqueIndex(['fileid'], 'fce_fileid_idx');
 			$table->addIndex(['creation_time'], 'fce_ctime_idx');
 			$table->addIndex(['upload_time'], 'fce_utime_idx');
 		}
diff --git a/core/Migrations/Version18000Date20191204114856.php b/core/Migrations/Version18000Date20191204114856.php
index e9bf1221dbf..0e5f3cbe272 100644
--- a/core/Migrations/Version18000Date20191204114856.php
+++ b/core/Migrations/Version18000Date20191204114856.php
@@ -58,7 +58,6 @@ class Version18000Date20191204114856 extends SimpleMigrationStep {
 			'length' => 4000,
 		]);
 
-
 		return $schema;
 	}
 }
diff --git a/core/js/setupchecks.js b/core/js/setupchecks.js
index e8bcc72cb33..214f148fa94 100644
--- a/core/js/setupchecks.js
+++ b/core/js/setupchecks.js
@@ -357,6 +357,21 @@
 							type: OC.SetupChecks.MESSAGE_TYPE_INFO
 						})
 					}
+					if (data.missingPrimaryKeys.length > 0) {
+						var listOfMissingPrimaryKeys = "";
+						data.missingPrimaryKeys.forEach(function(element){
+							listOfMissingPrimaryKeys += "<li>";
+							listOfMissingPrimaryKeys += t('core', 'Missing primary key on table "{tableName}".', element);
+							listOfMissingPrimaryKeys += "</li>";
+						});
+						messages.push({
+							msg: t(
+								'core',
+								'The database is missing some primary keys. Due to the fact that adding primary keys on big tables could take some time they were not added automatically. By running "occ db:add-missing-primary-keys" those missing primary keys could be added manually while the instance keeps running.'
+							) + "<ul>" + listOfMissingPrimaryKeys + "</ul>",
+							type: OC.SetupChecks.MESSAGE_TYPE_INFO
+						})
+					}
 					if (data.missingColumns.length > 0) {
 						var listOfMissingColumns = "";
 						data.missingColumns.forEach(function(element){
diff --git a/core/js/tests/specs/setupchecksSpec.js b/core/js/tests/specs/setupchecksSpec.js
index f4c81c6bf78..3b54a3ff66a 100644
--- a/core/js/tests/specs/setupchecksSpec.js
+++ b/core/js/tests/specs/setupchecksSpec.js
@@ -240,6 +240,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -294,6 +295,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -349,6 +351,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -402,6 +405,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -453,6 +457,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -504,6 +509,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -557,6 +563,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -608,6 +615,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: false,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -659,6 +667,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -731,6 +740,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -783,6 +793,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -835,6 +846,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -887,6 +899,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: false,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -938,6 +951,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -989,6 +1003,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {
@@ -1041,6 +1056,7 @@ describe('OC.SetupChecks tests', function() {
 					isSettimelimitAvailable: true,
 					hasFreeTypeSupport: true,
 					missingIndexes: [],
+					missingPrimaryKeys: [],
 					missingColumns: [],
 					cronErrors: [],
 					cronInfo: {