add occ command to update UUIDs (incomplete)

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>

8 files changed, 405 insertions, 28 deletions

diff --git a/apps/user_ldap/appinfo/info.xml b/apps/user_ldap/appinfo/info.xml
index 57b5fe478b6..2dae4845241 100644
--- a/apps/user_ldap/appinfo/info.xml
+++ b/apps/user_ldap/appinfo/info.xml
@@ -56,6 +56,7 @@ A user logs into Nextcloud with their LDAP or AD credentials, and is granted acc
 		<command>OCA\User_LDAP\Command\ShowConfig</command>
 		<command>OCA\User_LDAP\Command\ShowRemnants</command>
 		<command>OCA\User_LDAP\Command\TestConfig</command>
+		<command>OCA\User_LDAP\Command\UpdateUUID</command>
 	</commands>
 
 	<settings>
diff --git a/apps/user_ldap/composer/composer/ClassLoader.php b/apps/user_ldap/composer/composer/ClassLoader.php
index 0cd6055d1b7..afef3fa2ad8 100644
--- a/apps/user_ldap/composer/composer/ClassLoader.php
+++ b/apps/user_ldap/composer/composer/ClassLoader.php
@@ -149,7 +149,7 @@ class ClassLoader
 
     /**
      * @return string[] Array of classname => path
-     * @psalm-var array<string, string>
+     * @psalm-return array<string, string>
      */
     public function getClassMap()
     {
diff --git a/apps/user_ldap/composer/composer/autoload_classmap.php b/apps/user_ldap/composer/composer/autoload_classmap.php
index ed8d535a6c5..cffb2aaa9fe 100644
--- a/apps/user_ldap/composer/composer/autoload_classmap.php
+++ b/apps/user_ldap/composer/composer/autoload_classmap.php
@@ -20,6 +20,7 @@ return array(
     'OCA\\User_LDAP\\Command\\ShowConfig' => $baseDir . '/../lib/Command/ShowConfig.php',
     'OCA\\User_LDAP\\Command\\ShowRemnants' => $baseDir . '/../lib/Command/ShowRemnants.php',
     'OCA\\User_LDAP\\Command\\TestConfig' => $baseDir . '/../lib/Command/TestConfig.php',
+    'OCA\\User_LDAP\\Command\\UpdateUUID' => $baseDir . '/../lib/Command/UpdateUUID.php',
     'OCA\\User_LDAP\\Configuration' => $baseDir . '/../lib/Configuration.php',
     'OCA\\User_LDAP\\Connection' => $baseDir . '/../lib/Connection.php',
     'OCA\\User_LDAP\\ConnectionFactory' => $baseDir . '/../lib/ConnectionFactory.php',
diff --git a/apps/user_ldap/composer/composer/autoload_static.php b/apps/user_ldap/composer/composer/autoload_static.php
index 9ce20914307..5928ff78ef0 100644
--- a/apps/user_ldap/composer/composer/autoload_static.php
+++ b/apps/user_ldap/composer/composer/autoload_static.php
@@ -35,6 +35,7 @@ class ComposerStaticInitUser_LDAP
         'OCA\\User_LDAP\\Command\\ShowConfig' => __DIR__ . '/..' . '/../lib/Command/ShowConfig.php',
         'OCA\\User_LDAP\\Command\\ShowRemnants' => __DIR__ . '/..' . '/../lib/Command/ShowRemnants.php',
         'OCA\\User_LDAP\\Command\\TestConfig' => __DIR__ . '/..' . '/../lib/Command/TestConfig.php',
+        'OCA\\User_LDAP\\Command\\UpdateUUID' => __DIR__ . '/..' . '/../lib/Command/UpdateUUID.php',
         'OCA\\User_LDAP\\Configuration' => __DIR__ . '/..' . '/../lib/Configuration.php',
         'OCA\\User_LDAP\\Connection' => __DIR__ . '/..' . '/../lib/Connection.php',
         'OCA\\User_LDAP\\ConnectionFactory' => __DIR__ . '/..' . '/../lib/ConnectionFactory.php',
diff --git a/apps/user_ldap/composer/composer/installed.php b/apps/user_ldap/composer/composer/installed.php
index f84b9c452e8..5e942064485 100644
--- a/apps/user_ldap/composer/composer/installed.php
+++ b/apps/user_ldap/composer/composer/installed.php
@@ -5,7 +5,7 @@
         'type' => 'library',
         'install_path' => __DIR__ . '/../',
         'aliases' => array(),
-        'reference' => '6b960de47cabaa7a231e72479012ba4dcbc2e882',
+        'reference' => '9915dc6785d1660068a51604f9379e8b1dc1418c',
         'name' => '__root__',
         'dev' => false,
     ),
@@ -16,7 +16,7 @@
             'type' => 'library',
             'install_path' => __DIR__ . '/../',
             'aliases' => array(),
-            'reference' => '6b960de47cabaa7a231e72479012ba4dcbc2e882',
+            'reference' => '9915dc6785d1660068a51604f9379e8b1dc1418c',
             'dev_requirement' => false,
         ),
     ),
diff --git a/apps/user_ldap/lib/Command/UpdateUUID.php b/apps/user_ldap/lib/Command/UpdateUUID.php
new file mode 100644
index 00000000000..b655fe21ed5
--- /dev/null
+++ b/apps/user_ldap/lib/Command/UpdateUUID.php
@@ -0,0 +1,365 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2021 Arthur Schiwon <blizzz@arthur-schiwon.de>
+ *
+ * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\User_LDAP\Command;
+
+use OCA\User_LDAP\Access;
+use OCA\User_LDAP\Group_Proxy;
+use OCA\User_LDAP\Mapping\AbstractMapping;
+use OCA\User_LDAP\Mapping\GroupMapping;
+use OCA\User_LDAP\Mapping\UserMapping;
+use OCA\User_LDAP\User_Proxy;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use function sprintf;
+
+class UuidUpdateReport {
+	const UNCHANGED = 0;
+	const UNKNOWN = 1;
+	const UNREADABLE = 2;
+	const UPDATED = 3;
+	const UNWRITABLE = 4;
+	const UNMAPPED = 5;
+
+	public $id = '';
+	public $dn = '';
+	public $isUser = true;
+	public $state = self::UNCHANGED;
+	public $oldUuid = '';
+	public $newUuid = '';
+
+	public function __construct(string $id, string $dn, bool $isUser, int $state, $oldUuid = '', $newUuid = '') {
+		$this->id = $id;
+		$this->dn = $dn;
+		$this->isUser = $isUser;
+		$this->state = $state;
+		$this->oldUuid = $oldUuid;
+		$this->newUuid = $newUuid;
+	}
+}
+
+class UpdateUUID extends Command {
+	/** @var UserMapping */
+	private $userMapping;
+	/** @var GroupMapping */
+	private $groupMapping;
+	/** @var User_Proxy */
+	private $userProxy;
+	/** @var Group_Proxy */
+	private $groupProxy;
+	/** @var array<UuidUpdateReport> */
+	protected $reports = [];
+	/** @var LoggerInterface */
+	private $logger;
+	/** @var bool */
+	private $dryRun = false;
+
+	public function __construct(UserMapping $userMapping, GroupMapping $groupMapping, User_Proxy $userProxy, Group_Proxy $groupProxy, LoggerInterface $logger) {
+		$this->userMapping = $userMapping;
+		$this->groupMapping = $groupMapping;
+		$this->userProxy = $userProxy;
+		$this->groupProxy = $groupProxy;
+		$this->logger = $logger;
+		$this->reports = [
+			UuidUpdateReport::UPDATED => [],
+			UuidUpdateReport::UNKNOWN => [],
+			UuidUpdateReport::UNREADABLE => [],
+			UuidUpdateReport::UNWRITABLE => [],
+			UuidUpdateReport::UNMAPPED => [],
+		];
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		$this
+			->setName('ldap:update-uuid')
+			->setDescription('Attempts to update UUIDs of user and group entries. By default, the command attempts to update UUIDs that have been invalidated by a migration step.')
+			->addOption(
+				'all',
+				null,
+				InputOption::VALUE_NONE,
+				'updates every user and group. All other options are ignored.'
+			)
+			->addOption(
+				'userId',
+				null,
+				InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY,
+				'a user ID to update'
+			)
+			->addOption(
+				'groupId',
+				null,
+				InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY,
+				'a group ID to update'
+			)
+			->addOption(
+				'dn',
+				null,
+				InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY,
+				'a DN to update'
+			)
+			->addOption(
+				'dry-run',
+				null,
+				InputOption::VALUE_NONE,
+				'UUIDs will not be updated in the database'
+			)
+		;
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$this->dryRun = $input->getOption('dry-run');
+		$entriesToUpdates = $this->estimateNumberOfUpdates($input);
+		$progressBar = new ProgressBar($output);
+		$progressBar->iterate($this->handleUpdates($input), $entriesToUpdates);
+		$this->printReport($input, $output);
+		$this->printReport($output);
+		return count($this->reports[UuidUpdateReport::UNMAPPED]) === 0
+			&& count($this->reports[UuidUpdateReport::UNREADABLE]) === 0
+			&& count($this->reports[UuidUpdateReport::UNWRITABLE]) === 0
+			? 0
+			: 1;
+	}
+
+	protected function printReport(OutputInterface $output) {
+		if ($output->isQuiet()) {
+			return;
+		}
+
+		if (count($this->reports[UuidUpdateReport::UPDATED]) === 0) {
+			$output->writeln('<info>No record was updated.</info>');
+		} else {
+			$output->writeln(sprintf('<info>%d record(s) were updated.</info>', count($this->reports[UuidUpdateReport::UPDATED])));
+			if ($output->isVerbose()) {
+				/** @var UuidUpdateReport $report */
+				foreach ($this->reports[UuidUpdateReport::UPDATED] as $report) {
+					$output->writeln(sprintf('  %s had their old UUID %s updated to %s', $report->id, $report->oldUuid, $report->newUuid));
+				}
+				$output->writeln('');
+			}
+		}
+
+		if (count($this->reports[UuidUpdateReport::UNMAPPED]) > 0) {
+			$output->writeln(sprintf('<error>%d provided IDs were not mapped. These were:</error>', count($this->reports[UuidUpdateReport::UNMAPPED])));
+			/** @var UuidUpdateReport $report */
+			foreach ($this->reports[UuidUpdateReport::UNMAPPED] as $report) {
+				if (!empty($report->id)) {
+					$output->writeln(sprintf('  %s: %s',
+						$report->isUser ? 'User' : 'Group', $report->id));
+				} else if (!empty($report->dn)) {
+					$output->writeln(sprintf('  DN: %s', $report->dn));
+				}
+			}
+			$output->writeln('');
+		}
+
+		if (count($this->reports[UuidUpdateReport::UNKNOWN]) > 0) {
+			$output->writeln(sprintf('<info>%d provided IDs were unknown on LDAP.</info>', count($this->reports[UuidUpdateReport::UNKNOWN])));
+			if ($output->isVerbose()) {
+				/** @var UuidUpdateReport $report */
+				foreach ($this->reports[UuidUpdateReport::UNKNOWN] as $report) {
+					$output->writeln(sprintf('  %s: %s',$report->isUser ? 'User' : 'Group', $report->id));
+				}
+				$output->writeln(PHP_EOL . 'Old users can be removed along with their data per occ user:delete.' . PHP_EOL);
+			}
+		}
+
+		if (count($this->reports[UuidUpdateReport::UNREADABLE]) > 0) {
+			$output->writeln(sprintf('<error>For %d records, the UUID could not be read. Double-check your configuration.</error>', count($this->reports[UuidUpdateReport::UNREADABLE])));
+			if ($output->isVerbose()) {
+				/** @var UuidUpdateReport $report */
+				foreach ($this->reports[UuidUpdateReport::UNREADABLE] as $report) {
+					$output->writeln(sprintf('  %s: %s',$report->isUser ? 'User' : 'Group', $report->id));
+				}
+			}
+		}
+
+		if (count($this->reports[UuidUpdateReport::UNWRITABLE]) > 0) {
+			$output->writeln(sprintf('<error>For %d records, the UUID could not be saved to database. Double-check your configuration.</error>', count($this->reports[UuidUpdateReport::UNWRITABLE])));
+			if ($output->isVerbose()) {
+				/** @var UuidUpdateReport $report */
+				foreach ($this->reports[UuidUpdateReport::UNWRITABLE] as $report) {
+					$output->writeln(sprintf('  %s: %s',$report->isUser ? 'User' : 'Group', $report->id));
+				}
+			}
+		}
+	}
+
+	protected function handleUpdates(InputInterface $input): \Generator {
+		if ($input->getOption('all')) {
+			return $this->handleMappingBasedUpdates(false);
+		} else if ($input->getOption('userId')
+			|| $input->getOption('groupId')
+			|| $input->getOption('dn')
+		) {
+			while($this->handleUpdatesByUserId($input->getOption('userId'))) {
+				yield;
+			}
+			while($this->handleUpdatesByUserId($input->getOption('groupId'))) {
+				yield;
+			}
+			while($this->handleUpdatesByDN($input->getOption('dn'))) {
+				yield;
+			}
+		} else {
+			return $this->handleMappingBasedUpdates(true);
+		}
+	}
+
+	protected function handleUpdatesByUserId(array $userIds): \Generator {
+		while($this->handleUpdatesByEntryId($userIds, $this->userMapping)) {
+			yield;
+		}
+	}
+
+	protected function handleUpdatesByGroupId(array $groupIds): \Generator {
+		while($this->handleUpdatesByEntryId($groupIds, $this->groupMapping)) {
+			yield;
+		}
+	}
+
+	protected function handleUpdatesByDN(array $dns): \Generator {
+		$userList = $groupList = [];
+		while ($dn = array_pop($dns)) {
+			$uuid = $this->userMapping->getUUIDByDN($dn);
+			if ($uuid) {
+				$id = $this->userMapping->getNameByDN($dn);
+				$userList[] = ['name' => $id, 'uuid' => $uuid];
+				continue;
+			}
+			$uuid = $this->groupMapping->getUUIDByDN($dn);
+			if ($uuid) {
+				$id = $this->groupMapping->getNameByDN($dn);
+				$groupList[] = ['name' => $id, 'uuid' => $uuid];
+				continue;
+			}
+			$this->reports[UuidUpdateReport::UNMAPPED][] = new UuidUpdateReport('', $dn, true, UuidUpdateReport::UNMAPPED);
+			yield;
+		}
+		while($this->handleUpdatesByList($this->userMapping, $userList)) {
+			yield;
+		}
+		while($this->handleUpdatesByList($this->groupMapping, $groupList)) {
+			yield;
+		}
+	}
+
+	protected function handleUpdatesByEntryId(array $ids, AbstractMapping $mapping): \Generator {
+		$isUser = $mapping instanceof UserMapping;
+		$list = [];
+		while ($id = array_pop($ids)) {
+			if(!$dn = $mapping->getDNByName($id)) {
+				$this->reports[UuidUpdateReport::UNMAPPED][] = new UuidUpdateReport($id, '', $isUser, UuidUpdateReport::UNMAPPED);
+				yield;
+				continue;
+			}
+			// Since we know it was mapped the UUID is populated
+			$uuid = $mapping->getUUIDByDN($dn);
+			$list[] = ['name' => $id, 'uuid' => $uuid];
+		}
+		while($this->handleUpdatesByList($mapping, $list)) {
+			yield;
+		}
+	}
+
+	protected function handleMappingBasedUpdates(bool $invalidatedOnly): \Generator {
+		$limit = 1000;
+		/** @var AbstractMapping $mapping*/
+		foreach([$this->userMapping, $this->groupMapping] as $mapping) {
+			$offset = 0;
+			do {
+				$list = $mapping->getList($offset, $limit, $invalidatedOnly);
+				$offset += $limit;
+
+				foreach($this->handleUpdatesByList($mapping, $list) as $tick) {
+					yield; // null, for it only advances progress counter
+				}
+			} while (count($list) === $limit);
+		}
+	}
+
+	protected function handleUpdatesByList(AbstractMapping $mapping, array $list): \Generator {
+		if ($mapping instanceof UserMapping) {
+			$isUser = true;
+			$backendProxy = $this->userProxy;
+		} else {
+			$isUser = false;
+			$backendProxy = $this->groupProxy;
+		}
+
+		foreach ($list as $row) {
+			$access = $backendProxy->getLDAPAccess($row['name']);
+			if ($access instanceof Access
+				&& $dn = $mapping->getDNByName($row['name']))
+			{
+				if ($uuid = $access->getUUID($dn, $isUser)) {
+					if ($uuid !== $row['uuid']) {
+						if ($this->dryRun || $mapping->setUUIDbyDN($uuid, $dn)) {
+							$this->reports[UuidUpdateReport::UPDATED][]
+								= new UuidUpdateReport($row['name'], $dn, $isUser, UuidUpdateReport::UPDATED, $row['uuid'], $uuid);
+						} else {
+							$this->reports[UuidUpdateReport::UNWRITABLE][]
+								= new UuidUpdateReport($row['name'], $dn, $isUser, UuidUpdateReport::UNWRITABLE, $row['uuid'], $uuid);
+						}
+						$this->logger->info('UUID of {id} was updated from {from} to {to}',
+							[
+								'appid' => 'user_ldap',
+								'id' => $row['name'],
+								'from' => $row['uuid'],
+								'to' => $uuid,
+							]
+						);
+					}
+				} else {
+					$this->reports[UuidUpdateReport::UNREADABLE][] = new UuidUpdateReport($row['name'], $dn, $isUser, UuidUpdateReport::UNREADABLE);
+				}
+			} else {
+				$this->reports[UuidUpdateReport::UNKNOWN][] = new UuidUpdateReport($row['name'], '', $isUser, UuidUpdateReport::UNKNOWN);
+			}
+			yield; // null, for it only advances progress counter
+		}
+	}
+
+	protected function estimateNumberOfUpdates(InputInterface $input) {
+		if ($input->getOption('all')) {
+			return $this->userMapping->count() + $this->groupMapping->count();
+		} else if ($input->getOption('userId')
+			|| $input->getOption('groupId')
+			|| $input->getOption('dn')
+		) {
+			return count($input->getOption('userId'))
+				+ count($input->getOption('groupId'))
+				+ count($input->getOption('dn'));
+		} else {
+			return $this->userMapping->countInvalidated() + $this->groupMapping->countInvalidated();
+		}
+	}
+
+}
diff --git a/apps/user_ldap/lib/Mapping/AbstractMapping.php b/apps/user_ldap/lib/Mapping/AbstractMapping.php
index 1d3e1a221f7..653d0e5fbc4 100644
--- a/apps/user_ldap/lib/Mapping/AbstractMapping.php
+++ b/apps/user_ldap/lib/Mapping/AbstractMapping.php
@@ -175,7 +175,7 @@ abstract class AbstractMapping {
 	 * @param $fdn
 	 * @return bool
 	 */
-	public function setUUIDbyDN($uuid, $fdn) {
+	public function setUUIDbyDN($uuid, $fdn): bool {
 		$statement = $this->dbc->prepare('
 			UPDATE `' . $this->getTableName() . '`
 			SET `directory_uuid` = ?
@@ -329,26 +329,24 @@ abstract class AbstractMapping {
 		return $this->getXbyY('directory_uuid', 'ldap_dn_hash', $this->getDNHash($dn));
 	}
 
-	/**
-	 * gets a piece of the mapping list
-	 *
-	 * @param int $offset
-	 * @param int $limit
-	 * @return array
-	 */
-	public function getList($offset = null, $limit = null) {
-		$query = $this->dbc->prepare('
-			SELECT
-				`ldap_dn` AS `dn`,
-				`owncloud_name` AS `name`,
-				`directory_uuid` AS `uuid`
-			FROM `' . $this->getTableName() . '`',
-			$limit,
-			$offset
-		);
-
-		$query->execute();
-		return $query->fetchAll();
+	public function getList(int $offset = null, int $limit = null, $invalidatedOnly = false): array {
+		$select = $this->dbc->getQueryBuilder();
+		$select->selectAlias('ldap_dn', 'dn')
+			->selectAlias('owncloud_name', 'name')
+			->selectAlias('directory_uuid', 'uuid')
+			->from($this->getTableName(false))
+			->setMaxResults($limit)
+			->setFirstResult($offset);
+
+		if ($invalidatedOnly) {
+			$select->where($select->expr()->like('directory_uuid', $select->createNamedParameter('invalidated_%')));
+		}
+
+		$result = $select->executeQuery();
+		$entries = $result->fetchAll();
+		$result->closeCursor();
+
+		return $entries;
 	}
 
 	/**
@@ -458,13 +456,24 @@ abstract class AbstractMapping {
 	 *
 	 * @return int
 	 */
-	public function count() {
-		$qb = $this->dbc->getQueryBuilder();
-		$query = $qb->select($qb->func()->count('ldap_dn_hash'))
+	public function count(): int {
+		$query = $this->dbc->getQueryBuilder();
+		$query->select($query->func()->count('ldap_dn_hash'))
 			->from($this->getTableName());
 		$res = $query->execute();
 		$count = $res->fetchOne();
 		$res->closeCursor();
 		return (int)$count;
 	}
+
+	public function countInvalidated(): int {
+		$query = $this->dbc->getQueryBuilder();
+		$query->select($query->func()->count('ldap_dn_hash'))
+			->from($this->getTableName())
+			->where($query->expr()->like('directory_uuid', $query->createNamedParameter('invalidated_%')));
+		$res = $query->execute();
+		$count = $res->fetchOne();
+		$res->closeCursor();
+		return (int)$count;
+	}
 }
diff --git a/apps/user_ldap/tests/Mapping/AbstractMappingTest.php b/apps/user_ldap/tests/Mapping/AbstractMappingTest.php
index 9c25b1d9af6..0d21172445f 100644
--- a/apps/user_ldap/tests/Mapping/AbstractMappingTest.php
+++ b/apps/user_ldap/tests/Mapping/AbstractMappingTest.php
@@ -276,7 +276,7 @@ abstract class AbstractMappingTest extends \Test\TestCase {
 		$this->assertSame(count($data) - 1, count($results));
 
 		// get first 2 entries by limit, but not offset
-		$results = $mapper->getList(null, 2);
+		$results = $mapper->getList(0, 2);
 		$this->assertSame(2, count($results));
 
 		// get 2nd entry by specifying both offset and limit