feat: Move mimetype repair to background jobsenh/repair-mimetype-job

Signed-off-by: Julius Härtl <jus@bitgrid.net>

5 files changed, 341 insertions, 3 deletions

diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index bd4ff298f59..c6ca8727aa1 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -1499,6 +1499,7 @@ return array(
     'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php',
     'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php',
     'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
+    'OC\\Repair\\AddRepairMimeTypeJob' => $baseDir . '/lib/private/Repair/AddRepairMimeTypeJob.php',
     'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php',
     'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php',
     'OC\\Repair\\ClearFrontendCaches' => $baseDir . '/lib/private/Repair/ClearFrontendCaches.php',
@@ -1540,6 +1541,7 @@ return array(
     'OC\\Repair\\RemoveLinkShares' => $baseDir . '/lib/private/Repair/RemoveLinkShares.php',
     'OC\\Repair\\RepairDavShares' => $baseDir . '/lib/private/Repair/RepairDavShares.php',
     'OC\\Repair\\RepairInvalidShares' => $baseDir . '/lib/private/Repair/RepairInvalidShares.php',
+    'OC\\Repair\\RepairMimeTypeJob' => $baseDir . '/lib/private/Repair/RepairMimeTypeJob.php',
     'OC\\Repair\\RepairMimeTypes' => $baseDir . '/lib/private/Repair/RepairMimeTypes.php',
     'OC\\Repair\\SqliteAutoincrement' => $baseDir . '/lib/private/Repair/SqliteAutoincrement.php',
     'OC\\RichObjectStrings\\Validator' => $baseDir . '/lib/private/RichObjectStrings/Validator.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 17ba4c59fbc..f90e8c112f4 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -1532,6 +1532,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php',
         'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php',
         'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
+        'OC\\Repair\\AddRepairMimeTypeJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRepairMimeTypeJob.php',
         'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php',
         'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php',
         'OC\\Repair\\ClearFrontendCaches' => __DIR__ . '/../../..' . '/lib/private/Repair/ClearFrontendCaches.php',
@@ -1573,6 +1574,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Repair\\RemoveLinkShares' => __DIR__ . '/../../..' . '/lib/private/Repair/RemoveLinkShares.php',
         'OC\\Repair\\RepairDavShares' => __DIR__ . '/../../..' . '/lib/private/Repair/RepairDavShares.php',
         'OC\\Repair\\RepairInvalidShares' => __DIR__ . '/../../..' . '/lib/private/Repair/RepairInvalidShares.php',
+        'OC\\Repair\\RepairMimeTypeJob' => __DIR__ . '/../../..' . '/lib/private/Repair/RepairMimeTypeJob.php',
         'OC\\Repair\\RepairMimeTypes' => __DIR__ . '/../../..' . '/lib/private/Repair/RepairMimeTypes.php',
         'OC\\Repair\\SqliteAutoincrement' => __DIR__ . '/../../..' . '/lib/private/Repair/SqliteAutoincrement.php',
         'OC\\RichObjectStrings\\Validator' => __DIR__ . '/../../..' . '/lib/private/RichObjectStrings/Validator.php',
diff --git a/lib/private/Repair.php b/lib/private/Repair.php
index 9c6a6cd00f2..c0f97ee6103 100644
--- a/lib/private/Repair.php
+++ b/lib/private/Repair.php
@@ -34,6 +34,7 @@
  */
 namespace OC;
 
+use OC\Repair\AddRepairMimeTypeJob;
 use OC\Repair\CleanUpAbandonedApps;
 use OCP\AppFramework\QueryException;
 use OCP\AppFramework\Utility\ITimeFactory;
@@ -174,7 +175,7 @@ class Repair implements IOutput {
 	public static function getRepairSteps(): array {
 		return [
 			new Collation(\OC::$server->getConfig(), \OC::$server->get(LoggerInterface::class), \OC::$server->getDatabaseConnection(), false),
-			new RepairMimeTypes(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()),
+			\OCP\Server::get(AddRepairMimeTypeJob::class),
 			new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()),
 			new RepairInvalidShares(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()),
 			new MoveUpdaterStepFile(\OC::$server->getConfig()),
@@ -221,8 +222,9 @@ class Repair implements IOutput {
 	 */
 	public static function getExpensiveRepairSteps() {
 		return [
-			new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()),
-			\OC::$server->get(ValidatePhoneNumber::class),
+			\OCP\Server::get(OldGroupMembershipShares::class),
+			\OCP\Server::get(ValidatePhoneNumber::class),
+			\OCP\Server::get(RepairMimeTypes::class),
 		];
 	}
 
diff --git a/lib/private/Repair/AddRepairMimeTypeJob.php b/lib/private/Repair/AddRepairMimeTypeJob.php
new file mode 100644
index 00000000000..ef0cb2cb2df
--- /dev/null
+++ b/lib/private/Repair/AddRepairMimeTypeJob.php
@@ -0,0 +1,257 @@
+<?php
+/**
+ * @copyright Copyright (c) 2016, ownCloud, Inc.
+ *
+ * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
+ * @author Christoph Wurst <christoph@winzerhof-wurst.at>
+ * @author Joas Schilling <coding@schilljs.com>
+ * @author Julius Härtl <jus@bitgrid.net>
+ * @author Morris Jobke <hey@morrisjobke.de>
+ * @author nik gaffney <nik@fo.am>
+ * @author Olivier Paroz <github@oparoz.com>
+ * @author Rello <Rello@users.noreply.github.com>
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ * @author Stefan Weil <sw@weilnetz.de>
+ * @author Thomas Ebert <thomas.ebert@usability.de>
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ * @author Vincent Petry <vincent@nextcloud.com>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+namespace OC\Repair;
+
+use OCP\BackgroundJob\IJobList;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class AddRepairMimeTypeJob implements IRepairStep {
+	protected $config;
+	protected $connection;
+	protected $jobList;
+	public function __construct(IConfig $config,
+								IDBConnection $connection, IJobList $jobList) {
+		$this->config = $config;
+		$this->connection = $connection;
+		$this->jobList = $jobList;
+	}
+
+	public function getName() {
+		return 'Repair mime types through scheduling background jobs';
+	}
+
+	private function scheduleMimeTypeUpdateJob(array $updatedMimetypes): int {
+		$qb = $this->connection->getQueryBuilder();
+		$result = $qb
+			->select('numeric_id')
+			->from('storages')
+			->execute();
+
+		$jobCount = 0;
+		while ($row = $result->fetch()) {
+			$this->jobList->add(RepairMimeTypeJob::class, [
+				'storageId' => $row['numeric_id'],
+				'mimetypes' => $updatedMimetypes,
+			]);
+			$jobCount++;
+		}
+
+		return $jobCount;
+	}
+
+	private function introduceAsciidocType() {
+		$updatedMimetypes = [
+			'adoc' => 'text/asciidoc',
+			'asciidoc' => 'text/asciidoc',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceImageTypes() {
+		$updatedMimetypes = [
+			'jp2' => 'image/jp2',
+			'webp' => 'image/webp',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceWindowsProgramTypes() {
+		$updatedMimetypes = [
+			'htaccess' => 'text/plain',
+			'bat' => 'application/x-msdos-program',
+			'cmd' => 'application/cmd',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceLocationTypes() {
+		$updatedMimetypes = [
+			'gpx' => 'application/gpx+xml',
+			'kml' => 'application/vnd.google-earth.kml+xml',
+			'kmz' => 'application/vnd.google-earth.kmz',
+			'tcx' => 'application/vnd.garmin.tcx+xml',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceInternetShortcutTypes() {
+		$updatedMimetypes = [
+			'url' => 'application/internet-shortcut',
+			'webloc' => 'application/internet-shortcut'
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceStreamingTypes() {
+		$updatedMimetypes = [
+			'm3u' => 'audio/mpegurl',
+			'm3u8' => 'audio/mpegurl',
+			'pls' => 'audio/x-scpls'
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceVisioTypes() {
+		$updatedMimetypes = [
+			'vsdm' => 'application/vnd.visio',
+			'vsdx' => 'application/vnd.visio',
+			'vssm' => 'application/vnd.visio',
+			'vssx' => 'application/vnd.visio',
+			'vstm' => 'application/vnd.visio',
+			'vstx' => 'application/vnd.visio',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceComicbookTypes() {
+		$updatedMimetypes = [
+			'cb7' => 'application/comicbook+7z',
+			'cba' => 'application/comicbook+ace',
+			'cbr' => 'application/comicbook+rar',
+			'cbt' => 'application/comicbook+tar',
+			'cbtc' => 'application/comicbook+truecrypt',
+			'cbz' => 'application/comicbook+zip',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceOpenDocumentTemplates() {
+		$updatedMimetypes = [
+			'ott' => 'application/vnd.oasis.opendocument.text-template',
+			'ots' => 'application/vnd.oasis.opendocument.spreadsheet-template',
+			'otp' => 'application/vnd.oasis.opendocument.presentation-template',
+			'otg' => 'application/vnd.oasis.opendocument.graphics-template',
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceFlatOpenDocumentType() {
+		$updatedMimetypes = [
+			"fodt" => "application/vnd.oasis.opendocument.text-flat-xml",
+			"fods" => "application/vnd.oasis.opendocument.spreadsheet-flat-xml",
+			"fodg" => "application/vnd.oasis.opendocument.graphics-flat-xml",
+			"fodp" => "application/vnd.oasis.opendocument.presentation-flat-xml",
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceOrgModeType() {
+		$updatedMimetypes = [
+			'org' => 'text/org'
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+	private function introduceOnlyofficeFormType() {
+		$updatedMimetypes = [
+			"oform" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document.oform",
+			"docxf" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document.docxf",
+		];
+
+		return $this->scheduleMimeTypeUpdateJob($updatedMimetypes);
+	}
+
+
+	/**
+	 * Fix mime types
+	 */
+	public function run(IOutput $out) {
+		$ocVersionFromBeforeUpdate = $this->config->getSystemValueString('version', '0.0.0');
+
+		// NOTE TO DEVELOPERS: when adding new mime types, please make sure to
+		// add a version comparison to avoid doing it every time
+
+		if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.14', '<') && $this->introduceImageTypes()) {
+			$out->info('Fixed image mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) {
+			$out->info('Fixed windows program mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.0', '<') && $this->introduceLocationTypes()) {
+			$out->info('Fixed geospatial mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) {
+			$out->info('Fixed internet-shortcut mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.6', '<') && $this->introduceStreamingTypes()) {
+			$out->info('Fixed streaming mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.8', '<') && $this->introduceVisioTypes()) {
+			$out->info('Fixed visio mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.10', '<') && $this->introduceComicbookTypes()) {
+			$out->info('Fixed comicbook mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) {
+			$out->info('Fixed OpenDocument template mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '21.0.0.7', '<') && $this->introduceOrgModeType()) {
+			$out->info('Fixed orgmode mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) {
+			$out->info('Fixed Flat OpenDocument mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) {
+			$out->info('Fixed ONLYOFFICE Forms OpenXML mime types');
+		}
+
+		if (version_compare($ocVersionFromBeforeUpdate, '26.0.0.1', '<') && $this->introduceAsciidocType()) {
+			$out->info('Fixed AsciiDoc mime types');
+		}
+	}
+}
diff --git a/lib/private/Repair/RepairMimeTypeJob.php b/lib/private/Repair/RepairMimeTypeJob.php
new file mode 100644
index 00000000000..cf00919f2a4
--- /dev/null
+++ b/lib/private/Repair/RepairMimeTypeJob.php
@@ -0,0 +1,75 @@
+<?php
+
+namespace OC\Repair;
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\QueuedJob;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use Psr\Log\LoggerInterface;
+
+class RepairMimeTypeJob extends QueuedJob {
+	private IDBConnection $connection;
+	private LoggerInterface $logger;
+
+	public function __construct(ITimeFactory $time, IDBConnection $connection, LoggerInterface $logger) {
+		parent::__construct($time);
+		$this->connection = $connection;
+		$this->logger = $logger;
+	}
+
+	protected function run($argument) {
+		$storageId = (int)$argument['storageId'];
+		$mimetypes = $argument['mimetypes'];
+		$this->updateMimetypesForStorage($storageId, $mimetypes);
+	}
+
+	private function updateMimetypesForStorage(int $storageId, array $updatedMimetypes) {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('id')
+			->from('mimetypes')
+			->where($query->expr()->eq('mimetype', $query->createParameter('mimetype'), IQueryBuilder::PARAM_INT));
+		$insert = $this->connection->getQueryBuilder();
+		$insert->insert('mimetypes')
+			->setValue('mimetype', $insert->createParameter('mimetype'));
+
+		if (empty($this->folderMimeTypeId)) {
+			$query->setParameter('mimetype', 'httpd/unix-directory');
+			$result = $query->execute();
+			$this->folderMimeTypeId = (int)$result->fetchOne();
+			$result->closeCursor();
+		}
+
+		$update = $this->connection->getQueryBuilder();
+		$update->update('filecache')
+			->set('mimetype', $update->createParameter('mimetype'))
+			->where($update->expr()->eq('storage', $update->createParameter('storage'), IQueryBuilder::PARAM_INT))
+			->andWhere($update->expr()->neq('mimetype', $update->createParameter('mimetype'), IQueryBuilder::PARAM_INT))
+			->andWhere($update->expr()->neq('mimetype', $update->createParameter('folder'), IQueryBuilder::PARAM_INT))
+			->andWhere($update->expr()->iLike('name', $update->createParameter('name')))
+			->setParameter('folder', $this->folderMimeTypeId)
+			->setParameter('storage', $storageId);
+
+		$count = 0;
+		foreach ($updatedMimetypes as $extension => $mimetype) {
+			// get target mimetype id
+			$query->setParameter('mimetype', $mimetype);
+			$result = $query->execute();
+			$mimetypeId = (int)$result->fetchOne();
+			$result->closeCursor();
+
+			if (!$mimetypeId) {
+				// insert mimetype
+				$insert->setParameter('mimetype', $mimetype);
+				$insert->execute();
+				$mimetypeId = $insert->getLastInsertId();
+			}
+
+			// change mimetype for files with x extension
+			$update->setParameter('mimetype', $mimetypeId)
+				->setParameter('name', '%' . $this->connection->escapeLikeParameter('.' . $extension));
+			$count = $update->execute();
+			$this->logger->info('Updated storage ' . $storageId . ' with ' . $count . ' file entry mimetypes for ' . $extension . ' to ' . $mimetype);
+		}
+	}
+}