Merge pull request #37674 from nextcloud/feature/speech-to-text

feat(SpeechToText): Add SpeechToText OCP provider API

4 files changed, 254 insertions, 0 deletions

diff --git a/lib/private/AppFramework/Bootstrap/RegistrationContext.php b/lib/private/AppFramework/Bootstrap/RegistrationContext.php
index 9a6c298419a..8fcafab2d87 100644
--- a/lib/private/AppFramework/Bootstrap/RegistrationContext.php
+++ b/lib/private/AppFramework/Bootstrap/RegistrationContext.php
@@ -33,6 +33,7 @@ use Closure;
 use OCP\Calendar\Resource\IBackend as IResourceBackend;
 use OCP\Calendar\Room\IBackend as IRoomBackend;
 use OCP\Collaboration\Reference\IReferenceProvider;
+use OCP\SpeechToText\ISpeechToTextProvider;
 use OCP\Talk\ITalkBackend;
 use OCP\Translation\ITranslationProvider;
 use RuntimeException;
@@ -111,6 +112,9 @@ class RegistrationContext {
 	/** @var ServiceRegistration<IHandler>[] */
 	private $wellKnownHandlers = [];
 
+	/** @var ServiceRegistration<ISpeechToTextProvider>[] */
+	private $speechToTextProviders = [];
+
 	/** @var ServiceRegistration<ICustomTemplateProvider>[] */
 	private $templateProviders = [];
 
@@ -252,6 +256,13 @@ class RegistrationContext {
 				);
 			}
 
+			public function registerSpeechToTextProvider(string $providerClass): void {
+				$this->context->registerSpeechToTextProvider(
+					$this->appId,
+					$providerClass
+				);
+			}
+
 			public function registerTemplateProvider(string $providerClass): void {
 				$this->context->registerTemplateProvider(
 					$this->appId,
@@ -414,6 +425,10 @@ class RegistrationContext {
 		$this->wellKnownHandlers[] = new ServiceRegistration($appId, $class);
 	}
 
+	public function registerSpeechToTextProvider(string $appId, string $class): void {
+		$this->speechToTextProviders[] = new ServiceRegistration($appId, $class);
+	}
+
 	public function registerTemplateProvider(string $appId, string $class): void {
 		$this->templateProviders[] = new ServiceRegistration($appId, $class);
 	}
@@ -686,6 +701,13 @@ class RegistrationContext {
 	}
 
 	/**
+	 * @return ServiceRegistration<ISpeechToTextProvider>[]
+	 */
+	public function getSpeechToTextProviders(): array {
+		return $this->speechToTextProviders;
+	}
+
+	/**
 	 * @return ServiceRegistration<ICustomTemplateProvider>[]
 	 */
 	public function getTemplateProviders(): array {
diff --git a/lib/private/Server.php b/lib/private/Server.php
index 78099f2a8d0..4cf278c82a9 100644
--- a/lib/private/Server.php
+++ b/lib/private/Server.php
@@ -148,6 +148,7 @@ use OC\Security\VerificationToken\VerificationToken;
 use OC\Session\CryptoWrapper;
 use OC\Share20\ProviderFactory;
 use OC\Share20\ShareHelper;
+use OC\SpeechToText\SpeechToTextManager;
 use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
 use OC\Tagging\TagMapper;
 use OC\Talk\Broker;
@@ -246,6 +247,7 @@ use OCP\Security\ISecureRandom;
 use OCP\Security\ITrustedDomainHelper;
 use OCP\Security\VerificationToken\IVerificationToken;
 use OCP\Share\IShareHelper;
+use OCP\SpeechToText\ISpeechToTextManager;
 use OCP\SystemTag\ISystemTagManager;
 use OCP\SystemTag\ISystemTagObjectMapper;
 use OCP\Talk\IBroker;
@@ -1457,6 +1459,8 @@ class Server extends ServerContainer implements IServerContainer {
 
 		$this->registerAlias(ITranslationManager::class, TranslationManager::class);
 
+		$this->registerAlias(ISpeechToTextManager::class, SpeechToTextManager::class);
+
 		$this->connectDispatcher();
 	}
 
diff --git a/lib/private/SpeechToText/SpeechToTextManager.php b/lib/private/SpeechToText/SpeechToTextManager.php
new file mode 100644
index 00000000000..757fc02485e
--- /dev/null
+++ b/lib/private/SpeechToText/SpeechToTextManager.php
@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Julius Härtl <jus@bitgrid.net>
+ * @copyright Copyright (c) 2023 Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ * @author Marcel Klehr <mklehr@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+namespace OC\SpeechToText;
+
+use InvalidArgumentException;
+use OC\AppFramework\Bootstrap\Coordinator;
+use OCP\BackgroundJob\IJobList;
+use OCP\Files\File;
+use OCP\Files\InvalidPathException;
+use OCP\Files\NotFoundException;
+use OCP\IServerContainer;
+use OCP\PreConditionNotMetException;
+use OCP\SpeechToText\ISpeechToTextManager;
+use OCP\SpeechToText\ISpeechToTextProvider;
+use Psr\Container\ContainerExceptionInterface;
+use Psr\Container\NotFoundExceptionInterface;
+use Psr\Log\LoggerInterface;
+use RuntimeException;
+use Throwable;
+
+class SpeechToTextManager implements ISpeechToTextManager {
+	/** @var ?ISpeechToTextProvider[] */
+	private ?array $providers = null;
+
+	public function __construct(
+		private IServerContainer $serverContainer,
+		private Coordinator $coordinator,
+		private LoggerInterface $logger,
+		private IJobList $jobList,
+	) {
+	}
+
+	public function getProviders(): array {
+		$context = $this->coordinator->getRegistrationContext();
+		if ($context === null) {
+			return [];
+		}
+
+		if ($this->providers !== null) {
+			return $this->providers;
+		}
+
+		$this->providers = [];
+
+		foreach ($context->getSpeechToTextProviders() as $providerServiceRegistration) {
+			$class = $providerServiceRegistration->getService();
+			try {
+				$this->providers[$class] = $this->serverContainer->get($class);
+			} catch (NotFoundExceptionInterface|ContainerExceptionInterface|Throwable $e) {
+				$this->logger->error('Failed to load SpeechToText provider ' . $class, [
+					'exception' => $e,
+				]);
+			}
+		}
+
+		return $this->providers;
+	}
+
+	public function hasProviders(): bool {
+		$context = $this->coordinator->getRegistrationContext();
+		if ($context === null) {
+			return false;
+		}
+		return !empty($context->getSpeechToTextProviders());
+	}
+
+	public function scheduleFileTranscription(File $file, ?string $userId, string $appId): void {
+		if (!$this->hasProviders()) {
+			throw new PreConditionNotMetException('No SpeechToText providers have been registered');
+		}
+		try {
+			$this->jobList->add(TranscriptionJob::class, [
+				'fileId' => $file->getId(),
+				'owner' => $file->getOwner()->getUID(),
+				'userId' => $userId,
+				'appId' => $appId,
+			]);
+		} catch (NotFoundException|InvalidPathException $e) {
+			throw new InvalidArgumentException('Invalid file provided for file transcription: ' . $e->getMessage());
+		}
+	}
+
+	public function transcribeFile(File $file): string {
+		if (!$this->hasProviders()) {
+			throw new PreConditionNotMetException('No SpeechToText providers have been registered');
+		}
+
+		foreach ($this->getProviders() as $provider) {
+			try {
+				return $provider->transcribeFile($file);
+			} catch (\Throwable $e) {
+				$this->logger->info('SpeechToText transcription using provider ' . $provider->getName() . ' failed', ['exception' => $e]);
+			}
+		}
+
+		throw new RuntimeException('Could not transcribe file');
+	}
+}
diff --git a/lib/private/SpeechToText/TranscriptionJob.php b/lib/private/SpeechToText/TranscriptionJob.php
new file mode 100644
index 00000000000..d5cc9ed7c38
--- /dev/null
+++ b/lib/private/SpeechToText/TranscriptionJob.php
@@ -0,0 +1,104 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Marcel Klehr <mklehr@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+namespace OC\SpeechToText;
+
+use OC\User\NoUserException;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\QueuedJob;
+use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\File;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\Files\NotPermittedException;
+use OCP\PreConditionNotMetException;
+use OCP\SpeechToText\Events\TranscriptionFailedEvent;
+use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
+use OCP\SpeechToText\ISpeechToTextManager;
+use Psr\Log\LoggerInterface;
+
+class TranscriptionJob extends QueuedJob {
+	public function __construct(
+		ITimeFactory $timeFactory,
+		private ISpeechToTextManager $speechToTextManager,
+		private IEventDispatcher $eventDispatcher,
+		private IRootFolder $rootFolder,
+		private LoggerInterface $logger,
+	) {
+		parent::__construct($timeFactory);
+	}
+
+
+	/**
+	 * @inheritDoc
+	 */
+	protected function run($argument) {
+		$fileId = $argument['fileId'];
+		$owner = $argument['owner'];
+		$userId = $argument['userId'];
+		$appId = $argument['appId'];
+		$file = null;
+		try {
+			\OC_Util::setupFS($owner);
+			$userFolder = $this->rootFolder->getUserFolder($owner);
+			$file = current($userFolder->getById($fileId));
+			if (!($file instanceof File)) {
+				$this->logger->warning('Transcription of file ' . $fileId . ' failed. The file could not be found');
+				$this->eventDispatcher->dispatchTyped(
+					new TranscriptionFailedEvent(
+						$fileId,
+						null,
+						'File not found',
+						$userId,
+						$appId,
+					)
+				);
+				return;
+			}
+			$result = $this->speechToTextManager->transcribeFile($file);
+			$this->eventDispatcher->dispatchTyped(
+				new TranscriptionSuccessfulEvent(
+					$fileId,
+					$file,
+					$result,
+					$userId,
+					$appId,
+				)
+			);
+		} catch (PreConditionNotMetException|\RuntimeException|\InvalidArgumentException|NotFoundException|NotPermittedException|NoUserException $e) {
+			$this->logger->warning('Transcription of file ' . $fileId . ' failed', ['exception' => $e]);
+			$this->eventDispatcher->dispatchTyped(
+				new TranscriptionFailedEvent(
+					$fileId,
+					$file,
+					$e->getMessage(),
+					$userId,
+					$appId,
+				)
+			);
+		}
+	}
+}