Add tasks::last_updated column and vacate tasks after a week

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
(cherry picked from commit cb0f918d2101bf7644d7d84f811b37e3e672f091)

diff --git a/core/Migrations/Version28000Date20230616104802.php b/core/Migrations/Version28000Date20230616104802.php
index 76d8173861f41a7624e60b0a82b26e9fc1889c61..ab7347608e2001874a49f45fbe286b5b37bd912f 100644 (file)
--- a/core/Migrations/Version28000Date20230616104802.php
+++ b/core/Migrations/Version28000Date20230616104802.php
@@ -78,9 +78,16 @@ class Version28000Date20230616104802 extends SimpleMigrationStep {
                                'length' => 32,
                                'default' => '',
                        ]);
+                       $table->addColumn('last_updated', 'integer', [
+                               'notnull' => false,
+                               'length' => 4,
+                               'default' => 0,
+                               'unsigned' => true,
+                       ]);
 
                        $table->setPrimaryKey(['id'], 'llm_tasks_id_index');
                        $table->addUniqueIndex(['status', 'type'], 'llm_tasks_status_type');
+                       $table->addIndex(['last_updated'], 'llm_tasks_updated');
                }
 
                return $schema;

diff --git a/lib/private/LanguageModel/Db/Task.php b/lib/private/LanguageModel/Db/Task.php
index fe77d5595307a32418400c533f85e073f0ea6da6..895969e08bb91bac6c4622437733248d0c6d0336 100644 (file)
--- a/lib/private/LanguageModel/Db/Task.php
+++ b/lib/private/LanguageModel/Db/Task.php
@@ -8,6 +8,8 @@ use OCP\LanguageModel\ILanguageModelTask;
 /**
  * @method setType(string $type)
  * @method string getType()
+ * @method setLastUpdated(int $lastUpdated)
+ * @method int getLastUpdated()
  * @method setInput(string $type)
  * @method string getInput()
  * @method setStatus(int $type)
@@ -18,6 +20,8 @@ use OCP\LanguageModel\ILanguageModelTask;
  * @method string getAppId()
  */
 class Task extends Entity {
+       protected $lastUpdated;
+
        protected $type;
        protected $input;
        protected $status;
@@ -27,17 +31,18 @@ class Task extends Entity {
        /**
         * @var string[]
         */
-       public static array $columns = ['id', 'type', 'input', 'output', 'status', 'user_id', 'app_id'];
+       public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id'];
 
        /**
         * @var string[]
         */
-       public static array $fields = ['id', 'type', 'input', 'output', 'status', 'userId', 'appId'];
+       public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId'];
 
 
        public function __construct() {
                // add types in constructor
                $this->addType('id', 'integer');
+               $this->addType('lastUpdated', 'integer');
                $this->addType('type', 'string');
                $this->addType('input', 'string');
                $this->addType('status', 'integer');
@@ -48,6 +53,7 @@ class Task extends Entity {
        public static function fromLanguageModelTask(ILanguageModelTask $task): Task {
                return Task::fromParams([
                        'type' => $task->getType(),
+                       'lastUpdated' => time(),
                        'status' => $task->getStatus(),
                        'input' => $task->getInput(),
                        'output' => $task->getOutput(),

diff --git a/lib/private/LanguageModel/Db/TaskMapper.php b/lib/private/LanguageModel/Db/TaskMapper.php
index d7122ea79416528b952e0291e6698d3eb3bb8b7d..e0b06a1b62b3f20693ffa438ce032fb06aac8adc 100644 (file)
--- a/lib/private/LanguageModel/Db/TaskMapper.php
+++ b/lib/private/LanguageModel/Db/TaskMapper.php
@@ -3,6 +3,7 @@
 namespace OC\LanguageModel\Db;
 
 use OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Db\Entity;
 use OCP\AppFramework\Db\MultipleObjectsReturnedException;
 use OCP\AppFramework\Db\QBMapper;
 use OCP\DB\Exception;
@@ -30,4 +31,21 @@ class TaskMapper extends QBMapper {
                        ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id)));
                return $this->findEntity($qb);
        }
+
+       /**
+        * @param int $timeout
+        * @return int the number of deleted tasks
+        * @throws Exception
+        */
+       public function deleteOlderThan(int $timeout): int {
+               $qb = $this->db->getQueryBuilder();
+               $qb->delete($this->tableName)
+                       ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter(time() - $timeout)));
+               return $qb->executeStatement();
+       }
+
+       public function update(Entity $entity): Entity {
+               $entity->setLastUpdated(time());
+               return parent::update($entity);
+       }
 }

diff --git a/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php b/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php
new file mode 100644 (file)
index 0000000..fa3a716
--- /dev/null
+++ b/lib/private/LanguageModel/RemoveOldTasksBackgroundJob.php
@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Marcel Klehr <mklehr@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+namespace OC\LanguageModel;
+
+use OC\LanguageModel\Db\TaskMapper;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCP\DB\Exception;
+use Psr\Log\LoggerInterface;
+
+class RemoveOldTasksBackgroundJob extends TimedJob {
+       public const MAX_TASK_AGE_SECONDS = 60 * 50 * 24 * 7; // 1 week
+
+       public function __construct(
+               ITimeFactory $timeFactory,
+               private TaskMapper $taskMapper,
+               private LoggerInterface $logger,
+
+       ) {
+               parent::__construct($timeFactory);
+               $this->setInterval(60 * 60 * 24);
+       }
+
+       /**
+        * @param mixed $argument
+        * @inheritDoc
+        */
+       protected function run($argument) {
+               try {
+                       $this->taskMapper->deleteOlderThan(self::MAX_TASK_AGE_SECONDS);
+               } catch (Exception $e) {
+                       $this->logger->warning('Failed to delete stale language model tasks', ['exception' => $e]);
+               }
+       }
+}

diff --git a/lib/private/Repair.php b/lib/private/Repair.php
index 330fa241b1e090e5add9266f7ffedc9d4f6d91f1..00cc40a00b355159ab6c73c198a95a91cd73526c 100644 (file)
--- a/lib/private/Repair.php
+++ b/lib/private/Repair.php
@@ -34,6 +34,7 @@
  */
 namespace OC;
 
+use OC\Repair\AddRemoveOldTasksBackgroundJob;
 use OC\Repair\CleanUpAbandonedApps;
 use OCP\AppFramework\QueryException;
 use OCP\AppFramework\Utility\ITimeFactory;
@@ -210,6 +211,7 @@ class Repair implements IOutput {
                        \OCP\Server::get(AddTokenCleanupJob::class),
                        \OCP\Server::get(CleanUpAbandonedApps::class),
                        \OCP\Server::get(AddMissingSecretJob::class),
+                       \OCP\Server::get(AddRemoveOldTasksBackgroundJob::class),
                ];
        }
 

diff --git a/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php
new file mode 100644 (file)
index 0000000..713192b
--- /dev/null
+++ b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php
@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright 2023 Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Marcel Klehr <mklehr@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OC\Repair;
+
+use OC\LanguageModel\RemoveOldTasksBackgroundJob;
+use OCP\BackgroundJob\IJobList;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class AddRemoveOldTasksBackgroundJob implements IRepairStep {
+       private IJobList $jobList;
+
+       public function __construct(IJobList $jobList) {
+               $this->jobList = $jobList;
+       }
+
+       public function getName(): string {
+               return 'Add language model tasks cleanup job';
+       }
+
+       public function run(IOutput $output) {
+               $this->jobList->add(RemoveOldTasksBackgroundJob::class);
+       }
+}

diff --git a/lib/private/Setup.php b/lib/private/Setup.php
index 3a2e7614dd92e55d9dcc61bedfadd4b9538c571f..a80afee6d86c22daf6845d6d27ceb9db7f9b1498 100644 (file)
--- a/lib/private/Setup.php
+++ b/lib/private/Setup.php
@@ -53,6 +53,7 @@ use Exception;
 use InvalidArgumentException;
 use OC\Authentication\Token\PublicKeyTokenProvider;
 use OC\Authentication\Token\TokenCleanupJob;
+use OC\LanguageModel\RemoveOldTasksBackgroundJob;
 use OC\Log\Rotate;
 use OC\Preview\BackgroundCleanupJob;
 use OCP\AppFramework\Utility\ITimeFactory;
@@ -453,6 +454,7 @@ class Setup {
                $jobList->add(TokenCleanupJob::class);
                $jobList->add(Rotate::class);
                $jobList->add(BackgroundCleanupJob::class);
+               $jobList->add(RemoveOldTasksBackgroundJob::class);
        }
 
        /**