Merge pull request #9900 from nextcloud/feature/noid/wait-for-cron-to-finish

Wait for cron to finish before running upgrade command

2 files changed, 55 insertions, 9 deletions

diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php
index 1399683dc9b..a926194e9df 100644
--- a/lib/private/BackgroundJob/JobList.php
+++ b/lib/private/BackgroundJob/JobList.php
@@ -48,6 +48,9 @@ class JobList implements IJobList {
 	/**@var ITimeFactory */
 	protected $timeFactory;
 
+	/** @var int - 12 hours * 3600 seconds*/
+	private $jobTimeOut = 43200;
+
 	/**
 	 * @param IDBConnection $connection
 	 * @param IConfig $config
@@ -182,7 +185,7 @@ class JobList implements IJobList {
 		$query = $this->connection->getQueryBuilder();
 		$query->select('*')
 			->from('jobs')
-			->where($query->expr()->lte('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 12 * 3600, IQueryBuilder::PARAM_INT)))
+			->where($query->expr()->lte('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - $this->jobTimeOut, IQueryBuilder::PARAM_INT)))
 			->orderBy('last_checked', 'ASC')
 			->setMaxResults(1);
 
@@ -333,4 +336,39 @@ class JobList implements IJobList {
 			->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
 		$query->execute();
 	}
+
+	/**
+	 * checks if a job is still running (reserved_at time is smaller than 12 hours ago)
+	 *
+	 * Background information:
+	 *
+	 * The 12 hours is the same timeout that is also used to re-schedule an non-terminated
+	 * job (see getNext()). The idea here is to give a job enough time to run very
+	 * long but still be able to recognize that it maybe crashed and re-schedule it
+	 * after the timeout. It's more likely to be crashed at that time than it ran
+	 * that long.
+	 *
+	 * In theory it could lead to an nearly endless loop (as in - at most 12 hours).
+	 * The cron command will not start new jobs when maintenance mode is active and
+	 * this method is only executed in maintenance mode (see where it is called in
+	 * the upgrader class. So this means in the worst case we wait 12 hours when a
+	 * job has crashed. On the other hand: then the instance should be fixed anyways.
+	 *
+	 * @return bool
+	 */
+	public function isAnyJobRunning(): bool {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('*')
+			->from('jobs')
+			->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - $this->jobTimeOut, IQueryBuilder::PARAM_INT)))
+			->setMaxResults(1);
+		$result = $query->execute();
+		$row = $result->fetch();
+		$result->closeCursor();
+
+		if ($row) {
+			return true;
+		}
+		return false;
+	}
 }
diff --git a/lib/private/Updater.php b/lib/private/Updater.php
index e6e38798142..02b3138f30f 100644
--- a/lib/private/Updater.php
+++ b/lib/private/Updater.php
@@ -38,6 +38,7 @@ use OC\DB\MigrationService;
 use OC\Hooks\BasicEmitter;
 use OC\IntegrityCheck\Checker;
 use OC_App;
+use OCP\BackgroundJob\IJobList;
 use OCP\IConfig;
 use OCP\ILogger;
 use OCP\Util;
@@ -66,6 +67,9 @@ class Updater extends BasicEmitter {
 	/** @var Installer */
 	private $installer;
 
+	/** @var IJobList */
+	private $jobList;
+
 	private $logLevelNames = [
 		0 => 'Debug',
 		1 => 'Info',
@@ -74,20 +78,16 @@ class Updater extends BasicEmitter {
 		4 => 'Fatal',
 	];
 
-	/**
-	 * @param IConfig $config
-	 * @param Checker $checker
-	 * @param ILogger $log
-	 * @param Installer $installer
-	 */
 	public function __construct(IConfig $config,
 								Checker $checker,
-								ILogger $log = null,
-								Installer $installer) {
+								ILogger $log,
+								Installer $installer,
+								IJobList $jobList) {
 		$this->log = $log;
 		$this->config = $config;
 		$this->checker = $checker;
 		$this->installer = $installer;
+		$this->jobList = $jobList;
 	}
 
 	/**
@@ -111,6 +111,8 @@ class Updater extends BasicEmitter {
 			$this->emit('\OC\Updater', 'maintenanceEnabled');
 		}
 
+		$this->waitForCronToFinish();
+
 		$installedVersion = $this->config->getSystemValue('version', '0.0.0');
 		$currentVersion = implode('.', \OCP\Util::getVersion());
 		$this->log->debug('starting upgrade from ' . $installedVersion . ' to ' . $currentVersion, array('app' => 'core'));
@@ -604,6 +606,12 @@ class Updater extends BasicEmitter {
 		});
 
 	}
+	private function waitForCronToFinish() {
+		while ($this->jobList->isAnyJobRunning()) {
+			$this->emit('\OC\Updater', 'waitForCronToFinish');
+			sleep(5);
+		}
+	}
 
 }