From 0ebb2050102190b1186c7338a84f86bd6f3f9d43 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 10 Feb 2016 12:01:55 +0100 Subject: [PATCH] Chunk the queries to make sure they don't time out --- apps/files/command/deleteorphanedfiles.php | 30 +++++++++++--- .../lib/backgroundjob/deleteorphaneditems.php | 41 ++++++++++++++----- lib/private/repair/repairinvalidshares.php | 22 ++++++---- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/apps/files/command/deleteorphanedfiles.php b/apps/files/command/deleteorphanedfiles.php index d276e9a0993..f897c68fd8d 100644 --- a/apps/files/command/deleteorphanedfiles.php +++ b/apps/files/command/deleteorphanedfiles.php @@ -33,6 +33,8 @@ use Symfony\Component\Console\Output\OutputInterface; */ class DeleteOrphanedFiles extends Command { + const CHUNK_SIZE = 200; + /** * @var IDBConnection */ @@ -50,13 +52,31 @@ class DeleteOrphanedFiles extends Command { } public function execute(InputInterface $input, OutputInterface $output) { + $deletedEntries = 0; - $sql = - 'DELETE FROM `*PREFIX*filecache` ' . - 'WHERE NOT EXISTS ' . - '(SELECT 1 FROM `*PREFIX*storages` WHERE `storage` = `numeric_id`)'; + $query = $this->connection->getQueryBuilder(); + $query->select('fc.fileid') + ->from('filecache', 'fc') + ->where($query->expr()->isNull('s.numeric_id')) + ->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id')) + ->setMaxResults(self::CHUNK_SIZE); + + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete('filecache') + ->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid'))); + + $deletedInLastChunk = self::CHUNK_SIZE; + while ($deletedInLastChunk === self::CHUNK_SIZE) { + $deletedInLastChunk = 0; + $result = $query->execute(); + while ($row = $result->fetch()) { + $deletedInLastChunk++; + $deletedEntries += $deleteQuery->setParameter('objectid', (int) $row['fileid']) + ->execute(); + } + $result->closeCursor(); + } - $deletedEntries = $this->connection->executeUpdate($sql); $output->writeln("$deletedEntries orphaned file cache entries deleted"); } diff --git a/apps/files/lib/backgroundjob/deleteorphaneditems.php b/apps/files/lib/backgroundjob/deleteorphaneditems.php index cefa1d655de..773bb997e56 100644 --- a/apps/files/lib/backgroundjob/deleteorphaneditems.php +++ b/apps/files/lib/backgroundjob/deleteorphaneditems.php @@ -29,6 +29,8 @@ use OCP\DB\QueryBuilder\IQueryBuilder; */ class DeleteOrphanedItems extends TimedJob { + const CHUNK_SIZE = 200; + /** @var \OCP\IDBConnection */ protected $connection; @@ -66,19 +68,38 @@ class DeleteOrphanedItems extends TimedJob { /** * Deleting orphaned system tag mappings * + * @param string $table + * @param string $idCol + * @param string $typeCol * @return int Number of deleted entries */ protected function cleanUp($table, $idCol, $typeCol) { - $subQuery = $this->connection->getQueryBuilder(); - $subQuery->select($subQuery->expr()->literal('1')) - ->from('filecache', 'f') - ->where($subQuery->expr()->eq($idCol, 'f.fileid')); + $deletedEntries = 0; $query = $this->connection->getQueryBuilder(); - $deletedEntries = $query->delete($table) + $query->select('t1.' . $idCol) + ->from($table, 't1') ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) - ->andWhere($query->expr()->isNull($query->createFunction('(' . $subQuery->getSql() . ')'))) - ->execute(); + ->andWhere($query->expr()->isNull('t2.fileid')) + ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid')) + ->groupBy('t1.' . $idCol) + ->setMaxResults(self::CHUNK_SIZE); + + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete($table) + ->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid'))); + + $deletedInLastChunk = self::CHUNK_SIZE; + while ($deletedInLastChunk === self::CHUNK_SIZE) { + $result = $query->execute(); + $deletedInLastChunk = 0; + while ($row = $result->fetch()) { + $deletedInLastChunk++; + $deletedEntries += $deleteQuery->setParameter('objectid', (int) $row[$idCol]) + ->execute(); + } + $result->closeCursor(); + } return $deletedEntries; } @@ -111,8 +132,7 @@ class DeleteOrphanedItems extends TimedJob { * @return int Number of deleted entries */ protected function cleanComments() { - $qb = $this->connection->getQueryBuilder(); - $deletedEntries = $this->cleanUp('comments', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type'); + $deletedEntries = $this->cleanUp('comments', 'object_id', 'object_type'); $this->logger->debug("$deletedEntries orphaned comments deleted", ['app' => 'DeleteOrphanedItems']); return $deletedEntries; } @@ -123,8 +143,7 @@ class DeleteOrphanedItems extends TimedJob { * @return int Number of deleted entries */ protected function cleanCommentMarkers() { - $qb = $this->connection->getQueryBuilder(); - $deletedEntries = $this->cleanUp('comments_read_markers', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type'); + $deletedEntries = $this->cleanUp('comments_read_markers', 'object_id', 'object_type'); $this->logger->debug("$deletedEntries orphaned comment read marks deleted", ['app' => 'DeleteOrphanedItems']); return $deletedEntries; } diff --git a/lib/private/repair/repairinvalidshares.php b/lib/private/repair/repairinvalidshares.php index ee8b23906e5..beef5e37798 100644 --- a/lib/private/repair/repairinvalidshares.php +++ b/lib/private/repair/repairinvalidshares.php @@ -30,6 +30,8 @@ use OC\Hooks\BasicEmitter; */ class RepairInvalidShares extends BasicEmitter implements \OC\RepairStep { + const CHUNK_SIZE = 200; + /** * @var \OCP\IConfig */ @@ -83,18 +85,24 @@ class RepairInvalidShares extends BasicEmitter implements \OC\RepairStep { ->where($query->expr()->isNotNull('s1.parent')) ->andWhere($query->expr()->isNull('s2.id')) ->leftJoin('s1', 'share', 's2', $query->expr()->eq('s1.parent', 's2.id')) - ->groupBy('s1.parent'); + ->groupBy('s1.parent') + ->setMaxResults(self::CHUNK_SIZE); $deleteQuery = $this->connection->getQueryBuilder(); $deleteQuery->delete('share') - ->where($query->expr()->eq('parent', $deleteQuery->createParameter('parent'))); + ->where($deleteQuery->expr()->eq('parent', $deleteQuery->createParameter('parent'))); - $result = $query->execute(); - while ($row = $result->fetch()) { - $deletedEntries += $deleteQuery->setParameter('parent', (int) $row['parent']) - ->execute(); + $deletedInLastChunk = self::CHUNK_SIZE; + while ($deletedInLastChunk === self::CHUNK_SIZE) { + $deletedInLastChunk = 0; + $result = $query->execute(); + while ($row = $result->fetch()) { + $deletedInLastChunk++; + $deletedEntries += $deleteQuery->setParameter('parent', (int) $row['parent']) + ->execute(); + } + $result->closeCursor(); } - $result->closeCursor(); if ($deletedEntries) { $this->emit('\OC\Repair', 'info', array('Removed ' . $deletedEntries . ' shares where the parent did not exist'));