diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/private/Files/Cache/Propagator.php | 88 | ||||
| -rw-r--r-- | lib/private/Files/Utils/Scanner.php | 4 | ||||
| -rw-r--r-- | lib/public/Files/Cache/IPropagator.php | 19 |
3 files changed, 110 insertions, 1 deletions
diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index b998c6bcfae..52bb4dfdfca 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -29,6 +29,10 @@ use OCP\IDBConnection; * Propagate etags and mtimes within the storage */ class Propagator implements IPropagator { + private $inBatch = false; + + private $batch = []; + /** * @var \OC\Files\Storage\Storage */ @@ -59,6 +63,13 @@ class Propagator implements IPropagator { $parents = $this->getParents($internalPath); + if ($this->inBatch) { + foreach ($parents as $parent) { + $this->addToBatch($parent, $time, $sizeDifference); + } + return; + } + $parentHashes = array_map('md5', $parents); $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag @@ -68,7 +79,7 @@ class Propagator implements IPropagator { }, $parentHashes); $builder->update('filecache') - ->set('mtime', $builder->createFunction('GREATEST(`mtime`, ' . $builder->createNamedParameter($time) . ')')) + ->set('mtime', $builder->createFunction('GREATEST(`mtime`, ' . $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT) . ')')) ->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)) ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) ->andWhere($builder->expr()->in('path_hash', $hashParams)); @@ -98,4 +109,79 @@ class Propagator implements IPropagator { } return $parents; } + + /** + * Mark the beginning of a propagation batch + * + * Note that not all cache setups support propagation in which case this will be a noop + * + * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent + * before the batch is committed. + */ + public function beginBatch() { + $this->inBatch = true; + } + + private function addToBatch($internalPath, $time, $sizeDifference) { + if (!isset($this->batch[$internalPath])) { + $this->batch[$internalPath] = [ + 'hash' => md5($internalPath), + 'time' => $time, + 'size' => $sizeDifference + ]; + } else { + $this->batch[$internalPath]['size'] += $sizeDifference; + if ($time > $this->batch[$internalPath]['time']) { + $this->batch[$internalPath]['time'] = $time; + } + } + } + + /** + * Commit the active propagation batch + */ + public function commitBatch() { + if (!$this->inBatch) { + throw new \BadMethodCallException('Not in batch'); + } + $this->inBatch = false; + + $this->connection->beginTransaction(); + + $query = $this->connection->getQueryBuilder(); + $storageId = (int)$this->storage->getStorageCache()->getNumericId(); + + $query->update('filecache') + ->set('mtime', $query->createFunction('GREATEST(`mtime`, ' . $query->createParameter('time') . ')')) + ->set('etag', $query->expr()->literal(uniqid())) + ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); + + $sizeQuery = $this->connection->getQueryBuilder(); + $sizeQuery->update('filecache') + ->set('size', $sizeQuery->createFunction('`size` + ' . $sizeQuery->createParameter('size'))) + ->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))) + ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + + foreach ($this->batch as $item) { + $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT); + $query->setParameter('hash', $item['hash']); + + $query->execute(); + + if ($item['size']) { + $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT); + $sizeQuery->setParameter('hash', $item['hash']); + + $sizeQuery->execute(); + } + } + + $this->batch = []; + + $this->connection->commit(); + } + + } diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 8beba116fe1..e4e5e353f9f 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -138,7 +138,9 @@ class Scanner extends PublicEmitter { $this->triggerPropagator($storage, $path); }); + $storage->getPropagator()->beginBatch(); $scanner->backgroundScan(); + $storage->getPropagator()->commitBatch(); } } @@ -187,12 +189,14 @@ class Scanner extends PublicEmitter { $this->db->beginTransaction(); } try { + $storage->getPropagator()->beginBatch(); $scanner->scan($relativePath, \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); $cache = $storage->getCache(); if ($cache instanceof Cache) { // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner $cache->correctFolderSize($relativePath); } + $storage->getPropagator()->commitBatch(); } catch (StorageNotAvailableException $e) { $this->logger->error('Storage ' . $storage->getId() . ' not available'); $this->logger->logException($e); diff --git a/lib/public/Files/Cache/IPropagator.php b/lib/public/Files/Cache/IPropagator.php index 5494ec9a54e..541135b9e60 100644 --- a/lib/public/Files/Cache/IPropagator.php +++ b/lib/public/Files/Cache/IPropagator.php @@ -28,6 +28,25 @@ namespace OCP\Files\Cache; */ interface IPropagator { /** + * Mark the beginning of a propagation batch + * + * Note that not all cache setups support propagation in which case this will be a noop + * + * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent + * before the batch is committed. + * + * @since 9.1.0 + */ + public function beginBatch(); + + /** + * Commit the active propagation batch + * + * @since 9.1.0 + */ + public function commitBatch(); + + /** * @param string $internalPath * @param int $time * @since 9.0.0 |