From e3218ddffda93e1eecd4a307c70f1a8929823f9a Mon Sep 17 00:00:00 2001
From: Vincent Petry <pvince81@owncloud.com>
Date: Tue, 28 Mar 2017 19:18:58 +0200
Subject: OCC scanner commit in batches

---
 lib/private/Files/Utils/Scanner.php | 44 ++++++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 6 deletions(-)

diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php
index 02f355fd4d9..a0981beeaba 100644
--- a/lib/private/Files/Utils/Scanner.php
+++ b/lib/private/Files/Utils/Scanner.php
@@ -47,6 +47,8 @@ use OCP\ILogger;
  * @package OC\Files\Utils
  */
 class Scanner extends PublicEmitter {
+	const MAX_ENTRIES_TO_COMMIT = 10000;
+
 	/**
 	 * @var string $user
 	 */
@@ -62,6 +64,20 @@ class Scanner extends PublicEmitter {
 	 */
 	protected $logger;
 
+	/**
+	 * Whether to use a DB transaction
+	 *
+	 * @var bool
+	 */
+	protected $useTransaction;
+
+	/**
+	 * Number of entries scanned to commit
+	 *
+	 * @var int
+	 */
+	protected $entriesToCommit;
+
 	/**
 	 * @param string $user
 	 * @param \OCP\IDBConnection $db
@@ -71,6 +87,7 @@ class Scanner extends PublicEmitter {
 		$this->logger = $logger;
 		$this->user = $user;
 		$this->db = $db;
+		$this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider);
 	}
 
 	/**
@@ -200,22 +217,22 @@ class Scanner extends PublicEmitter {
 			$scanner = $storage->getScanner();
 			$scanner->setUseTransactions(false);
 			$this->attachListener($mount);
-			$isDbLocking = \OC::$server->getLockingProvider() instanceof DBLockingProvider;
 
 			$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 			$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 			$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 
 			if (!$storage->file_exists($relativePath)) {
 				throw new NotFoundException($dir);
 			}
-			if (!$isDbLocking) {
+
+			if ($this->useTransaction) {
 				$this->db->beginTransaction();
 			}
 			try {
@@ -233,7 +250,7 @@ class Scanner extends PublicEmitter {
 				$this->logger->logException($e);
 				$this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]);
 			}
-			if (!$isDbLocking) {
+			if ($this->useTransaction) {
 				$this->db->commit();
 			}
 		}
@@ -242,5 +259,20 @@ class Scanner extends PublicEmitter {
 	private function triggerPropagator(IStorage $storage, $internalPath) {
 		$storage->getPropagator()->propagateChange($internalPath, time());
 	}
+
+	private function postProcessEntry(IStorage $storage, $internalPath) {
+		$this->triggerPropagator($storage, $internalPath);
+		$this->entriesToCommit++;
+		if ($this->useTransaction) {
+			$propagator = $storage->getPropagator();
+			if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) {
+				$this->entriesToCommit = 0;
+				$this->db->commit();
+				$propagator->commitBatch();
+				$this->db->beginTransaction();
+				$propagator->beginBatch();
+			}
+		}
+	}
 }
 
-- 
cgit v1.2.3


From 0721a7d5cc68689a04ff0d60ed6e6e33595e267a Mon Sep 17 00:00:00 2001
From: Vincent Petry <pvince81@owncloud.com>
Date: Mon, 3 Apr 2017 11:49:30 +0200
Subject: Fix scanner after review

---
 lib/private/Files/Utils/Scanner.php | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php
index a0981beeaba..fac95462ce5 100644
--- a/lib/private/Files/Utils/Scanner.php
+++ b/lib/private/Files/Utils/Scanner.php
@@ -87,6 +87,7 @@ class Scanner extends PublicEmitter {
 		$this->logger = $logger;
 		$this->user = $user;
 		$this->db = $db;
+		// when DB locking is used, no DB transactions will be used
 		$this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider);
 	}
 
@@ -262,10 +263,10 @@ class Scanner extends PublicEmitter {
 
 	private function postProcessEntry(IStorage $storage, $internalPath) {
 		$this->triggerPropagator($storage, $internalPath);
-		$this->entriesToCommit++;
 		if ($this->useTransaction) {
-			$propagator = $storage->getPropagator();
+			$this->entriesToCommit++;
 			if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) {
+				$propagator = $storage->getPropagator();
 				$this->entriesToCommit = 0;
 				$this->db->commit();
 				$propagator->commitBatch();
-- 
cgit v1.2.3