Catch duplicate insertion errors while scanning files

When two scanning processed run at the same time, for example when
scan.php and list.php were called at the same time on an external
storage mountpoint, duplicate insertion errors can occurs.

These errors are now logged and ignored.

Since both scans are running in parallel transactions, they don't see
each other's changes directly in the DB which can cause duplicate
insertion errors for mime types as well, but those mime types can't be
selected yet. The solution to this is to force-reload the mimetypes list
after the transaction is finished.

Fixes #5199

2 files changed, 38 insertions, 14 deletions

diff --git a/lib/private/files/cache/cache.php b/lib/private/files/cache/cache.php
index 364a50d377c..fc2d965d7f9 100644
--- a/lib/private/files/cache/cache.php
+++ b/lib/private/files/cache/cache.php
@@ -69,9 +69,15 @@ class Cache {
 		}
 		
 		if (!isset(self::$mimetypeIds[$mime])) {
-			$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
-			self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
-			self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			try{
+				$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
+				self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
+				self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			}
+			catch (\Doctrine\DBAL\DBALException $e){
+				\OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
+				return -1;
+			}
 		} 
 				
 		return self::$mimetypeIds[$mime];
@@ -84,8 +90,8 @@ class Cache {
 
 		return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
 	}
-	
-	protected function loadMimetypes(){
+
+	public function loadMimetypes(){
 			$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
 			if ($result) {
 				while ($row = $result->fetchRow()) {
diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php
index 96f84609cf2..f63abf2d4fc 100644
--- a/lib/private/files/cache/scanner.php
+++ b/lib/private/files/cache/scanner.php
@@ -190,24 +190,34 @@ class Scanner extends BasicEmitter {
 		}
 		$newChildren = array();
 		if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
+			$exceptionOccurred = false;
 			\OC_DB::beginTransaction();
 			if (is_resource($dh)) {
 				while (($file = readdir($dh)) !== false) {
 					$child = ($path) ? $path . '/' . $file : $file;
 					if (!Filesystem::isIgnoredDir($file)) {
 						$newChildren[] = $file;
-						$data = $this->scanFile($child, $reuse, true);
-						if ($data) {
-							if ($data['size'] === -1) {
-								if ($recursive === self::SCAN_RECURSIVE) {
-									$childQueue[] = $child;
-								} else {
-									$size = -1;
+						try {
+							$data = $this->scanFile($child, $reuse, true);
+							if ($data) {
+								if ($data['size'] === -1) {
+									if ($recursive === self::SCAN_RECURSIVE) {
+										$childQueue[] = $child;
+									} else {
+										$size = -1;
+									}
+								} else if ($size !== -1) {
+									$size += $data['size'];
 								}
-							} else if ($size !== -1) {
-								$size += $data['size'];
 							}
 						}
+						catch (\Doctrine\DBAL\DBALException $ex){
+							// might happen if inserting duplicate while a scanning
+							// process is running in parallel
+							// log and ignore
+							\OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
+							$exceptionOccurred = true;
+						}
 					}
 				}
 			}
@@ -217,6 +227,14 @@ class Scanner extends BasicEmitter {
 				$this->cache->remove($child);
 			}
 			\OC_DB::commit();
+			if ($exceptionOccurred){
+				// It might happen that the parallel scan process has already
+				// inserted mimetypes but those weren't available yet inside the transaction
+				// To make sure to have the updated mime types in such cases,
+				// we reload them here
+				$this->cache->loadMimetypes();
+			}
+
 			foreach ($childQueue as $child) {
 				$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
 				if ($childSize === -1) {