Catch duplicate insertion errors while scanning files

When two scanning processed run at the same time, for example when
scan.php and list.php were called at the same time on an external
storage mountpoint, duplicate insertion errors can occurs.

These errors are now logged and ignored.

Since both scans are running in parallel transactions, they don't see
each other's changes directly in the DB which can cause duplicate
insertion errors for mime types as well, but those mime types can't be
selected yet. The solution to this is to force-reload the mimetypes list
after the transaction is finished.

Fixes #5199

diff --git a/lib/private/files/cache/cache.php b/lib/private/files/cache/cache.php
index 364a50d377c28de1b90ee12cb7713d59a627f249..fc2d965d7f97fff2d273ae2de1f90dcbb701b7f5 100644 (file)
--- a/lib/private/files/cache/cache.php
+++ b/lib/private/files/cache/cache.php
@@ -69,9 +69,15 @@ class Cache {
                }
                
                if (!isset(self::$mimetypeIds[$mime])) {
-                       $result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
-                       self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
-                       self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+                       try{
+                               $result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
+                               self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
+                               self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+                       }
+                       catch (\Doctrine\DBAL\DBALException $e){
+                               \OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
+                               return -1;
+                       }
                } 
                                
                return self::$mimetypeIds[$mime];
@@ -84,8 +90,8 @@ class Cache {
 
                return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
        }
-       
-       protected function loadMimetypes(){
+
+       public function loadMimetypes(){
                        $result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
                        if ($result) {
                                while ($row = $result->fetchRow()) {

diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php
index 96f84609cf231fb45374c24b0f70a87459eef638..f63abf2d4fc6a48878833103e911317fe3bfa85b 100644 (file)
--- a/lib/private/files/cache/scanner.php
+++ b/lib/private/files/cache/scanner.php
@@ -190,24 +190,34 @@ class Scanner extends BasicEmitter {
                }
                $newChildren = array();
                if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
+                       $exceptionOccurred = false;
                        \OC_DB::beginTransaction();
                        if (is_resource($dh)) {
                                while (($file = readdir($dh)) !== false) {
                                        $child = ($path) ? $path . '/' . $file : $file;
                                        if (!Filesystem::isIgnoredDir($file)) {
                                                $newChildren[] = $file;
-                                               $data = $this->scanFile($child, $reuse, true);
-                                               if ($data) {
-                                                       if ($data['size'] === -1) {
-                                                               if ($recursive === self::SCAN_RECURSIVE) {
-                                                                       $childQueue[] = $child;
-                                                               } else {
-                                                                       $size = -1;
+                                               try {
+                                                       $data = $this->scanFile($child, $reuse, true);
+                                                       if ($data) {
+                                                               if ($data['size'] === -1) {
+                                                                       if ($recursive === self::SCAN_RECURSIVE) {
+                                                                               $childQueue[] = $child;
+                                                                       } else {
+                                                                               $size = -1;
+                                                                       }
+                                                               } else if ($size !== -1) {
+                                                                       $size += $data['size'];
                                                                }
-                                                       } else if ($size !== -1) {
-                                                               $size += $data['size'];
                                                        }
                                                }
+                                               catch (\Doctrine\DBAL\DBALException $ex){
+                                                       // might happen if inserting duplicate while a scanning
+                                                       // process is running in parallel
+                                                       // log and ignore
+                                                       \OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
+                                                       $exceptionOccurred = true;
+                                               }
                                        }
                                }
                        }
@@ -217,6 +227,14 @@ class Scanner extends BasicEmitter {
                                $this->cache->remove($child);
                        }
                        \OC_DB::commit();
+                       if ($exceptionOccurred){
+                               // It might happen that the parallel scan process has already
+                               // inserted mimetypes but those weren't available yet inside the transaction
+                               // To make sure to have the updated mime types in such cases,
+                               // we reload them here
+                               $this->cache->loadMimetypes();
+                       }
+
                        foreach ($childQueue as $child) {
                                $childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
                                if ($childSize === -1) {