]> source.dussan.org Git - nextcloud-server.git/commitdiff
Add a metadata service to store file metadata
authorCarl Schwan <carl@carlschwan.eu>
Mon, 4 Apr 2022 21:15:00 +0000 (23:15 +0200)
committerCarl Schwan <carl@carlschwan.eu>
Wed, 13 Apr 2022 12:06:29 +0000 (14:06 +0200)
Signed-off-by: Carl Schwan <carl@carlschwan.eu>
39 files changed:
3rdparty
apps/dav/composer/autoload.php
apps/dav/composer/composer/autoload_real.php
apps/dav/composer/composer/installed.php
apps/dav/lib/Connector/Sabre/Directory.php
apps/dav/lib/Connector/Sabre/File.php
apps/dav/lib/Connector/Sabre/FilesPlugin.php
apps/dav/lib/Files/FileSearchBackend.php
apps/dav/lib/Files/LazySearchBackend.php
build/psalm-baseline.xml
config/config.sample.php
core/Application.php
core/Migrations/Version240000Date20220404230027.php [new file with mode: 0644]
lib/composer/composer/InstalledVersions.php
lib/composer/composer/LICENSE
lib/composer/composer/autoload_classmap.php
lib/composer/composer/autoload_namespaces.php
lib/composer/composer/autoload_psr4.php
lib/composer/composer/autoload_real.php
lib/composer/composer/autoload_static.php
lib/composer/composer/installed.php
lib/private/Files/ObjectStore/NoopScanner.php
lib/private/Metadata/Capabilities.php [new file with mode: 0644]
lib/private/Metadata/FileEventListener.php [new file with mode: 0644]
lib/private/Metadata/FileMetadata.php [new file with mode: 0644]
lib/private/Metadata/FileMetadataMapper.php [new file with mode: 0644]
lib/private/Metadata/IMetadataManager.php [new file with mode: 0644]
lib/private/Metadata/IMetadataProvider.php [new file with mode: 0644]
lib/private/Metadata/MetadataManager.php [new file with mode: 0644]
lib/private/Metadata/Provider/ExifProvider.php [new file with mode: 0644]
lib/private/Server.php
lib/public/AppFramework/Db/Entity.php
lib/public/AppFramework/Db/QBMapper.php
lib/public/DB/QueryBuilder/IQueryBuilder.php
lib/public/DB/Types.php
tests/lib/AppFramework/Db/QBMapperTest.php
tests/lib/DB/MigratorTest.php
tests/lib/Metadata/FileMetadataMapperTest.php [new file with mode: 0644]
version.php

index d80ec1fa2dad1c3ede272583e3c4f1f77f40141b..6176112be9428026897d958dc2b558d1bde4fec2 160000 (submodule)
--- a/3rdparty
+++ b/3rdparty
@@ -1 +1 @@
-Subproject commit d80ec1fa2dad1c3ede272583e3c4f1f77f40141b
+Subproject commit 6176112be9428026897d958dc2b558d1bde4fec2
index 06b2e993e943803917b84dfbe75a1c62f2ff310d..a3040af8caa0eee1a40d120fa06f6ec6628aa5a7 100644 (file)
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitDAV::getLoader();
index 8416efa9d7e0bc21cf21ff08b24b7b011d7f72db..4b2290344f5f1efbb9841bef83c46b14fd1e35cc 100644 (file)
@@ -27,7 +27,7 @@ class ComposerAutoloaderInitDAV
         spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);
index baf72c4fb3446da9da29e98a86e094e982272d8e..628db5d793b90ac3d21aa6bf2f18cc05df95bb1e 100644 (file)
@@ -5,7 +5,7 @@
         'type' => 'library',
         'install_path' => __DIR__ . '/../',
         'aliases' => array(),
-        'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+        'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
         'name' => '__root__',
         'dev' => false,
     ),
@@ -16,7 +16,7 @@
             'type' => 'library',
             'install_path' => __DIR__ . '/../',
             'aliases' => array(),
-            'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+            'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
             'dev_requirement' => false,
         ),
     ),
index 8b616b0cb8a1ba2df0c6560f4a892370dbcea4e4..9e0b89596cdeeade50454e5069b04bd81c25e2c2 100644 (file)
@@ -34,6 +34,8 @@ namespace OCA\DAV\Connector\Sabre;
 
 use OC\Files\Mount\MoveableMount;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
+use OC\Metadata\MetadataGroup;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
@@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol
         */
        private $tree;
 
+       /** @var array<string, array<int, FileMetadata>> */
+       private array $metadata = [];
+
        /**
         * Sets up the node, expects a full path name
         *
index a46ca372be772ee9b1c010c2373591427622b23a..6c37998499524f96e2f367e7f1d60c5db139adc1 100644 (file)
@@ -43,6 +43,7 @@ use OC\AppFramework\Http\Request;
 use OC\Files\Filesystem;
 use OC\Files\Stream\HashWrapper;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
 use OCA\DAV\AppInfo\Application;
 use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
@@ -80,6 +81,9 @@ class File extends Node implements IFile {
 
        protected IL10N $l10n;
 
+       /** @var array<string, FileMetadata> */
+       private array $metadata = [];
+
        /**
         * Sets up the node, expects a full path name
         *
@@ -757,4 +761,16 @@ class File extends Node implements IFile {
        public function getNode(): \OCP\Files\File {
                return $this->node;
        }
+
+       public function getMetadata(string $group): FileMetadata {
+               return $this->metadata[$group];
+       }
+
+       public function setMetadata(string $group, FileMetadata $metadata): void {
+               $this->metadata[$group] = $metadata;
+       }
+
+       public function hasMetadata(string $group) {
+               return array_key_exists($group, $this->metadata);
+       }
 }
index 180f05c0e7eaf092fc0ac1935aa237d57d4c1a9f..5cc562e0ff82bf06a60dfdd3e28aceba2e09d4d4 100644 (file)
@@ -34,6 +34,7 @@
 namespace OCA\DAV\Connector\Sabre;
 
 use OC\AppFramework\Http\Request;
+use OC\Metadata\IMetadataManager;
 use OCP\Constants;
 use OCP\Files\ForbiddenException;
 use OCP\Files\StorageNotAvailableException;
@@ -41,6 +42,7 @@ use OCP\IConfig;
 use OCP\IPreview;
 use OCP\IRequest;
 use OCP\IUserSession;
+use Psr\Log\LoggerInterface;
 use Sabre\DAV\Exception\Forbidden;
 use Sabre\DAV\Exception\NotFound;
 use Sabre\DAV\IFile;
@@ -50,6 +52,7 @@ use Sabre\DAV\ServerPlugin;
 use Sabre\DAV\Tree;
 use Sabre\HTTP\RequestInterface;
 use Sabre\HTTP\ResponseInterface;
+use Sabre\Uri;
 
 class FilesPlugin extends ServerPlugin {
 
@@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin {
        public const SHARE_NOTE = '{http://nextcloud.org/ns}note';
        public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count';
        public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count';
+       public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size';
 
        /**
         * Reference to main server object
@@ -436,6 +440,29 @@ class FilesPlugin extends ServerPlugin {
                        $propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) {
                                return $node->getFileInfo()->getUploadTime();
                        });
+
+                       if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
+                               $propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) {
+                                       if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) {
+                                               return json_encode([]);
+                                       }
+
+                                       if ($node->hasMetadata('size')) {
+                                               $sizeMetadata = $node->getMetadata('size');
+                                       } else {
+                                               // This code path should not be called since we try to preload
+                                               // the metadata when loading the folder or the search results
+                                               // in one go
+                                               $metadataManager = \OC::$server->get(IMetadataManager::class);
+                                               $sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()];
+
+                                               // TODO would be nice to display this in the profiler...
+                                               \OC::$server->get(LoggerInterface::class)->warning('Inefficient fetching of metadata');
+                                       }
+
+                                       return json_encode($sizeMetadata->getMetadata());
+                               });
+                       }
                }
 
                if ($node instanceof Directory) {
@@ -448,6 +475,32 @@ class FilesPlugin extends ServerPlugin {
                        });
 
                        $requestProperties = $propFind->getRequestedProperties();
+
+                       // TODO detect dynamically which metadata groups are requested and
+                       // preload all of them and not just size
+                       if ($this->config->getSystemValueBool('enable_file_metadata', true)
+                               && in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) {
+                               // Preloading of the metadata
+                               $fileIds = [];
+                               foreach ($node->getChildren() as $child) {
+                                       /** @var \OCP\Files\Node|Node $child */
+                                       if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) {
+                                               /** @var File $child */
+                                               $fileIds[] = $child->getFileInfo()->getId();
+                                       }
+                               }
+                               /** @var IMetaDataManager $metadataManager */
+                               $metadataManager = \OC::$server->get(IMetadataManager::class);
+                               $preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
+                               foreach ($node->getChildren() as $child) {
+                                       /** @var \OCP\Files\Node|Node $child */
+                                       if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) {
+                                               /** @var File $child */
+                                               $child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]);
+                                       }
+                               }
+                       }
+
                        if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true)
                                || in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) {
                                $nbFiles = 0;
index 45e911db182a00fe1fd8068d0c6035b482d1c31f..21eb14a29bd1f7ac7a9ab09f689773345f770b70 100644 (file)
@@ -30,6 +30,7 @@ use OC\Files\Search\SearchComparison;
 use OC\Files\Search\SearchOrder;
 use OC\Files\Search\SearchQuery;
 use OC\Files\View;
+use OC\Metadata\IMetadataManager;
 use OCA\DAV\Connector\Sabre\CachingTree;
 use OCA\DAV\Connector\Sabre\Directory;
 use OCA\DAV\Connector\Sabre\FilesPlugin;
@@ -44,6 +45,7 @@ use OCP\Files\Search\ISearchQuery;
 use OCP\IUser;
 use OCP\Share\IManager;
 use Sabre\DAV\Exception\NotFound;
+use Sabre\DAV\INode;
 use SearchDAV\Backend\ISearchBackend;
 use SearchDAV\Backend\SearchPropertyDefinition;
 use SearchDAV\Backend\SearchResult;
@@ -88,14 +90,12 @@ class FileSearchBackend implements ISearchBackend {
 
        /**
         * Search endpoint will be remote.php/dav
-        *
-        * @return string
         */
-       public function getArbiterPath() {
+       public function getArbiterPath(): string {
                return '';
        }
 
-       public function isValidScope($href, $depth, $path) {
+       public function isValidScope(string $href, $depth, ?string $path): bool {
                // only allow scopes inside the dav server
                if (is_null($path)) {
                        return false;
@@ -109,7 +109,7 @@ class FileSearchBackend implements ISearchBackend {
                }
        }
 
-       public function getPropertyDefinitionsForScope($href, $path) {
+       public function getPropertyDefinitionsForScope(string $href, ?string $path): array {
                // all valid scopes support the same schema
 
                //todo dynamically load all propfind properties that are supported
@@ -132,15 +132,44 @@ class FileSearchBackend implements ISearchBackend {
                        new SearchPropertyDefinition(FilesPlugin::OWNER_DISPLAY_NAME_PROPERTYNAME, false, true, false),
                        new SearchPropertyDefinition(FilesPlugin::DATA_FINGERPRINT_PROPERTYNAME, false, true, false),
                        new SearchPropertyDefinition(FilesPlugin::HAS_PREVIEW_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_BOOLEAN),
+                       new SearchPropertyDefinition(FilesPlugin::FILE_METADATA_SIZE, false, true, false, SearchPropertyDefinition::DATATYPE_STRING),
                        new SearchPropertyDefinition(FilesPlugin::FILEID_PROPERTYNAME, false, true, false, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER),
                ];
        }
 
+       /**
+        * @param INode[] $nodes
+        * @param string[] $requestProperties
+        */
+       public function preloadPropertyFor(array $nodes, array $requestProperties): void {
+               if (in_array(FilesPlugin::FILE_METADATA_SIZE, $requestProperties, true)) {
+                       // Preloading of the metadata
+                       $fileIds = [];
+                       foreach ($nodes as $node) {
+                               /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */
+                               if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) {
+                                       /** @var \OCA\DAV\Connector\Sabre\File $node */
+                                       $fileIds[] = $node->getFileInfo()->getId();
+                               }
+                       }
+                       /** @var IMetaDataManager $metadataManager */
+                       $metadataManager = \OC::$server->get(IMetadataManager::class);
+                       $preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
+                       foreach ($nodes as $node) {
+                               /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */
+                               if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) {
+                                       /** @var \OCA\DAV\Connector\Sabre\File $node */
+                                       $node->setMetadata('size', $preloadedMetadata[$node->getFileInfo()->getId()]);
+                               }
+                       }
+               }
+       }
+
        /**
         * @param Query $search
         * @return SearchResult[]
         */
-       public function search(Query $search) {
+       public function search(Query $search): array {
                if (count($search->from) !== 1) {
                        throw new \InvalidArgumentException('Searching more than one folder is not supported');
                }
index d84c11306e3604cf957a58d335723516bce15049..c3b2f27d72a7b5d729e3e3a5fbfe9d9944a874c2 100644 (file)
@@ -22,6 +22,7 @@
  */
 namespace OCA\DAV\Files;
 
+use Sabre\DAV\INode;
 use SearchDAV\Backend\ISearchBackend;
 use SearchDAV\Query\Query;
 
@@ -35,7 +36,7 @@ class LazySearchBackend implements ISearchBackend {
                $this->backend = $backend;
        }
 
-       public function getArbiterPath() {
+       public function getArbiterPath(): string {
                if ($this->backend) {
                        return $this->backend->getArbiterPath();
                } else {
@@ -43,27 +44,30 @@ class LazySearchBackend implements ISearchBackend {
                }
        }
 
-       public function isValidScope($href, $depth, $path) {
+       public function isValidScope(string $href, $depth, ?string $path): bool {
                if ($this->backend) {
                        return $this->backend->getArbiterPath();
-               } else {
-                       return false;
                }
+               return false;
        }
 
-       public function getPropertyDefinitionsForScope($href, $path) {
+       public function getPropertyDefinitionsForScope(string $href, ?String $path): array {
                if ($this->backend) {
                        return $this->backend->getPropertyDefinitionsForScope($href, $path);
-               } else {
-                       return [];
                }
+               return [];
        }
 
-       public function search(Query $query) {
+       public function search(Query $query): array {
                if ($this->backend) {
                        return $this->backend->search($query);
-               } else {
-                       return [];
+               }
+               return [];
+       }
+
+       public function preloadPropertyFor(array $nodes, array $requestProperties): void {
+               if ($this->backend) {
+                       $this->backend->preloadPropertyFor($nodes, $requestProperties);
                }
        }
 }
index 73891b59918a855ed5f1f735943d16f96a84843a..863226922d88b14e9444828c6844d0dfbeb2be45 100644 (file)
     </InvalidScalarArgument>
   </file>
   <file src="apps/dav/lib/Files/FileSearchBackend.php">
-    <InvalidArgument occurrences="2">
-      <code>$argument</code>
-      <code>$operator-&gt;arguments</code>
-    </InvalidArgument>
     <InvalidReturnStatement occurrences="1">
       <code>$value</code>
     </InvalidReturnStatement>
     <ParamNameMismatch occurrences="1">
       <code>$search</code>
     </ParamNameMismatch>
-    <UndefinedDocblockClass occurrences="1">
-      <code>$operator-&gt;arguments[0]-&gt;name</code>
-    </UndefinedDocblockClass>
     <UndefinedPropertyFetch occurrences="1">
       <code>$operator-&gt;arguments[0]-&gt;name</code>
     </UndefinedPropertyFetch>
     <InvalidReturnStatement occurrences="1">
       <code>$this-&gt;backend-&gt;getArbiterPath()</code>
     </InvalidReturnStatement>
-    <InvalidReturnType occurrences="1">
-      <code>isValidScope</code>
-    </InvalidReturnType>
+    <InvalidReturnType occurrences="1"/>
   </file>
   <file src="apps/dav/lib/Files/RootCollection.php">
     <UndefinedFunction occurrences="1">
index 4d8dcfa5660ab724e7b63b1dbe16a684031ee6a5..378d88168cdf377c8eb949231a063c0b290a6b26 100644 (file)
@@ -2125,4 +2125,15 @@ $CONFIG = [
  * Defaults to ``true``
  */
 'profile.enabled' => true,
+
+/**
+ * Enable file metadata collection
+ *
+ * This is helpful for the mobile clients and will enable a few optimization in
+ * the future for the preview generation.
+ *
+ * Note that when enabled, this data will be stored in the database and might increase
+ * the database storage.
+ */
+'enable_file_metadata' => true,
 ];
index 545588ab208a7c3b551d11af8059d37a006d2b0e..34932cab183994edfe6786b2f1f12f70c6de2ade 100644 (file)
@@ -48,12 +48,17 @@ use OC\DB\MissingColumnInformation;
 use OC\DB\MissingIndexInformation;
 use OC\DB\MissingPrimaryKeyInformation;
 use OC\DB\SchemaWrapper;
+use OC\Metadata\FileEventListener;
 use OCP\AppFramework\App;
 use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\Events\Node\NodeDeletedEvent;
+use OCP\Files\Events\Node\NodeWrittenEvent;
+use OCP\Files\Events\NodeRemovedFromCache;
 use OCP\IDBConnection;
 use OCP\User\Events\BeforeUserDeletedEvent;
 use OCP\User\Events\UserDeletedEvent;
 use OCP\Util;
+use OCP\IConfig;
 use Symfony\Component\EventDispatcher\GenericEvent;
 
 /**
@@ -301,5 +306,15 @@ class Application extends App {
                $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, UserDeletedFilesCleanupListener::class);
                $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedFilesCleanupListener::class);
                $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedWebAuthnCleanupListener::class);
+
+               // Metadata
+               /** @var IConfig $config */
+               $config = $container->get(IConfig::class);
+               if ($config->getSystemValueBool('enable_file_metadata', true)) {
+                       $eventDispatcher = \OC::$server->get(IEventDispatcher::class);
+                       $eventDispatcher->addServiceListener(NodeDeletedEvent::class, FileEventListener::class);
+                       $eventDispatcher->addServiceListener(NodeRemovedFromCache::class, FileEventListener::class);
+                       $eventDispatcher->addServiceListener(NodeWrittenEvent::class, FileEventListener::class);
+               }
        }
 }
diff --git a/core/Migrations/Version240000Date20220404230027.php b/core/Migrations/Version240000Date20220404230027.php
new file mode 100644 (file)
index 0000000..f45f8d5
--- /dev/null
@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Add oc_file_metadata table
+ * @see OC\Metadata\FileMetadata
+ */
+class Version240000Date20220404230027 extends SimpleMigrationStep {
+       /**
+        * @param IOutput $output
+        * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+        * @param array $options
+        * @return null|ISchemaWrapper
+        */
+       public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+               /** @var ISchemaWrapper $schema */
+               $schema = $schemaClosure();
+
+               if (!$schema->hasTable('file_metadata')) {
+                       $table = $schema->createTable('file_metadata');
+                       $table->addColumn('id', Types::INTEGER, [
+                               'notnull' => true,
+                       ]);
+                       $table->addColumn('group_name', Types::STRING, [
+                               'notnull' => true,
+                               'length' => 50,
+                       ]);
+                       $table->addColumn('metadata', Types::JSON, [
+                               'notnull' => true,
+                       ]);
+                       $table->setPrimaryKey(['id', 'group_name'], 'file_metadata_idx');
+               }
+               return $schema;
+       }
+}
index d50e0c9fcc47df4f65268ae1a0b4074990160486..fc50a9f86224f81178e9e8b743ac5838181728ed 100644 (file)
@@ -264,7 +264,7 @@ class InstalledVersions
         if (null === self::$installed) {
             // only require the installed.php file if this file is loaded from its dumped location,
             // and not from its source location in the composer/composer package, see https://github.com/composer/composer/issues/9937
-            if (substr(__DIR__, -8, 1) !== 'C') {
+            if (substr(__DIR__, -8, 1) !== 'C' && is_file(__DIR__ . '/installed.php')) {
                 self::$installed = include __DIR__ . '/installed.php';
             } else {
                 self::$installed = array();
@@ -337,7 +337,7 @@ class InstalledVersions
         if (null === self::$installed) {
             // only require the installed.php file if this file is loaded from its dumped location,
             // and not from its source location in the composer/composer package, see https://github.com/composer/composer/issues/9937
-            if (substr(__DIR__, -8, 1) !== 'C') {
+            if (substr(__DIR__, -8, 1) !== 'C' && is_file(__DIR__ . '/installed.php')) {
                 self::$installed = require __DIR__ . '/installed.php';
             } else {
                 self::$installed = array();
index f27399a042d95c4708af3a8c74d35d338763cf8f..62ecfd8d0046b60517ea7370300f52744f1ab85d 100644 (file)
@@ -1,4 +1,3 @@
-
 Copyright (c) Nils Adermann, Jordi Boggiano
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
-
index acc0f6bf2ad21c7bb64fff83475c014f545936d4..be40cd4c60713556c52cfdd7bbe95e8221f79995 100644 (file)
@@ -2,7 +2,7 @@
 
 // autoload_classmap.php @generated by Composer
 
-$vendorDir = dirname(__DIR__);
+$vendorDir = dirname(dirname(__FILE__));
 $baseDir = dirname(dirname($vendorDir));
 
 return array(
@@ -1021,6 +1021,7 @@ return array(
     'OC\\Core\\Migrations\\Version23000Date20211203110726' => $baseDir . '/core/Migrations/Version23000Date20211203110726.php',
     'OC\\Core\\Migrations\\Version23000Date20211213203940' => $baseDir . '/core/Migrations/Version23000Date20211213203940.php',
     'OC\\Core\\Migrations\\Version240000Date20220202150027' => $baseDir . '/core/Migrations/Version240000Date20220202150027.php',
+    'OC\\Core\\Migrations\\Version240000Date20220404230027' => $baseDir . '/core/Migrations/Version240000Date20220404230027.php',
     'OC\\Core\\Migrations\\Version24000Date20211210141942' => $baseDir . '/core/Migrations/Version24000Date20211210141942.php',
     'OC\\Core\\Migrations\\Version24000Date20211213081506' => $baseDir . '/core/Migrations/Version24000Date20211213081506.php',
     'OC\\Core\\Migrations\\Version24000Date20211213081604' => $baseDir . '/core/Migrations/Version24000Date20211213081604.php',
@@ -1302,6 +1303,14 @@ return array(
     'OC\\Memcache\\ProfilerWrapperCache' => $baseDir . '/lib/private/Memcache/ProfilerWrapperCache.php',
     'OC\\Memcache\\Redis' => $baseDir . '/lib/private/Memcache/Redis.php',
     'OC\\MemoryInfo' => $baseDir . '/lib/private/MemoryInfo.php',
+    'OC\\Metadata\\Capabilities' => $baseDir . '/lib/private/Metadata/Capabilities.php',
+    'OC\\Metadata\\FileEventListener' => $baseDir . '/lib/private/Metadata/FileEventListener.php',
+    'OC\\Metadata\\FileMetadata' => $baseDir . '/lib/private/Metadata/FileMetadata.php',
+    'OC\\Metadata\\FileMetadataMapper' => $baseDir . '/lib/private/Metadata/FileMetadataMapper.php',
+    'OC\\Metadata\\IMetadataManager' => $baseDir . '/lib/private/Metadata/IMetadataManager.php',
+    'OC\\Metadata\\IMetadataProvider' => $baseDir . '/lib/private/Metadata/IMetadataProvider.php',
+    'OC\\Metadata\\MetadataManager' => $baseDir . '/lib/private/Metadata/MetadataManager.php',
+    'OC\\Metadata\\Provider\\ExifProvider' => $baseDir . '/lib/private/Metadata/Provider/ExifProvider.php',
     'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php',
     'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php',
     'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php',
index f1ae7a0ffec5ea6b442e7a0483e3ba7feb2e2997..4a9c20beed0715a492b947b0a858acdf2fe0066d 100644 (file)
@@ -2,7 +2,7 @@
 
 // autoload_namespaces.php @generated by Composer
 
-$vendorDir = dirname(__DIR__);
+$vendorDir = dirname(dirname(__FILE__));
 $baseDir = dirname(dirname($vendorDir));
 
 return array(
index 74e48cf69ae282da037509240c583d4634fbe094..b641d9c6a03195b92c308e6fe4929b1fbd9fe508 100644 (file)
@@ -2,7 +2,7 @@
 
 // autoload_psr4.php @generated by Composer
 
-$vendorDir = dirname(__DIR__);
+$vendorDir = dirname(dirname(__FILE__));
 $baseDir = dirname(dirname($vendorDir));
 
 return array(
index 4b1ab7678ec95b217bd76030aca3c3223e75333e..a5748c7a891a8de1828c44495e119f08538af6a7 100644 (file)
@@ -23,11 +23,30 @@ class ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c
         }
 
         spl_autoload_register(array('ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c', 'loadClassLoader'), true, true);
-        self::$loader = $loader = new \Composer\Autoload\ClassLoader(\dirname(__DIR__));
+        self::$loader = $loader = new \Composer\Autoload\ClassLoader(\dirname(\dirname(__FILE__)));
         spl_autoload_unregister(array('ComposerAutoloaderInit53792487c5a8370acc0b06b1a864ff4c', 'loadClassLoader'));
 
-        require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c::getInitializer($loader)();
+        $useStaticLoader = PHP_VERSION_ID >= 50600 && !defined('HHVM_VERSION') && (!function_exists('zend_loader_file_encoded') || !zend_loader_file_encoded());
+        if ($useStaticLoader) {
+            require __DIR__ . '/autoload_static.php';
+
+            call_user_func(\Composer\Autoload\ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c::getInitializer($loader));
+        } else {
+            $map = require __DIR__ . '/autoload_namespaces.php';
+            foreach ($map as $namespace => $path) {
+                $loader->set($namespace, $path);
+            }
+
+            $map = require __DIR__ . '/autoload_psr4.php';
+            foreach ($map as $namespace => $path) {
+                $loader->setPsr4($namespace, $path);
+            }
+
+            $classMap = require __DIR__ . '/autoload_classmap.php';
+            if ($classMap) {
+                $loader->addClassMap($classMap);
+            }
+        }
 
         $loader->register(true);
 
index 09e8d3a627e82018bfd203f49a1aeef0f62edba0..7e778d73b8323b2c4ba7ec477511ec14c1cca8aa 100644 (file)
@@ -1050,6 +1050,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Core\\Migrations\\Version23000Date20211203110726' => __DIR__ . '/../../..' . '/core/Migrations/Version23000Date20211203110726.php',
         'OC\\Core\\Migrations\\Version23000Date20211213203940' => __DIR__ . '/../../..' . '/core/Migrations/Version23000Date20211213203940.php',
         'OC\\Core\\Migrations\\Version240000Date20220202150027' => __DIR__ . '/../../..' . '/core/Migrations/Version240000Date20220202150027.php',
+        'OC\\Core\\Migrations\\Version240000Date20220404230027' => __DIR__ . '/../../..' . '/core/Migrations/Version240000Date20220404230027.php',
         'OC\\Core\\Migrations\\Version24000Date20211210141942' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211210141942.php',
         'OC\\Core\\Migrations\\Version24000Date20211213081506' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211213081506.php',
         'OC\\Core\\Migrations\\Version24000Date20211213081604' => __DIR__ . '/../../..' . '/core/Migrations/Version24000Date20211213081604.php',
@@ -1331,6 +1332,14 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Memcache\\ProfilerWrapperCache' => __DIR__ . '/../../..' . '/lib/private/Memcache/ProfilerWrapperCache.php',
         'OC\\Memcache\\Redis' => __DIR__ . '/../../..' . '/lib/private/Memcache/Redis.php',
         'OC\\MemoryInfo' => __DIR__ . '/../../..' . '/lib/private/MemoryInfo.php',
+        'OC\\Metadata\\Capabilities' => __DIR__ . '/../../..' . '/lib/private/Metadata/Capabilities.php',
+        'OC\\Metadata\\FileEventListener' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileEventListener.php',
+        'OC\\Metadata\\FileMetadata' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadata.php',
+        'OC\\Metadata\\FileMetadataMapper' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadataMapper.php',
+        'OC\\Metadata\\IMetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataManager.php',
+        'OC\\Metadata\\IMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataProvider.php',
+        'OC\\Metadata\\MetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/MetadataManager.php',
+        'OC\\Metadata\\Provider\\ExifProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/Provider/ExifProvider.php',
         'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php',
         'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php',
         'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php',
index f12a8e00dbe5f2034c37ace5214e926921e2e7a9..67a1ddfbd8c4ec11e4312ab25bc2b7d79a9b7020 100644 (file)
@@ -5,7 +5,7 @@
         'type' => 'library',
         'install_path' => __DIR__ . '/../../../',
         'aliases' => array(),
-        'reference' => '1225189f74d06606aafc4150d07584b90cea50dd',
+        'reference' => '42c7886f80c7a5e767b192d07474114dd0848b16',
         'name' => '__root__',
         'dev' => false,
     ),
@@ -16,7 +16,7 @@
             'type' => 'library',
             'install_path' => __DIR__ . '/../../../',
             'aliases' => array(),
-            'reference' => '1225189f74d06606aafc4150d07584b90cea50dd',
+            'reference' => '42c7886f80c7a5e767b192d07474114dd0848b16',
             'dev_requirement' => false,
         ),
     ),
index 42e212271d58ccc16a8c91d28b7f79a9c285b2fe..3b8cbdb18bb3ebece33a27c3c88c71d0ada320f2 100644 (file)
@@ -31,7 +31,7 @@ use OC\Files\Storage\Storage;
 
 class NoopScanner extends Scanner {
        public function __construct(Storage $storage) {
-               //we don't need the storage, so do nothing here
+               // we don't need the storage, so do nothing here
        }
 
        /**
diff --git a/lib/private/Metadata/Capabilities.php b/lib/private/Metadata/Capabilities.php
new file mode 100644 (file)
index 0000000..2fa0006
--- /dev/null
@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Metadata;
+
+use OCP\Capabilities\IPublicCapability;
+use OCP\IConfig;
+
+class Capabilities implements IPublicCapability {
+       private IMetadataManager $manager;
+       private IConfig $config;
+
+       public function __construct(IMetadataManager $manager, IConfig $config) {
+               $this->manager = $manager;
+               $this->config = $config;
+       }
+
+       public function getCapabilities() {
+               if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
+                       return ['metadataAvailable' => $this->manager->getCapabilities()];
+               }
+
+               return [];
+       }
+}
diff --git a/lib/private/Metadata/FileEventListener.php b/lib/private/Metadata/FileEventListener.php
new file mode 100644 (file)
index 0000000..fdec891
--- /dev/null
@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Metadata;
+
+use OC\Files\Filesystem;
+use OCP\EventDispatcher\Event;
+use OCP\EventDispatcher\IEventListener;
+use OCP\Files\Events\Node\NodeDeletedEvent;
+use OCP\Files\Events\Node\NodeWrittenEvent;
+use OCP\Files\Events\NodeRemovedFromCache;
+use OCP\Files\File;
+use OCP\Files\Node;
+use OCP\Files\NotFoundException;
+use OCP\Files\FileInfo;
+
+class FileEventListener implements IEventListener {
+       private IMetadataManager $manager;
+
+       public function __construct(IMetadataManager $manager) {
+               $this->manager = $manager;
+       }
+
+       private function shouldExtractMetadata(Node $node): bool {
+               try {
+                       if ($node->getMimetype() === 'httpd/unix-directory') {
+                               return false;
+                       }
+               } catch (NotFoundException $e) {
+                       return false;
+               }
+               if ($node->getSize(false) <= 0) {
+                       return false;
+               }
+
+               $path = $node->getPath();
+               // TODO make this more dynamic, we have the same issue in other places
+               return !str_starts_with($path, 'appdata_') && !str_starts_with($path, 'files_versions/') && !str_starts_with($path, 'files_trashbin/');
+       }
+
+       public function handle(Event $event): void {
+               if ($event instanceof NodeRemovedFromCache) {
+                       $view = Filesystem::getView();
+                       $info = $view->getFileInfo($event->getPath());
+                       if ($info && $info->getType() === FileInfo::TYPE_FILE) {
+                               $this->manager->clearMetadata($info->getId());
+                       }
+               }
+
+               if ($event instanceof NodeDeletedEvent) {
+                       $node = $event->getNode();
+                       if ($this->shouldExtractMetadata($node)) {
+                               /** @var File $node */
+                               $this->manager->clearMetadata($event->getNode()->getId());
+                       }
+               }
+
+               if ($event instanceof NodeWrittenEvent) {
+                       $node = $event->getNode();
+                       if ($this->shouldExtractMetadata($node)) {
+                               /** @var File $node */
+                               $this->manager->generateMetadata($event->getNode(), false);
+                       }
+               }
+       }
+}
diff --git a/lib/private/Metadata/FileMetadata.php b/lib/private/Metadata/FileMetadata.php
new file mode 100644 (file)
index 0000000..c53f5d7
--- /dev/null
@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Metadata;
+
+use OCP\AppFramework\Db\Entity;
+use OCP\DB\Types;
+
+/**
+ * @method string getGroupName()
+ * @method void setGroupName(string $groupName)
+ * @method string getMetadata()
+ * @method void setMetadata(array $metadata)
+ * @see OC\Core\Migrations\Version240000Date20220404230027
+ */
+class FileMetadata extends Entity {
+       protected ?string $groupName = null;
+       protected ?array $metadata = null;
+
+       public function __construct() {
+               $this->addType('groupName', 'string');
+               $this->addType('metadata', Types::JSON);
+       }
+}
diff --git a/lib/private/Metadata/FileMetadataMapper.php b/lib/private/Metadata/FileMetadataMapper.php
new file mode 100644 (file)
index 0000000..53f750a
--- /dev/null
@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Metadata;
+
+use OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Db\MultipleObjectsReturnedException;
+use OCP\AppFramework\Db\QBMapper;
+use OCP\DB\Exception;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+
+class FileMetadataMapper extends QBMapper {
+       public function __construct(IDBConnection $db) {
+               parent::__construct($db, 'file_metadata', FileMetadata::class);
+       }
+
+       /**
+        * @return FileMetadata[]
+        * @throws Exception
+        */
+       public function findForFile(int $fileId): array {
+               $qb = $this->db->getQueryBuilder();
+               $qb->select('*')
+                       ->from($this->getTableName())
+                       ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+
+               return $this->findEntities($qb);
+       }
+
+       /**
+        * @throws DoesNotExistException
+        * @throws MultipleObjectsReturnedException
+        * @throws Exception
+        */
+       public function findForGroupForFile(int $fileId, string $groupName): FileMetadata {
+               $qb = $this->db->getQueryBuilder();
+               $qb->select('*')
+                       ->from($this->getTableName())
+                       ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT_ARRAY)))
+                       ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR)));
+
+               return $this->findEntity($qb);
+       }
+
+       /**
+        * @return array<int, FileMetadata>
+        * @throws Exception
+        */
+       public function findForGroupForFiles(array $fileIds, string $groupName): array {
+               $qb = $this->db->getQueryBuilder();
+               $qb->select('*')
+                       ->from($this->getTableName())
+                       ->where($qb->expr()->in('id', $qb->createParameter('fileIds')))
+                       ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR)));
+
+               $metadata = [];
+               foreach (array_chunk($fileIds, 1000) as $fileIdsChunk) {
+                       $qb->setParameter('fileIds', $fileIdsChunk, IQueryBuilder::PARAM_INT_ARRAY);
+                       /** @var FileMetadata[] $rawEntities */
+                       $rawEntities = $this->findEntities($qb);
+                       foreach ($rawEntities as $entity) {
+                               $metadata[$entity->getId()] = $entity;
+                       }
+               }
+
+               foreach ($fileIds as $id) {
+                       if (isset($metadata[$id])) {
+                               continue;
+                       }
+                       $empty = new FileMetadata();
+                       $empty->setMetadata([]);
+                       $empty->setGroupName($groupName);
+                       $empty->setId($id);
+                       $metadata[$id] = $empty;
+               }
+               return $metadata;
+       }
+
+       public function clear(int $fileId): void {
+               $qb = $this->db->getQueryBuilder();
+               $qb->delete($this->getTableName())
+                       ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+
+               $qb->executeStatement();
+       }
+}
diff --git a/lib/private/Metadata/IMetadataManager.php b/lib/private/Metadata/IMetadataManager.php
new file mode 100644 (file)
index 0000000..d2d37f1
--- /dev/null
@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\Metadata;
+
+use OCP\Files\File;
+
+/**
+ * Interface to manage additional metadata for files
+ */
+interface IMetadataManager {
+       /**
+        * @param class-string<IMetadataProvider> $className
+        */
+       public function registerProvider(string $className): void;
+
+       /**
+        * Generate the metadata for one file
+        */
+       public function generateMetadata(File $file, bool $checkExisting = false): void;
+
+       /**
+        * Clear the metadata for one file
+        */
+       public function clearMetadata(int $fileId): void;
+
+       /** @return array<int, FileMetadata> */
+       public function fetchMetadataFor(string $group, array $fileIds): array;
+
+       /**
+        * Get the capabilites as an array of mimetype regex to the type provided
+        */
+       public function getCapabilities(): array;
+}
diff --git a/lib/private/Metadata/IMetadataProvider.php b/lib/private/Metadata/IMetadataProvider.php
new file mode 100644 (file)
index 0000000..7cbe102
--- /dev/null
@@ -0,0 +1,41 @@
+<?php
+
+namespace OC\Metadata;
+
+use OCP\Files\File;
+
+/**
+ * Interface for the metadata providers. If you want an application to provide
+ * some metadata, you can use this to store them.
+ */
+interface IMetadataProvider {
+       /**
+        * The list of groups that this metadata provider is able to provide.
+        *
+        * @return string[]
+        */
+       public static function groupsProvided(): array;
+
+       /**
+        * Check if the metadata provider is available. A metadata provider might be
+        * unavailable due to a php extension not being installed.
+        */
+       public static function isAvailable(): bool;
+
+       /**
+        * Get the mimetypes supported as a regex.
+        */
+       public static function getMimetypesSupported(): string;
+
+       /**
+        * Execute the extraction on the specified file. The metadata should be
+        * grouped by metadata
+        *
+        * Each group should be json serializable and the string representation
+        * shouldn't be longer than 4000 characters.
+        *
+        * @param File $file The file to extract the metadata from
+        * @param array<string, FileMetadata> An array containing all the metadata fetched.
+        */
+       public function execute(File $file): array;
+}
diff --git a/lib/private/Metadata/MetadataManager.php b/lib/private/Metadata/MetadataManager.php
new file mode 100644 (file)
index 0000000..69e9cb3
--- /dev/null
@@ -0,0 +1,100 @@
+<?php
+/**
+ * @copyright Copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program. If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Metadata;
+
+use OC\Metadata\Provider\ExifProvider;
+use OCP\Files\File;
+use OCP\IConfig;
+use Psr\Log\LoggerInterface;
+
+class MetadataManager implements IMetadataManager {
+       /** @var array<string, IMetadataProvider> */
+       private array $providers;
+       private array $providerClasses;
+       private FileMetadataMapper $fileMetadataMapper;
+       private IConfig $config;
+       private LoggerInterface $logger;
+
+       public function __construct(
+               FileMetadataMapper $fileMetadataMapper,
+               IConfig $config,
+               LoggerInterface $logger
+       ) {
+               $this->providers = [];
+               $this->providerClasses = [];
+               $this->fileMetadataMapper = $fileMetadataMapper;
+               $this->config = $config;
+               $this->logger = $logger;
+
+               // TODO move to another place, where?
+               $this->registerProvider(ExifProvider::class);
+       }
+
+       /**
+        * @param class-string<IMetadataProvider> $className
+        */
+       public function registerProvider(string $className):void {
+               if (in_array($className, $this->providerClasses)) {
+                       return;
+               }
+
+               if (call_user_func([$className, 'isAvailable'])) {
+                       $this->providers[call_user_func([$className, 'getMimetypesSupported'])] = \OC::$server->get($className);
+               }
+       }
+
+       public function generateMetadata(File $file, bool $checkExisting = false): void {
+               $existingMetadataGroups = [];
+
+               if ($checkExisting) {
+                       $existingMetadata = $this->fileMetadataMapper->findForFile($file->getId());
+                       foreach ($existingMetadata as $metadata) {
+                               $existingMetadataGroups[] = $metadata->getGroupName();
+                       }
+               }
+
+               foreach ($this->providers as $supportedMimetype => $provider) {
+                       if (preg_match($supportedMimetype, $file->getMimeType())) {
+                               if (count(array_diff($provider::groupsProvided(), $existingMetadataGroups)) > 0) {
+                                       $metaDataGroup = $provider->execute($file);
+                                       foreach ($metaDataGroup as $group => $metadata) {
+                                               $this->fileMetadataMapper->insertOrUpdate($metadata);
+                                       }
+                               }
+                       }
+               }
+       }
+
+       public function clearMetadata(int $fileId): void {
+               $this->fileMetadataMapper->clear($fileId);
+       }
+
+       public function fetchMetadataFor(string $group, array $fileIds): array {
+               return $this->fileMetadataMapper->findForGroupForFiles($fileIds, $group);
+       }
+
+       public function getCapabilities(): array {
+               $capabilities = [];
+               foreach ($this->providers as $supportedMimetype => $provider) {
+                       $capabilities[$supportedMimetype] = $provider::groupsProvided();
+               }
+               return $capabilities;
+       }
+}
diff --git a/lib/private/Metadata/Provider/ExifProvider.php b/lib/private/Metadata/Provider/ExifProvider.php
new file mode 100644 (file)
index 0000000..91c858f
--- /dev/null
@@ -0,0 +1,51 @@
+<?php
+
+namespace OC\Metadata\Provider;
+
+use OC\Metadata\FileMetadata;
+use OC\Metadata\IMetadataProvider;
+use OCP\Files\File;
+
+class ExifProvider implements IMetadataProvider {
+       public static function groupsProvided(): array {
+               return ['size'];
+       }
+
+       public static function isAvailable(): bool {
+               return extension_loaded('exif');
+       }
+
+       public function execute(File $file): array {
+               $fileDescriptor = $file->fopen('rb');
+               $data = @exif_read_data($fileDescriptor, 'ANY_TAG', true);
+
+               $size = new FileMetadata();
+               $size->setGroupName('size');
+               $size->setId($file->getId());
+               $size->setMetadata([]);
+
+               if (!$data) {
+                       return [
+                               'size' => $size,
+                       ];
+               }
+
+               if (array_key_exists('COMPUTED', $data)
+                       && array_key_exists('Width', $data['COMPUTED'])
+                       && array_key_exists('Height', $data['COMPUTED'])
+               ) {
+                       $size->setMetadata([
+                               'width' => $data['COMPUTED']['Width'],
+                               'height' => $data['COMPUTED']['Height'],
+                       ]);
+               }
+
+               return [
+                       'size' => $size,
+               ];
+       }
+
+       public static function getMimetypesSupported(): string {
+               return '/image\/.*/';
+       }
+}
index 7817d1beafeacd04c83e95d71bcad517e9ca6831..e9d673d3746edaa005f2cf21dffa061864878c90 100644 (file)
@@ -122,6 +122,9 @@ use OC\Log\PsrLoggerAdapter;
 use OC\Mail\Mailer;
 use OC\Memcache\ArrayCache;
 use OC\Memcache\Factory;
+use OC\Metadata\Capabilities as MetadataCapabilities;
+use OC\Metadata\IMetadataManager;
+use OC\Metadata\MetadataManager;
 use OC\Notification\Manager;
 use OC\OCS\DiscoveryService;
 use OC\Preview\GeneratorHelper;
@@ -151,7 +154,6 @@ use OC\Template\JSCombiner;
 use OCA\Theming\ImageManager;
 use OCA\Theming\ThemingDefaults;
 use OCA\Theming\Util;
-use OCA\WorkflowEngine\Service\Logger;
 use OCP\Accounts\IAccountManager;
 use OCP\App\IAppManager;
 use OCP\Authentication\LoginCredentials\IStore;
@@ -241,15 +243,12 @@ use OCP\SystemTag\ISystemTagManager;
 use OCP\SystemTag\ISystemTagObjectMapper;
 use OCP\Talk\IBroker;
 use OCP\User\Events\BeforePasswordUpdatedEvent;
-use OCP\User\Events\BeforeUserCreatedEvent;
-use OCP\User\Events\BeforeUserDeletedEvent;
 use OCP\User\Events\BeforeUserLoggedInEvent;
 use OCP\User\Events\BeforeUserLoggedInWithCookieEvent;
 use OCP\User\Events\BeforeUserLoggedOutEvent;
 use OCP\User\Events\PasswordUpdatedEvent;
 use OCP\User\Events\PostLoginEvent;
 use OCP\User\Events\UserChangedEvent;
-use OCP\User\Events\UserDeletedEvent;
 use OCP\User\Events\UserLoggedInEvent;
 use OCP\User\Events\UserLoggedInWithCookieEvent;
 use OCP\User\Events\UserLoggedOutEvent;
@@ -1163,6 +1162,9 @@ class Server extends ServerContainer implements IServerContainer {
                        $manager->registerCapability(function () use ($c) {
                                return $c->get(\OC\Security\Bruteforce\Capabilities::class);
                        });
+                       $manager->registerCapability(function () use ($c) {
+                               return $c->get(MetadataCapabilities::class);
+                       });
                        return $manager;
                });
                /** @deprecated 19.0.0 */
@@ -1433,6 +1435,8 @@ class Server extends ServerContainer implements IServerContainer {
 
                $this->registerAlias(IBroker::class, Broker::class);
 
+               $this->registerAlias(IMetadataManager::class, MetadataManager::class);
+
                $this->connectDispatcher();
        }
 
index 89e8f69859e87a659cd3a1c13258c5482156f6a5..a059e3a27b0cba471ae1c8b0ddd61edc65bd90fe 100644 (file)
@@ -120,6 +120,10 @@ abstract class Entity {
                                        if (!$args[0] instanceof \DateTime) {
                                                $args[0] = new \DateTime($args[0]);
                                        }
+                               } elseif ($type === 'json') {
+                                       if (!is_array($args[0])) {
+                                               $args[0] = json_decode($args[0], true);
+                                       }
                                } else {
                                        settype($args[0], $type);
                                }
index 5124650bc193a5c9ca451a2a1eddfd00fa0e4ef2..fa753a09dcfaab6c40d35d175edf61b30ec73c5c 100644 (file)
@@ -253,6 +253,8 @@ abstract class QBMapper {
                                return IQueryBuilder::PARAM_LOB;
                        case 'datetime':
                                return IQueryBuilder::PARAM_DATE;
+                       case 'json':
+                               return IQueryBuilder::PARAM_JSON;
                }
 
                return IQueryBuilder::PARAM_STR;
index 76754f7bf4146206e7d25736fdd74dbadace8bb0..afca9e372ee51203e13ea54a9d552c0a15364f09 100644 (file)
@@ -64,6 +64,11 @@ interface IQueryBuilder {
         */
        public const PARAM_DATE = 'datetime';
 
+       /**
+        * @since 24.0.0
+        */
+       public const PARAM_JSON = 'json';
+
        /**
         * @since 9.0.0
         */
index 4636ac3389fca5feb6fe6ecea349e40c0a5acf68..31a474b03a078e54d31c53a9eb12df70765ef81e 100644 (file)
@@ -110,4 +110,10 @@ final class Types {
         * @since 21.0.0
         */
        public const TIME = 'time';
+
+       /**
+        * @var string
+        * @since 24.0.0
+        */
+       public const JSON = 'json';
 }
index b03034056d2d207e10f63bdaf7b58aa21f501ead..96d319923b393f95c709a6b606a80ae9519ea803 100644 (file)
@@ -47,6 +47,7 @@ class QBTestEntity extends Entity {
        protected $stringProp;
        protected $integerProp;
        protected $booleanProp;
+       protected $jsonProp;
 
        public function __construct() {
                $this->addType('intProp', 'int');
@@ -54,11 +55,10 @@ class QBTestEntity extends Entity {
                $this->addType('stringProp', 'string');
                $this->addType('integerProp', 'integer');
                $this->addType('booleanProp', 'boolean');
+               $this->addType('jsonProp', 'json');
        }
 }
 
-;
-
 /**
  * Class QBTestMapper
  *
@@ -69,7 +69,7 @@ class QBTestMapper extends QBMapper {
                parent::__construct($db, 'table');
        }
 
-       public function getParameterTypeForPropertyForTest(Entity $entity, string $property): int {
+       public function getParameterTypeForPropertyForTest(Entity $entity, string $property) {
                return parent::getParameterTypeForProperty($entity, $property);
        }
 }
@@ -171,6 +171,7 @@ class QBMapperTest extends \Test\TestCase {
                $entity->setStringProp('string');
                $entity->setIntegerProp(456);
                $entity->setBooleanProp(false);
+               $entity->setJsonProp(["hello" => "world"]);
 
                $idParam = $this->qb->createNamedParameter('id', IQueryBuilder::PARAM_INT);
                $intParam = $this->qb->createNamedParameter('int_prop', IQueryBuilder::PARAM_INT);
@@ -178,8 +179,9 @@ class QBMapperTest extends \Test\TestCase {
                $stringParam = $this->qb->createNamedParameter('string_prop', IQueryBuilder::PARAM_STR);
                $integerParam = $this->qb->createNamedParameter('integer_prop', IQueryBuilder::PARAM_INT);
                $booleanParam = $this->qb->createNamedParameter('boolean_prop', IQueryBuilder::PARAM_BOOL);
+               $jsonParam = $this->qb->createNamedParameter('json_prop', IQueryBuilder::PARAM_JSON);
 
-               $this->qb->expects($this->exactly(6))
+               $this->qb->expects($this->exactly(7))
                        ->method('createNamedParameter')
                        ->withConsecutive(
                                [$this->equalTo(123), $this->equalTo(IQueryBuilder::PARAM_INT)],
@@ -187,17 +189,19 @@ class QBMapperTest extends \Test\TestCase {
                                [$this->equalTo('string'), $this->equalTo(IQueryBuilder::PARAM_STR)],
                                [$this->equalTo(456), $this->equalTo(IQueryBuilder::PARAM_INT)],
                                [$this->equalTo(false), $this->equalTo(IQueryBuilder::PARAM_BOOL)],
-                               [$this->equalTo(789), $this->equalTo(IQueryBuilder::PARAM_INT)]
+                               [$this->equalTo(["hello" => "world"]), $this->equalTo(IQueryBuilder::PARAM_JSON)],
+                               [$this->equalTo(789), $this->equalTo(IQueryBuilder::PARAM_INT)],
                        );
 
-               $this->qb->expects($this->exactly(5))
+               $this->qb->expects($this->exactly(6))
                        ->method('set')
                        ->withConsecutive(
                                [$this->equalTo('int_prop'), $this->equalTo($intParam)],
                                [$this->equalTo('bool_prop'), $this->equalTo($boolParam)],
                                [$this->equalTo('string_prop'), $this->equalTo($stringParam)],
                                [$this->equalTo('integer_prop'), $this->equalTo($integerParam)],
-                               [$this->equalTo('boolean_prop'), $this->equalTo($booleanParam)]
+                               [$this->equalTo('boolean_prop'), $this->equalTo($booleanParam)],
+                               [$this->equalTo('json_prop'), $this->equalTo($jsonParam)]
                        );
 
                $this->expr->expects($this->once())
@@ -227,6 +231,9 @@ class QBMapperTest extends \Test\TestCase {
                $stringType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'stringProp');
                $this->assertEquals(IQueryBuilder::PARAM_STR, $stringType, 'String type property mapping incorrect');
 
+               $jsonType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'jsonProp');
+               $this->assertEquals(IQueryBuilder::PARAM_JSON, $jsonType, 'JSON type property mapping incorrect');
+
                $unknownType = $this->mapper->getParameterTypeForPropertyForTest($entity, 'someProp');
                $this->assertEquals(IQueryBuilder::PARAM_STR, $unknownType, 'Unknown type property mapping incorrect');
        }
index 114cf03d7bee921bf47b50d4014dd41149fa510d..af44159efa3528fda7fc054ad26edbdfdb6c2794 100644 (file)
@@ -267,6 +267,8 @@ class MigratorTest extends \Test\TestCase {
 
                        [ParameterType::INTEGER, 1234, Types::INTEGER, false],
                        [ParameterType::INTEGER, 0, Types::INTEGER, false], // Integer 0 is not stored as Null and therefor works
+
+                       [ParameterType::STRING, '{"a": 2}', Types::JSON, false],
                ];
        }
 
diff --git a/tests/lib/Metadata/FileMetadataMapperTest.php b/tests/lib/Metadata/FileMetadataMapperTest.php
new file mode 100644 (file)
index 0000000..8e38535
--- /dev/null
@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright 2022 Carl Schwan <carl@carlschwan.eu>
+ * @license AGPL-3.0-or-later
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace Test\Metadata;
+
+use OC\Metadata\FileMetadataMapper;
+use OC\Metadata\FileMetadata;
+
+/**
+ * @group DB
+ * @package Test\DB\QueryBuilder
+ */
+class FileMetadataMapperTest extends \Test\TestCase {
+       /** @var IDBConnection */
+       protected $connection;
+
+       /** @var SystemConfig|\PHPUnit\Framework\MockObject\MockObject */
+       protected $config;
+
+       protected function setUp(): void {
+               parent::setUp();
+
+               $this->connection = \OC::$server->getDatabaseConnection();
+               $this->mapper = new FileMetadataMapper($this->connection);
+       }
+
+       public function testFindForGroupForFiles() {
+               $file1 = new FileMetadata();
+               $file1->setId(1);
+               $file1->setGroupName('size');
+               $file1->setMetadata([]);
+
+               $file2 = new FileMetadata();
+               $file2->setId(2);
+               $file2->setGroupName('size');
+               $file2->setMetadata(['width' => 293, 'height' => 23]);
+
+               // not added, it's the default
+               $file3 = new FileMetadata();
+               $file3->setId(3);
+               $file3->setGroupName('size');
+               $file3->setMetadata([]);
+
+               $file4 = new FileMetadata();
+               $file4->setId(4);
+               $file4->setGroupName('size');
+               $file4->setMetadata(['complex' => ["yes", "maybe" => 34.0]]);
+
+               $this->mapper->insert($file1);
+               $this->mapper->insert($file2);
+               $this->mapper->insert($file4);
+
+               $files = $this->mapper->findForGroupForFiles([1, 2, 3, 4], 'size');
+
+               $this->assertEquals($files[1]->getMetadata(), $file1->getMetadata());
+               $this->assertEquals($files[2]->getMetadata(), $file2->getMetadata());
+               $this->assertEquals($files[3]->getMetadata(), $file3->getMetadata());
+               $this->assertEquals($files[4]->getMetadata(), $file4->getMetadata());
+
+               $this->mapper->clear(1);
+               $this->mapper->clear(2);
+               $this->mapper->clear(4);
+       }
+}
index 8ceb2dd58c029d554ce3400dffd3c2f52e413d69..d805557260d201266d7b1f218d9a567fa578093c 100644 (file)
@@ -30,7 +30,7 @@
 // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel
 // when updating major/minor version number.
 
-$OC_Version = [24, 0, 0, 8];
+$OC_Version = [24, 0, 0, 9];
 
 // The human readable string
 $OC_VersionString = '24.0.0 beta 3';