IFilesMetadata

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

3 files changed, 112 insertions, 58 deletions

diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php
index 34d2177b84e..f799d6aa72e 100644
--- a/lib/private/Files/Cache/CacheQueryBuilder.php
+++ b/lib/private/Files/Cache/CacheQueryBuilder.php
@@ -5,6 +5,7 @@ declare(strict_types=1);
 /**
  * @copyright Copyright (c) 2019 Robin Appelman <robin@icewind.nl>
  *
+ * @author Maxence Lange <maxence@artificial-owl.com>
  * @author Robin Appelman <robin@icewind.nl>
  *
  * @license GNU AGPL version 3 or any later version
@@ -35,7 +36,7 @@ use Psr\Log\LoggerInterface;
  * Query builder with commonly used helpers for filecache queries
  */
 class CacheQueryBuilder extends QueryBuilder {
-	private $alias = null;
+	private ?string $alias = null;
 
 	public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) {
 		parent::__construct($connection, $systemConfig, $logger);
diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php
index 15c089a0f11..ca54133a243 100644
--- a/lib/private/Files/Cache/QuerySearchHelper.php
+++ b/lib/private/Files/Cache/QuerySearchHelper.php
@@ -3,6 +3,7 @@
  * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
  *
  * @author Christoph Wurst <christoph@winzerhof-wurst.at>
+ * @author Maxence Lange <maxence@artificial-owl.com>
  * @author Robin Appelman <robin@icewind.nl>
  * @author Roeland Jago Douma <roeland@famdouma.nl>
  * @author Tobias Kaminsky <tobias@kaminsky.me>
@@ -37,41 +38,24 @@ use OCP\Files\IRootFolder;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\Search\ISearchBinaryOperator;
 use OCP\Files\Search\ISearchQuery;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use OCP\FilesMetadata\Model\IMetadataQuery;
 use OCP\IDBConnection;
 use OCP\IGroupManager;
 use OCP\IUser;
 use Psr\Log\LoggerInterface;
 
 class QuerySearchHelper {
-	/** @var IMimeTypeLoader */
-	private $mimetypeLoader;
-	/** @var IDBConnection */
-	private $connection;
-	/** @var SystemConfig */
-	private $systemConfig;
-	private LoggerInterface $logger;
-	/** @var SearchBuilder */
-	private $searchBuilder;
-	/** @var QueryOptimizer */
-	private $queryOptimizer;
-	private IGroupManager $groupManager;
-
 	public function __construct(
-		IMimeTypeLoader $mimetypeLoader,
-		IDBConnection $connection,
-		SystemConfig $systemConfig,
-		LoggerInterface $logger,
-		SearchBuilder $searchBuilder,
-		QueryOptimizer $queryOptimizer,
-		IGroupManager $groupManager,
+		private IMimeTypeLoader $mimetypeLoader,
+		private IDBConnection $connection,
+		private SystemConfig $systemConfig,
+		private LoggerInterface $logger,
+		private SearchBuilder $searchBuilder,
+		private QueryOptimizer $queryOptimizer,
+		private IGroupManager $groupManager,
+		private IFilesMetadataManager $filesMetadataManager,
 	) {
-		$this->mimetypeLoader = $mimetypeLoader;
-		$this->connection = $connection;
-		$this->systemConfig = $systemConfig;
-		$this->logger = $logger;
-		$this->searchBuilder = $searchBuilder;
-		$this->queryOptimizer = $queryOptimizer;
-		$this->groupManager = $groupManager;
 	}
 
 	protected function getQueryBuilder() {
@@ -82,7 +66,12 @@ class QuerySearchHelper {
 		);
 	}
 
-	protected function applySearchConstraints(CacheQueryBuilder $query, ISearchQuery $searchQuery, array $caches): void {
+	protected function applySearchConstraints(
+		CacheQueryBuilder $query,
+		ISearchQuery $searchQuery,
+		array $caches,
+		?IMetadataQuery $metadataQuery = null
+	): void {
 		$storageFilters = array_values(array_map(function (ICache $cache) {
 			return $cache->getQueryFilterForStorage();
 		}, $caches));
@@ -90,12 +79,12 @@ class QuerySearchHelper {
 		$filter = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [$searchQuery->getSearchOperation(), $storageFilter]);
 		$this->queryOptimizer->processOperator($filter);
 
-		$searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter);
+		$searchExpr = $this->searchBuilder->searchOperatorToDBExpr($query, $filter, $metadataQuery);
 		if ($searchExpr) {
 			$query->andWhere($searchExpr);
 		}
 
-		$this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder());
+		$this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder(), $metadataQuery);
 
 		if ($searchQuery->getLimit()) {
 			$query->setMaxResults($searchQuery->getLimit());
@@ -144,6 +133,20 @@ class QuerySearchHelper {
 			));
 	}
 
+
+	/**
+	 * left join metadata and its indexes to the filecache table
+	 *
+	 * @param CacheQueryBuilder $query
+	 *
+	 * @return IMetadataQuery
+	 */
+	protected function equipQueryForMetadata(CacheQueryBuilder $query): IMetadataQuery {
+		$metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'file', 'fileid');
+		$metadataQuery->retrieveMetadata();
+		return $metadataQuery;
+	}
+
 	/**
 	 * Perform a file system search in multiple caches
 	 *
@@ -175,6 +178,7 @@ class QuerySearchHelper {
 		$query = $builder->selectFileCache('file', false);
 
 		$requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation());
+
 		if (in_array('systemtag', $requestedFields)) {
 			$this->equipQueryForSystemTags($query, $this->requireUser($searchQuery));
 		}
@@ -182,12 +186,14 @@ class QuerySearchHelper {
 			$this->equipQueryForDavTags($query, $this->requireUser($searchQuery));
 		}
 
-		$this->applySearchConstraints($query, $searchQuery, $caches);
+		$metadataQuery = $this->equipQueryForMetadata($query);
+		$this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery);
 
 		$result = $query->execute();
 		$files = $result->fetchAll();
 
-		$rawEntries = array_map(function (array $data) {
+		$rawEntries = array_map(function (array $data) use ($metadataQuery) {
+			$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
 			return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
 		}, $files);
 
diff --git a/lib/private/Files/Cache/SearchBuilder.php b/lib/private/Files/Cache/SearchBuilder.php
index b9a70bbd39b..ba4aabf2b4f 100644
--- a/lib/private/Files/Cache/SearchBuilder.php
+++ b/lib/private/Files/Cache/SearchBuilder.php
@@ -3,6 +3,7 @@
  * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
  *
  * @author Christoph Wurst <christoph@winzerhof-wurst.at>
+ * @author Maxence Lange <maxence@artificial-owl.com>
  * @author Robin Appelman <robin@icewind.nl>
  * @author Roeland Jago Douma <roeland@famdouma.nl>
  * @author Tobias Kaminsky <tobias@kaminsky.me>
@@ -32,6 +33,7 @@ use OCP\Files\Search\ISearchBinaryOperator;
 use OCP\Files\Search\ISearchComparison;
 use OCP\Files\Search\ISearchOperator;
 use OCP\Files\Search\ISearchOrder;
+use OCP\FilesMetadata\Model\IMetadataQuery;
 
 /**
  * Tools for transforming search queries into database queries
@@ -76,7 +78,7 @@ class SearchBuilder {
 			return array_reduce($operator->getArguments(), function (array $fields, ISearchOperator $operator) {
 				return array_unique(array_merge($fields, $this->extractRequestedFields($operator)));
 			}, []);
-		} elseif ($operator instanceof ISearchComparison) {
+		} elseif ($operator instanceof ISearchComparison && !$operator->isExtra()) {
 			return [$operator->getField()];
 		}
 		return [];
@@ -86,13 +88,21 @@ class SearchBuilder {
 	 * @param IQueryBuilder $builder
 	 * @param ISearchOperator[] $operators
 	 */
-	public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) {
-		return array_filter(array_map(function ($operator) use ($builder) {
-			return $this->searchOperatorToDBExpr($builder, $operator);
+	public function searchOperatorArrayToDBExprArray(
+		IQueryBuilder $builder,
+		array $operators,
+		?IMetadataQuery $metadataQuery = null
+	) {
+		return array_filter(array_map(function ($operator) use ($builder, $metadataQuery) {
+			return $this->searchOperatorToDBExpr($builder, $operator, $metadataQuery);
 		}, $operators));
 	}
 
-	public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) {
+	public function searchOperatorToDBExpr(
+		IQueryBuilder $builder,
+		ISearchOperator $operator,
+		?IMetadataQuery $metadataQuery = null
+	) {
 		$expr = $builder->expr();
 
 		if ($operator instanceof ISearchBinaryOperator) {
@@ -104,29 +114,37 @@ class SearchBuilder {
 				case ISearchBinaryOperator::OPERATOR_NOT:
 					$negativeOperator = $operator->getArguments()[0];
 					if ($negativeOperator instanceof ISearchComparison) {
-						return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap);
+						return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap, $metadataQuery);
 					} else {
 						throw new \InvalidArgumentException('Binary operators inside "not" is not supported');
 					}
 					// no break
 				case ISearchBinaryOperator::OPERATOR_AND:
-					return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
+					return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery));
 				case ISearchBinaryOperator::OPERATOR_OR:
-					return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
+					return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments(), $metadataQuery));
 				default:
 					throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType());
 			}
 		} elseif ($operator instanceof ISearchComparison) {
-			return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap);
+			return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap, $metadataQuery);
 		} else {
 			throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator));
 		}
 	}
 
-	private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) {
-		$this->validateComparison($comparison);
+	private function searchComparisonToDBExpr(
+		IQueryBuilder $builder,
+		ISearchComparison $comparison,
+		array $operatorMap,
+		?IMetadataQuery $metadataQuery = null
+	) {
+		if ($comparison->getExtra()) {
+			[$field, $value, $type] = $this->getExtraOperatorField($comparison, $metadataQuery);
+		} else {
+			[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
+		}
 
-		[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
 		if (isset($operatorMap[$type])) {
 			$queryOperator = $operatorMap[$type];
 			return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
@@ -136,9 +154,12 @@ class SearchBuilder {
 	}
 
 	private function getOperatorFieldAndValue(ISearchComparison $operator) {
+		$this->validateComparison($operator);
+
 		$field = $operator->getField();
 		$value = $operator->getValue();
 		$type = $operator->getType();
+
 		if ($field === 'mimetype') {
 			$value = (string)$value;
 			if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
@@ -213,6 +234,24 @@ class SearchBuilder {
 		}
 	}
 
+
+	private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array {
+		$field = $operator->getField();
+		$value = $operator->getValue();
+		$type = $operator->getType();
+
+		switch($operator->getExtra()) {
+			case IMetadataQuery::EXTRA:
+				$metadataQuery->joinIndex($field); // join index table if not joined yet
+				$field = $metadataQuery->getMetadataValueField($field);
+				break;
+			default:
+				throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra());
+		}
+
+		return [$field, $value, $type];
+	}
+
 	private function getParameterForValue(IQueryBuilder $builder, $value) {
 		if ($value instanceof \DateTime) {
 			$value = $value->getTimestamp();
@@ -228,24 +267,32 @@ class SearchBuilder {
 	/**
 	 * @param IQueryBuilder $query
 	 * @param ISearchOrder[] $orders
+	 * @param IMetadataQuery|null $metadataQuery
 	 */
-	public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) {
+	public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders, ?IMetadataQuery $metadataQuery = null): void {
 		foreach ($orders as $order) {
 			$field = $order->getField();
-			if ($field === 'fileid') {
-				$field = 'file.fileid';
-			}
+			switch ($order->getExtra()) {
+				case IMetadataQuery::EXTRA:
+					$metadataQuery->joinIndex($field); // join index table if not joined yet
+					$field = $metadataQuery->getMetadataValueField($order->getField());
+					break;
 
-			// Mysql really likes to pick an index for sorting if it can't fully satisfy the where
-			// filter with an index, since search queries pretty much never are fully filtered by index
-			// mysql often picks an index for sorting instead of the much more useful index for filtering.
-			//
-			// By changing the order by to an expression, mysql isn't smart enough to see that it could still
-			// use the index, so it instead picks an index for the filtering
-			if ($field === 'mtime') {
-				$field = $query->func()->add($field, $query->createNamedParameter(0));
-			}
+				default:
+					if ($field === 'fileid') {
+						$field = 'file.fileid';
+					}
 
+					// Mysql really likes to pick an index for sorting if it can't fully satisfy the where
+					// filter with an index, since search queries pretty much never are fully filtered by index
+					// mysql often picks an index for sorting instead of the much more useful index for filtering.
+					//
+					// By changing the order by to an expression, mysql isn't smart enough to see that it could still
+					// use the index, so it instead picks an index for the filtering
+					if ($field === 'mtime') {
+						$field = $query->func()->add($field, $query->createNamedParameter(0));
+					}
+			}
 			$query->addOrderBy($field, $order->getDirection());
 		}
 	}