add a search query step to split IN statements that are to large for oci

Signed-off-by: Robin Appelman <robin@icewind.nl>

3 files changed, 78 insertions, 0 deletions

diff --git a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
index 6240ef3367e..86cd784b760 100644
--- a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
+++ b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
@@ -37,6 +37,7 @@ class QueryOptimizer {
 			new FlattenSingleArgumentBinaryOperation(),
 			new OrEqualsToIn(),
 			new FlattenNestedBool(),
+			new SplitLargeIn(),
 		];
 	}
 
diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php
new file mode 100644
index 00000000000..450ffae42f1
--- /dev/null
+++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+
+/**
+ * transform IN (1000+ element) into (IN (1000 elements) OR IN(...))
+ */
+class SplitLargeIn extends ReplacingOptimizerStep {
+	public function processOperator(ISearchOperator &$operator): bool {
+		if (
+			$operator instanceof ISearchComparison &&
+			$operator->getType() === ISearchComparison::COMPARE_IN &&
+			count($operator->getValue()) > 1000
+		) {
+			$chunks = array_chunk($operator->getValue(), 1000);
+			$chunkComparisons = array_map(function(array $values) use ($operator) {
+				return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values);
+			}, $chunks);
+
+			$operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons);
+			return true;
+		}
+		parent::processOperator($operator);
+		return false;
+	}
+}
+
diff --git a/tests/lib/Files/Search/SearchIntegrationTest.php b/tests/lib/Files/Search/SearchIntegrationTest.php
new file mode 100644
index 00000000000..74018a597d9
--- /dev/null
+++ b/tests/lib/Files/Search/SearchIntegrationTest.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace Test\Files\Search;
+
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
+use OC\Files\Storage\Temporary;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use Test\TestCase;
+
+/**
+ * @group DB
+ */
+class SearchIntegrationTest extends TestCase {
+	private $cache;
+	private $storage;
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$this->storage = new Temporary([]);
+		$this->cache = $this->storage->getCache();
+		$this->storage->getScanner()->scan('');
+	}
+
+
+	public function testThousandAndOneFilters() {
+		$id = $this->cache->put("file10", ['size' => 1, 'mtime' => 50, 'mimetype' => 'foo/folder']);
+
+		$comparisons = [];
+		for($i = 1; $i <= 1001; $i++) {
+			$comparisons[] = new SearchComparison(ISearchComparison::COMPARE_EQUAL, "name", "file$i");
+		}
+		$operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $comparisons);
+		$query = new SearchQuery($operator, 10, 0, []);
+
+		$results = $this->cache->searchQuery($query);
+
+		$this->assertCount(1, $results);
+		$this->assertEquals($id, $results[0]->getId());
+	}
+}