aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobin Appelman <robin@icewind.nl>2024-02-05 18:56:43 +0100
committerRobin Appelman <robin@icewind.nl>2024-02-15 17:55:43 +0100
commit7ca516773f2866b3a6bb2e8cb63b5df95d8da03e (patch)
tree8b9fa1fa362c542223b51fc218a8e735bb3c9924
parent2dcd0a875920be09944dc51f360303c11f3e858a (diff)
downloadnextcloud-server-7ca516773f2866b3a6bb2e8cb63b5df95d8da03e.tar.gz
nextcloud-server-7ca516773f2866b3a6bb2e8cb63b5df95d8da03e.zip
add a search query step to split IN statements that are to large for oci
Signed-off-by: Robin Appelman <robin@icewind.nl>
-rw-r--r--lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php1
-rw-r--r--lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php33
-rw-r--r--tests/lib/Files/Search/SearchIntegrationTest.php44
3 files changed, 78 insertions, 0 deletions
diff --git a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
index 6240ef3367e..86cd784b760 100644
--- a/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
+++ b/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php
@@ -37,6 +37,7 @@ class QueryOptimizer {
new FlattenSingleArgumentBinaryOperation(),
new OrEqualsToIn(),
new FlattenNestedBool(),
+ new SplitLargeIn(),
];
}
diff --git a/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php
new file mode 100644
index 00000000000..450ffae42f1
--- /dev/null
+++ b/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+
+/**
+ * transform IN (1000+ element) into (IN (1000 elements) OR IN(...))
+ */
+class SplitLargeIn extends ReplacingOptimizerStep {
+ public function processOperator(ISearchOperator &$operator): bool {
+ if (
+ $operator instanceof ISearchComparison &&
+ $operator->getType() === ISearchComparison::COMPARE_IN &&
+ count($operator->getValue()) > 1000
+ ) {
+ $chunks = array_chunk($operator->getValue(), 1000);
+ $chunkComparisons = array_map(function(array $values) use ($operator) {
+ return new SearchComparison(ISearchComparison::COMPARE_IN, $operator->getField(), $values);
+ }, $chunks);
+
+ $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $chunkComparisons);
+ return true;
+ }
+ parent::processOperator($operator);
+ return false;
+ }
+}
+
diff --git a/tests/lib/Files/Search/SearchIntegrationTest.php b/tests/lib/Files/Search/SearchIntegrationTest.php
new file mode 100644
index 00000000000..74018a597d9
--- /dev/null
+++ b/tests/lib/Files/Search/SearchIntegrationTest.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace Test\Files\Search;
+
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
+use OC\Files\Storage\Temporary;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use Test\TestCase;
+
+/**
+ * @group DB
+ */
+class SearchIntegrationTest extends TestCase {
+ private $cache;
+ private $storage;
+
+ protected function setUp(): void {
+ parent::setUp();
+
+ $this->storage = new Temporary([]);
+ $this->cache = $this->storage->getCache();
+ $this->storage->getScanner()->scan('');
+ }
+
+
+ public function testThousandAndOneFilters() {
+ $id = $this->cache->put("file10", ['size' => 1, 'mtime' => 50, 'mimetype' => 'foo/folder']);
+
+ $comparisons = [];
+ for($i = 1; $i <= 1001; $i++) {
+ $comparisons[] = new SearchComparison(ISearchComparison::COMPARE_EQUAL, "name", "file$i");
+ }
+ $operator = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $comparisons);
+ $query = new SearchQuery($operator, 10, 0, []);
+
+ $results = $this->cache->searchQuery($query);
+
+ $this->assertCount(1, $results);
+ $this->assertEquals($id, $results[0]->getId());
+ }
+}
#008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
<?php
/**
 * @copyright 2017, Roeland Jago Douma <roeland@famdouma.nl>
 *
 * @author Lukas Reschke <lukas@statuscode.ch>
 * @author Morris Jobke <hey@morrisjobke.de>
 * @author Roeland Jago Douma <roeland@famdouma.nl>
 *
 * @license GNU AGPL version 3 or any later version
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
namespace OC\Template;

use OC\SystemConfig;
use OCP\ICache;
use OCP\Files\IAppData;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\Files\SimpleFS\ISimpleFolder;
use OCP\ICacheFactory;
use OCP\ILogger;
use OCP\IURLGenerator;

class JSCombiner {

	/** @var IAppData */
	protected $appData;

	/** @var IURLGenerator */
	protected $urlGenerator;

	/** @var ICache */
	protected $depsCache;

	/** @var SystemConfig */
	protected $config;

	/** @var ILogger */
	protected $logger;

	/** @var ICacheFactory */
	private $cacheFactory;

	/**
	 * @param IAppData $appData
	 * @param IURLGenerator $urlGenerator
	 * @param ICacheFactory $cacheFactory
	 * @param SystemConfig $config
	 * @param ILogger $logger
	 */
	public function __construct(IAppData $appData,
								IURLGenerator $urlGenerator,
								ICacheFactory $cacheFactory,
								SystemConfig $config,
								ILogger $logger) {
		$this->appData = $appData;
		$this->urlGenerator = $urlGenerator;
		$this->cacheFactory = $cacheFactory;
		$this->depsCache = $this->cacheFactory->createDistributed('JS-' . md5($this->urlGenerator->getBaseUrl()));
		$this->config = $config;
		$this->logger = $logger;
	}

	/**
	 * @param string $root
	 * @param string $file
	 * @param string $app
	 * @return bool
	 */
	public function process($root, $file, $app) {
		if ($this->config->getValue('debug') || !$this->config->getValue('installed')) {
			return false;
		}

		$path = explode('/', $root . '/' . $file);

		$fileName = array_pop($path);
		$path = implode('/', $path);

		try {
			$folder = $this->appData->getFolder($app);
		} catch(NotFoundException $e) {
			// creating css appdata folder
			$folder = $this->appData->newFolder($app);
		}

		if($this->isCached($fileName, $folder)) {
			return true;
		}
		return $this->cache($path, $fileName, $folder);
	}

	/**
	 * @param string $fileName
	 * @param ISimpleFolder $folder
	 * @return bool
	 */
	protected function isCached($fileName, ISimpleFolder $folder) {
		$fileName = str_replace('.json', '.js', $fileName);

		if (!$folder->fileExists($fileName)) {
			return false;
		}

		$fileName = $fileName . '.deps';
		try {
			$deps = $this->depsCache->get($folder->getName() . '-' . $fileName);
			if ($deps === null || $deps === '') {
				$depFile = $folder->getFile($fileName);
				$deps = $depFile->getContent();
			}

			// check again
			if ($deps === null || $deps === '') {
				$this->logger->info('JSCombiner: deps file empty: ' . $fileName);
				return false;
			}

			$deps = json_decode($deps, true);

			if ($deps === NULL) {
				return false;
			}

			foreach ($deps as $file=>$mtime) {
				if (!file_exists($file) || filemtime($file) > $mtime) {
					return false;
				}
			}

			return true;
		} catch(NotFoundException $e) {
			return false;
		}
	}

	/**
	 * @param string $path
	 * @param string $fileName
	 * @param ISimpleFolder $folder
	 * @return bool
	 */
	protected function cache($path, $fileName, ISimpleFolder $folder) {
		$deps = [];
		$fullPath = $path . '/' . $fileName;
		$data = json_decode(file_get_contents($fullPath));
		$deps[$fullPath] = filemtime($fullPath);

		$res = '';
		foreach ($data as $file) {
			$filePath = $path . '/' . $file;

			if (is_file($filePath)) {
				$res .= file_get_contents($filePath);
				$res .= PHP_EOL . PHP_EOL;
				$deps[$filePath] = filemtime($filePath);
			}
		}

		$fileName = str_replace('.json', '.js', $fileName);
		try {
			$cachedfile = $folder->getFile($fileName);
		} catch(NotFoundException $e) {
			$cachedfile = $folder->newFile($fileName);
		}

		$depFileName = $fileName . '.deps';
		try {
			$depFile = $folder->getFile($depFileName);
		} catch (NotFoundException $e) {
			$depFile = $folder->newFile($depFileName);
		}

		try {
			$gzipFile = $folder->getFile($fileName . '.gzip'); # Safari doesn't like .gz
		} catch (NotFoundException $e) {
			$gzipFile = $folder->newFile($fileName . '.gzip'); # Safari doesn't like .gz
		}

		try {
			$cachedfile->putContent($res);
			$deps = json_encode($deps);
			$depFile->putContent($deps);
			$this->depsCache->set($folder->getName() . '-' . $depFileName, $deps);
			$gzipFile->putContent(gzencode($res, 9));
			$this->logger->debug('JSCombiner: successfully cached: ' . $fileName);
			return true;
		} catch (NotPermittedException $e) {
			$this->logger->error('JSCombiner: unable to cache: ' . $fileName);
			return false;
		}
	}

	/**
	 * @param string $appName
	 * @param string $fileName
	 * @return string
	 */
	public function getCachedJS($appName, $fileName) {
		$tmpfileLoc = explode('/', $fileName);
		$fileName = array_pop($tmpfileLoc);
		$fileName = str_replace('.json', '.js', $fileName);

		return substr($this->urlGenerator->linkToRoute('core.Js.getJs', array('fileName' => $fileName, 'appName' => $appName)), strlen(\OC::$WEBROOT) + 1);
	}

	/**
	 * @param string $root
	 * @param string $file
	 * @return string[]
	 */
	public function getContent($root, $file) {
		/** @var array $data */
		$data = json_decode(file_get_contents($root . '/' . $file));
		if(!is_array($data)) {
			return [];
		}

		$path = explode('/', $file);
		array_pop($path);
		$path = implode('/', $path);

		$result = [];
		foreach ($data as $f) {
			$result[] = $path . '/' . $f;
		}

		return $result;
	}


	/**
	 * Clear cache with combined javascript files
	 *
	 * @throws NotFoundException
	 */
	public function resetCache() {
		$this->cacheFactory->createDistributed('JS-')->clear();
		$appDirectory = $this->appData->getDirectoryListing();
		foreach ($appDirectory as $folder) {
			foreach ($folder->getDirectoryListing() as $file) {
				$file->delete();
			}
		}
	}
}