diff options
Diffstat (limited to 'lib/private/DB')
66 files changed, 4593 insertions, 2545 deletions
diff --git a/lib/private/DB/Adapter.php b/lib/private/DB/Adapter.php index 62514e7d83a..8f1b8e6d75f 100644 --- a/lib/private/DB/Adapter.php +++ b/lib/private/DB/Adapter.php @@ -1,44 +1,21 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jonny007-MKD <1-23-4-5@web.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Ole Ostergaard <ole.c.ostergaard@gmail.com> - * @author Ole Ostergaard <ole.ostergaard@knime.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; +use OC\DB\Exceptions\DbalException; /** * This handles the way we use to write queries, into something that can be * handled by the database abstraction layer. */ class Adapter { - /** * @var \OC\DB\Connection $conn */ @@ -55,7 +32,7 @@ class Adapter { * @throws Exception */ public function lastInsertId($table) { - return (int) $this->conn->realLastInsertId($table); + return (int)$this->conn->realLastInsertId($table); } /** @@ -69,13 +46,12 @@ class Adapter { /** * Create an exclusive read+write lock on a table * - * @param string $tableName * @throws Exception * @since 9.1.0 */ - public function lockTable($tableName) { + public function lockTable(string $tableName) { $this->conn->beginTransaction(); - $this->conn->executeUpdate('LOCK TABLE `' .$tableName . '` IN EXCLUSIVE MODE'); + $this->conn->executeUpdate('LOCK TABLE `' . $tableName . '` IN EXCLUSIVE MODE'); } /** @@ -96,19 +72,21 @@ class Adapter { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 */ - public function insertIfNotExist($table, $input, array $compare = null) { - if (empty($compare)) { - $compare = array_keys($input); - } - $query = 'INSERT INTO `' .$table . '` (`' - . implode('`,`', array_keys($input)) . '`) SELECT ' - . str_repeat('?,', count($input) - 1).'? ' // Is there a prettier alternative? + public function insertIfNotExist($table, $input, ?array $compare = null) { + $compare = $compare ?: array_keys($input); + + // Prepare column names and generate placeholders + $columns = '`' . implode('`,`', array_keys($input)) . '`'; + $placeholders = implode(', ', array_fill(0, count($input), '?')); + + $query = 'INSERT INTO `' . $table . '` (' . $columns . ') ' + . 'SELECT ' . $placeholders . ' ' . 'FROM `' . $table . '` WHERE '; $inserts = array_values($input); @@ -127,10 +105,9 @@ class Adapter { try { return $this->conn->executeUpdate($query, $inserts); } catch (UniqueConstraintViolationException $e) { - // if this is thrown then a concurrent insert happened between the insert and the sub-select in the insert, that should have avoided it - // it's fine to ignore this then - // - // more discussions about this can be found at https://github.com/nextcloud/server/pull/12315 + // This exception indicates a concurrent insert happened between + // the insert and the sub-select in the insert, which is safe to ignore. + // More details: https://github.com/nextcloud/server/pull/12315 return 0; } } @@ -138,16 +115,19 @@ class Adapter { /** * @throws \OCP\DB\Exception */ - public function insertIgnoreConflict(string $table,array $values) : int { + public function insertIgnoreConflict(string $table, array $values) : int { try { $builder = $this->conn->getQueryBuilder(); $builder->insert($table); foreach ($values as $key => $value) { $builder->setValue($key, $builder->createNamedParameter($value)); } - return $builder->execute(); - } catch (UniqueConstraintViolationException $e) { - return 0; + return $builder->executeStatement(); + } catch (DbalException $e) { + if ($e->getReason() === \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + return 0; + } + throw $e; } } } diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index c925f9362f5..63c75607379 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -1,39 +1,21 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class AdapterMySQL extends Adapter { - /** @var string */ - protected $charset; + protected $collation; /** * @param string $tableName */ public function lockTable($tableName) { - $this->conn->executeUpdate('LOCK TABLES `' .$tableName . '` WRITE'); + $this->conn->executeUpdate('LOCK TABLES `' . $tableName . '` WRITE'); } public function unlockTable() { @@ -41,16 +23,44 @@ class AdapterMySQL extends Adapter { } public function fixupStatement($statement) { - $statement = str_replace(' ILIKE ', ' COLLATE ' . $this->getCharset() . '_general_ci LIKE ', $statement); + $statement = str_replace(' ILIKE ', ' COLLATE ' . $this->getCollation() . ' LIKE ', $statement); return $statement; } - protected function getCharset() { - if (!$this->charset) { + protected function getCollation(): string { + if (!$this->collation) { $params = $this->conn->getParams(); - $this->charset = isset($params['charset']) ? $params['charset'] : 'utf8'; + $this->collation = $params['collation'] ?? (($params['charset'] ?? 'utf8') . '_general_ci'); + } + + return $this->collation; + } + + public function insertIgnoreConflict(string $table, array $values): int { + $builder = $this->conn->getQueryBuilder(); + $builder->insert($table); + $updates = []; + foreach ($values as $key => $value) { + $builder->setValue($key, $builder->createNamedParameter($value)); } - return $this->charset; + /* + * We can't use ON DUPLICATE KEY UPDATE here because Nextcloud use the CLIENT_FOUND_ROWS flag + * With this flag the MySQL returns the number of selected rows + * instead of the number of affected/modified rows + * It's impossible to change this behaviour at runtime or for a single query + * Then, the result is 1 if a row is inserted and also 1 if a row is updated with same or different values + * + * With INSERT IGNORE, the result is 1 when a row is inserted, 0 otherwise + * + * Risk: it can also ignore other errors like type mismatch or truncated data… + */ + $res = $this->conn->executeStatement( + preg_replace('/^INSERT/i', 'INSERT IGNORE', $builder->getSQL()), + $builder->getParameters(), + $builder->getParameterTypes() + ); + + return $res; } } diff --git a/lib/private/DB/AdapterOCI8.php b/lib/private/DB/AdapterOCI8.php index 553945691a6..0a509090bca 100644 --- a/lib/private/DB/AdapterOCI8.php +++ b/lib/private/DB/AdapterOCI8.php @@ -1,29 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class AdapterOCI8 extends Adapter { diff --git a/lib/private/DB/AdapterPgSql.php b/lib/private/DB/AdapterPgSql.php index 0d8794d59ac..db48c81c2c5 100644 --- a/lib/private/DB/AdapterPgSql.php +++ b/lib/private/DB/AdapterPgSql.php @@ -1,34 +1,13 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Arthur Schiwon <blizzz@arthur-schiwon.de> - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Ole Ostergaard <ole.c.ostergaard@gmail.com> - * @author Ole Ostergaard <ole.ostergaard@knime.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class AdapterPgSql extends Adapter { - protected $compatModePre9_5 = null; public function lastInsertId($table) { $result = $this->conn->executeQuery('SELECT lastval()'); @@ -44,11 +23,7 @@ class AdapterPgSql extends Adapter { return $statement; } - public function insertIgnoreConflict(string $table,array $values) : int { - if ($this->isPre9_5CompatMode() === true) { - return parent::insertIgnoreConflict($table, $values); - } - + public function insertIgnoreConflict(string $table, array $values) : int { // "upsert" is only available since PgSQL 9.5, but the generic way // would leave error logs in the DB. $builder = $this->conn->getQueryBuilder(); @@ -59,17 +34,4 @@ class AdapterPgSql extends Adapter { $queryString = $builder->getSQL() . ' ON CONFLICT DO NOTHING'; return $this->conn->executeUpdate($queryString, $builder->getParameters(), $builder->getParameterTypes()); } - - protected function isPre9_5CompatMode(): bool { - if ($this->compatModePre9_5 !== null) { - return $this->compatModePre9_5; - } - - $result = $this->conn->executeQuery('SHOW SERVER_VERSION'); - $version = $result->fetchOne(); - $result->free(); - $this->compatModePre9_5 = version_compare($version, '9.5', '<'); - - return $this->compatModePre9_5; - } } diff --git a/lib/private/DB/AdapterSqlite.php b/lib/private/DB/AdapterSqlite.php index a62960bae1a..aeadf55ecf7 100644 --- a/lib/private/DB/AdapterSqlite.php +++ b/lib/private/DB/AdapterSqlite.php @@ -1,36 +1,15 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; class AdapterSqlite extends Adapter { - /** * @param string $tableName */ @@ -59,19 +38,19 @@ class AdapterSqlite extends Adapter { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws \Doctrine\DBAL\Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 */ - public function insertIfNotExist($table, $input, array $compare = null) { + public function insertIfNotExist($table, $input, ?array $compare = null) { if (empty($compare)) { $compare = array_keys($input); } $fieldList = '`' . implode('`,`', array_keys($input)) . '`'; $query = "INSERT INTO `$table` ($fieldList) SELECT " - . str_repeat('?,', count($input) - 1).'? ' + . str_repeat('?,', count($input) - 1) . '? ' . " WHERE NOT EXISTS (SELECT 1 FROM `$table` WHERE "; $inserts = array_values($input); @@ -97,4 +76,19 @@ class AdapterSqlite extends Adapter { return 0; } } + + public function insertIgnoreConflict(string $table, array $values): int { + $builder = $this->conn->getQueryBuilder(); + $builder->insert($table); + $updates = []; + foreach ($values as $key => $value) { + $builder->setValue($key, $builder->createNamedParameter($value)); + } + + return $this->conn->executeStatement( + $builder->getSQL() . ' ON CONFLICT DO NOTHING', + $builder->getParameters(), + $builder->getParameterTypes() + ); + } } diff --git a/lib/private/DB/ArrayResult.php b/lib/private/DB/ArrayResult.php new file mode 100644 index 00000000000..b567ad23d57 --- /dev/null +++ b/lib/private/DB/ArrayResult.php @@ -0,0 +1,74 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB; + +use OCP\DB\IResult; +use PDO; + +/** + * Wrap an array or rows into a result interface + */ +class ArrayResult implements IResult { + protected int $count; + + public function __construct( + protected array $rows, + ) { + $this->count = count($this->rows); + } + + public function closeCursor(): bool { + // noop + return true; + } + + public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { + $row = array_shift($this->rows); + if (!$row) { + return false; + } + return match ($fetchMode) { + PDO::FETCH_ASSOC => $row, + PDO::FETCH_NUM => array_values($row), + PDO::FETCH_COLUMN => current($row), + default => throw new \InvalidArgumentException('Fetch mode not supported for array result'), + }; + + } + + public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->rows, + PDO::FETCH_NUM => array_map(function ($row) { + return array_values($row); + }, $this->rows), + PDO::FETCH_COLUMN => array_map(function ($row) { + return current($row); + }, $this->rows), + default => throw new \InvalidArgumentException('Fetch mode not supported for array result'), + }; + } + + public function fetchColumn() { + return $this->fetchOne(); + } + + public function fetchOne() { + $row = $this->fetch(); + if ($row) { + return current($row); + } else { + return false; + } + } + + public function rowCount(): int { + return $this->count; + } +} diff --git a/lib/private/DB/BacktraceDebugStack.php b/lib/private/DB/BacktraceDebugStack.php new file mode 100644 index 00000000000..4afd3ce6a13 --- /dev/null +++ b/lib/private/DB/BacktraceDebugStack.php @@ -0,0 +1,20 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB; + +use Doctrine\DBAL\Logging\DebugStack; + +class BacktraceDebugStack extends DebugStack { + public function startQuery($sql, ?array $params = null, ?array $types = null) { + parent::startQuery($sql, $params, $types); + $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + $this->queries[$this->currentQuery]['backtrace'] = $backtrace; + $this->queries[$this->currentQuery]['start'] = $this->start; + } +} diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 4d9433122ce..f86cbc341a4 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -1,59 +1,55 @@ <?php declare(strict_types=1); - /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Julius Härtl <jus@bitgrid.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Ole Ostergaard <ole.c.ostergaard@gmail.com> - * @author Ole Ostergaard <ole.ostergaard@knime.com> - * @author Philipp Schaffrath <github@philipp.schaffrath.email> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\Common\EventManager; use Doctrine\DBAL\Cache\QueryCacheProfile; use Doctrine\DBAL\Configuration; +use Doctrine\DBAL\Connections\PrimaryReadReplicaConnection; use Doctrine\DBAL\Driver; +use Doctrine\DBAL\Driver\ServerInfoAwareConnection; use Doctrine\DBAL\Exception; -use Doctrine\DBAL\Exception\ConstraintViolationException; -use Doctrine\DBAL\Exception\NotNullConstraintViolationException; +use Doctrine\DBAL\Exception\ConnectionLost; +use Doctrine\DBAL\Platforms\MariaDBPlatform; use Doctrine\DBAL\Platforms\MySQLPlatform; +use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; +use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Statement; +use OC\DB\QueryBuilder\Partitioned\PartitionedQueryBuilder; +use OC\DB\QueryBuilder\Partitioned\PartitionSplit; use OC\DB\QueryBuilder\QueryBuilder; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\CrossShardMoveHelper; +use OC\DB\QueryBuilder\Sharded\RoundRobinShardMapper; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\Sharded\IShardMapper; +use OCP\Diagnostics\IEventLogger; +use OCP\ICacheFactory; +use OCP\IDBConnection; use OCP\ILogger; +use OCP\IRequestId; use OCP\PreConditionNotMetException; - -class Connection extends \Doctrine\DBAL\Connection { +use OCP\Profiler\IProfiler; +use OCP\Security\ISecureRandom; +use OCP\Server; +use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; +use function count; +use function in_array; + +class Connection extends PrimaryReadReplicaConnection { /** @var string */ protected $tablePrefix; @@ -63,8 +59,9 @@ class Connection extends \Doctrine\DBAL\Connection { /** @var SystemConfig */ private $systemConfig; - /** @var ILogger */ - private $logger; + private ClockInterface $clock; + + private LoggerInterface $logger; protected $lockedTable = null; @@ -74,18 +71,183 @@ class Connection extends \Doctrine\DBAL\Connection { /** @var int */ protected $queriesExecuted = 0; + /** @var DbDataCollector|null */ + protected $dbDataCollector = null; + private array $lastConnectionCheck = []; + + protected ?float $transactionActiveSince = null; + + /** @var array<string, int> */ + protected $tableDirtyWrites = []; + + protected bool $logDbException = false; + private ?array $transactionBacktrace = null; + + protected bool $logRequestId; + protected string $requestId; + + /** @var array<string, list<string>> */ + protected array $partitions; + /** @var ShardDefinition[] */ + protected array $shards = []; + protected ShardConnectionManager $shardConnectionManager; + protected AutoIncrementHandler $autoIncrementHandler; + protected bool $isShardingEnabled; + + public const SHARD_PRESETS = [ + 'filecache' => [ + 'companion_keys' => [ + 'file_id', + ], + 'companion_tables' => [ + 'filecache_extended', + 'files_metadata', + ], + 'primary_key' => 'fileid', + 'shard_key' => 'storage', + 'table' => 'filecache', + ], + ]; + + /** + * Initializes a new instance of the Connection class. + * + * @throws \Exception + */ + public function __construct( + private array $params, + Driver $driver, + ?Configuration $config = null, + ?EventManager $eventManager = null, + ) { + if (!isset($params['adapter'])) { + throw new \Exception('adapter not set'); + } + if (!isset($params['tablePrefix'])) { + throw new \Exception('tablePrefix not set'); + } + /** + * @psalm-suppress InternalMethod + */ + parent::__construct($params, $driver, $config, $eventManager); + $this->adapter = new $params['adapter']($this); + $this->tablePrefix = $params['tablePrefix']; + $this->isShardingEnabled = isset($this->params['sharding']) && !empty($this->params['sharding']); + + if ($this->isShardingEnabled) { + /** @psalm-suppress InvalidArrayOffset */ + $this->shardConnectionManager = $this->params['shard_connection_manager'] ?? Server::get(ShardConnectionManager::class); + /** @psalm-suppress InvalidArrayOffset */ + $this->autoIncrementHandler = $this->params['auto_increment_handler'] ?? new AutoIncrementHandler( + Server::get(ICacheFactory::class), + $this->shardConnectionManager, + ); + } + $this->systemConfig = \OC::$server->getSystemConfig(); + $this->clock = Server::get(ClockInterface::class); + $this->logger = Server::get(LoggerInterface::class); + + $this->logRequestId = $this->systemConfig->getValue('db.log_request_id', false); + $this->logDbException = $this->systemConfig->getValue('db.log_exceptions', false); + $this->requestId = Server::get(IRequestId::class)->getId(); + + /** @var \OCP\Profiler\IProfiler */ + $profiler = Server::get(IProfiler::class); + if ($profiler->isEnabled()) { + $this->dbDataCollector = new DbDataCollector($this); + $profiler->add($this->dbDataCollector); + $debugStack = new BacktraceDebugStack(); + $this->dbDataCollector->setDebugStack($debugStack); + $this->_config->setSQLLogger($debugStack); + } + + /** @var array<string, array{shards: array[], mapper: ?string, from_primary_key: ?int, from_shard_key: ?int}> $shardConfig */ + $shardConfig = $this->params['sharding'] ?? []; + $shardNames = array_keys($shardConfig); + $this->shards = array_map(function (array $config, string $name) { + if (!isset(self::SHARD_PRESETS[$name])) { + throw new \Exception("Shard preset $name not found"); + } + + $shardMapperClass = $config['mapper'] ?? RoundRobinShardMapper::class; + $shardMapper = Server::get($shardMapperClass); + if (!$shardMapper instanceof IShardMapper) { + throw new \Exception("Invalid shard mapper: $shardMapperClass"); + } + return new ShardDefinition( + self::SHARD_PRESETS[$name]['table'], + self::SHARD_PRESETS[$name]['primary_key'], + self::SHARD_PRESETS[$name]['companion_keys'], + self::SHARD_PRESETS[$name]['shard_key'], + $shardMapper, + self::SHARD_PRESETS[$name]['companion_tables'], + $config['shards'], + $config['from_primary_key'] ?? 0, + $config['from_shard_key'] ?? 0, + ); + }, $shardConfig, $shardNames); + $this->shards = array_combine($shardNames, $this->shards); + $this->partitions = array_map(function (ShardDefinition $shard) { + return array_merge([$shard->table], $shard->companionTables); + }, $this->shards); + + $this->setNestTransactionsWithSavepoints(true); + } + + /** + * @return IDBConnection[] + */ + public function getShardConnections(): array { + $connections = []; + if ($this->isShardingEnabled) { + foreach ($this->shards as $shardDefinition) { + foreach ($shardDefinition->getAllShards() as $shard) { + if ($shard !== ShardDefinition::MIGRATION_SHARD) { + /** @var ConnectionAdapter $connection */ + $connections[] = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + } + } + } + } + return $connections; + } + /** * @throws Exception */ - public function connect() { + public function connect($connectionName = null) { try { - return parent::connect(); + if ($this->_conn) { + $this->reconnectIfNeeded(); + /** @psalm-suppress InternalMethod */ + return parent::connect(); + } + + // Only trigger the event logger for the initial connect call + $eventLogger = Server::get(IEventLogger::class); + $eventLogger->start('connect:db', 'db connection opened'); + /** @psalm-suppress InternalMethod */ + $status = parent::connect(); + $eventLogger->end('connect:db'); + + $this->lastConnectionCheck[$this->getConnectionName()] = time(); + + return $status; } catch (Exception $e) { // throw a new exception to prevent leaking info from the stacktrace throw new Exception('Failed to connect to the database: ' . $e->getMessage(), $e->getCode()); } } + protected function performConnect(?string $connectionName = null): bool { + if (($connectionName ?? 'replica') === 'replica' + && count($this->params['replica']) === 1 + && $this->params['primary'] === $this->params['replica'][0]) { + return parent::performConnect('primary'); + } + return parent::performConnect($connectionName); + } + public function getStats(): array { return [ 'built' => $this->queriesBuilt, @@ -98,22 +260,38 @@ class Connection extends \Doctrine\DBAL\Connection { */ public function getQueryBuilder(): IQueryBuilder { $this->queriesBuilt++; - return new QueryBuilder( + + $builder = new QueryBuilder( new ConnectionAdapter($this), $this->systemConfig, $this->logger ); + if ($this->isShardingEnabled && count($this->partitions) > 0) { + $builder = new PartitionedQueryBuilder( + $builder, + $this->shards, + $this->shardConnectionManager, + $this->autoIncrementHandler, + ); + foreach ($this->partitions as $name => $tables) { + $partition = new PartitionSplit($name, $tables); + $builder->addPartition($partition); + } + return $builder; + } else { + return $builder; + } } /** * Gets the QueryBuilder for the connection. * * @return \Doctrine\DBAL\Query\QueryBuilder - * @deprecated please use $this->getQueryBuilder() instead + * @deprecated 8.0.0 please use $this->getQueryBuilder() instead */ public function createQueryBuilder() { $backtrace = $this->getCallerBacktrace(); - \OC::$server->getLogger()->debug('Doctrine QueryBuilder retrieved in {backtrace}', ['app' => 'core', 'backtrace' => $backtrace]); + $this->logger->debug('Doctrine QueryBuilder retrieved in {backtrace}', ['app' => 'core', 'backtrace' => $backtrace]); $this->queriesBuilt++; return parent::createQueryBuilder(); } @@ -122,11 +300,11 @@ class Connection extends \Doctrine\DBAL\Connection { * Gets the ExpressionBuilder for the connection. * * @return \Doctrine\DBAL\Query\Expression\ExpressionBuilder - * @deprecated please use $this->getQueryBuilder()->expr() instead + * @deprecated 8.0.0 please use $this->getQueryBuilder()->expr() instead */ public function getExpressionBuilder() { $backtrace = $this->getCallerBacktrace(); - \OC::$server->getLogger()->debug('Doctrine ExpressionBuilder retrieved in {backtrace}', ['app' => 'core', 'backtrace' => $backtrace]); + $this->logger->debug('Doctrine ExpressionBuilder retrieved in {backtrace}', ['app' => 'core', 'backtrace' => $backtrace]); $this->queriesBuilt++; return parent::getExpressionBuilder(); } @@ -156,53 +334,29 @@ class Connection extends \Doctrine\DBAL\Connection { } /** - * Initializes a new instance of the Connection class. - * - * @param array $params The connection parameters. - * @param \Doctrine\DBAL\Driver $driver - * @param \Doctrine\DBAL\Configuration $config - * @param \Doctrine\Common\EventManager $eventManager - * @throws \Exception - */ - public function __construct(array $params, Driver $driver, Configuration $config = null, - EventManager $eventManager = null) { - if (!isset($params['adapter'])) { - throw new \Exception('adapter not set'); - } - if (!isset($params['tablePrefix'])) { - throw new \Exception('tablePrefix not set'); - } - /** - * @psalm-suppress InternalMethod - */ - parent::__construct($params, $driver, $config, $eventManager); - $this->adapter = new $params['adapter']($this); - $this->tablePrefix = $params['tablePrefix']; - - $this->systemConfig = \OC::$server->getSystemConfig(); - $this->logger = \OC::$server->getLogger(); - } - - /** * Prepares an SQL statement. * * @param string $statement The SQL statement to prepare. - * @param int $limit - * @param int $offset + * @param int|null $limit + * @param int|null $offset * * @return Statement The prepared statement. * @throws Exception */ - public function prepare($statement, $limit = null, $offset = null): Statement { - if ($limit === -1) { + public function prepare($sql, $limit = null, $offset = null): Statement { + if ($limit === -1 || $limit === null) { $limit = null; + } else { + $limit = (int)$limit; + } + if ($offset !== null) { + $offset = (int)$offset; } if (!is_null($limit)) { $platform = $this->getDatabasePlatform(); - $statement = $platform->modifyLimitQuery($statement, $limit, $offset); + $sql = $platform->modifyLimitQuery($sql, $limit, $offset); } - $statement = $this->replaceTablePrefix($statement); - $statement = $this->adapter->fixupStatement($statement); + $statement = $this->finishQuery($sql); return parent::prepare($statement); } @@ -213,30 +367,78 @@ class Connection extends \Doctrine\DBAL\Connection { * If the query is parametrized, a prepared statement is used. * If an SQLLogger is configured, the execution is logged. * - * @param string $sql The SQL query to execute. - * @param array $params The parameters to bind to the query, if any. - * @param array $types The types the previous parameters are in. - * @param \Doctrine\DBAL\Cache\QueryCacheProfile|null $qcp The query cache profile, optional. + * @param string $sql The SQL query to execute. + * @param array $params The parameters to bind to the query, if any. + * @param array $types The types the previous parameters are in. + * @param \Doctrine\DBAL\Cache\QueryCacheProfile|null $qcp The query cache profile, optional. * * @return Result The executed statement. * * @throws \Doctrine\DBAL\Exception */ - public function executeQuery(string $sql, array $params = [], $types = [], QueryCacheProfile $qcp = null): Result { - $sql = $this->replaceTablePrefix($sql); - $sql = $this->adapter->fixupStatement($sql); + public function executeQuery(string $sql, array $params = [], $types = [], ?QueryCacheProfile $qcp = null): Result { + $tables = $this->getQueriedTables($sql); + $now = $this->clock->now()->getTimestamp(); + $dirtyTableWrites = []; + foreach ($tables as $table) { + $lastAccess = $this->tableDirtyWrites[$table] ?? 0; + // Only very recent writes are considered dirty + if ($lastAccess >= ($now - 3)) { + $dirtyTableWrites[] = $table; + } + } + if ($this->isTransactionActive()) { + // Transacted queries go to the primary. The consistency of the primary guarantees that we can not run + // into a dirty read. + } elseif (count($dirtyTableWrites) === 0) { + // No tables read that could have been written already in the same request and no transaction active + // so we can switch back to the replica for reading as long as no writes happen that switch back to the primary + // We cannot log here as this would log too early in the server boot process + $this->ensureConnectedToReplica(); + } else { + // Read to a table that has been written to previously + // While this might not necessarily mean that we did a read after write it is an indication for a code path to check + $this->logger->log( + (int)($this->systemConfig->getValue('loglevel_dirty_database_queries', null) ?? 0), + 'dirty table reads: ' . $sql, + [ + 'tables' => array_keys($this->tableDirtyWrites), + 'reads' => $tables, + 'exception' => new \Exception('dirty table reads: ' . $sql), + ], + ); + // To prevent a dirty read on a replica that is slightly out of sync, we + // switch back to the primary. This is detrimental for performance but + // safer for consistency. + $this->ensureConnectedToPrimary(); + } + + $sql = $this->finishQuery($sql); $this->queriesExecuted++; - return parent::executeQuery($sql, $params, $types, $qcp); + $this->logQueryToFile($sql, $params); + try { + return parent::executeQuery($sql, $params, $types, $qcp); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } + } + + /** + * Helper function to get the list of tables affected by a given query + * used to track dirty tables that received a write with the current request + */ + private function getQueriedTables(string $sql): array { + $re = '/(\*PREFIX\*\w+)/mi'; + preg_match_all($re, $sql, $matches); + return array_map([$this, 'replaceTablePrefix'], $matches[0] ?? []); } /** * @throws Exception */ public function executeUpdate(string $sql, array $params = [], array $types = []): int { - $sql = $this->replaceTablePrefix($sql); - $sql = $this->adapter->fixupStatement($sql); - $this->queriesExecuted++; - return parent::executeUpdate($sql, $params, $types); + return $this->executeStatement($sql, $params, $types); } /** @@ -245,19 +447,60 @@ class Connection extends \Doctrine\DBAL\Connection { * * This method supports PDO binding types as well as DBAL mapping types. * - * @param string $sql The SQL query. - * @param array $params The query parameters. - * @param array $types The parameter types. + * @param string $sql The SQL query. + * @param array $params The query parameters. + * @param array $types The parameter types. * * @return int The number of affected rows. * * @throws \Doctrine\DBAL\Exception */ public function executeStatement($sql, array $params = [], array $types = []): int { - $sql = $this->replaceTablePrefix($sql); - $sql = $this->adapter->fixupStatement($sql); + $tables = $this->getQueriedTables($sql); + foreach ($tables as $table) { + $this->tableDirtyWrites[$table] = $this->clock->now()->getTimestamp(); + } + $sql = $this->finishQuery($sql); $this->queriesExecuted++; - return parent::executeStatement($sql, $params, $types); + $this->logQueryToFile($sql, $params); + try { + return (int)parent::executeStatement($sql, $params, $types); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } + } + + protected function logQueryToFile(string $sql, array $params): void { + $logFile = $this->systemConfig->getValue('query_log_file'); + if ($logFile !== '' && is_writable(dirname($logFile)) && (!file_exists($logFile) || is_writable($logFile))) { + $prefix = ''; + if ($this->systemConfig->getValue('query_log_file_requestid') === 'yes') { + $prefix .= Server::get(IRequestId::class)->getId() . "\t"; + } + + $postfix = ''; + if ($this->systemConfig->getValue('query_log_file_parameters') === 'yes') { + $postfix .= '; ' . json_encode($params); + } + + if ($this->systemConfig->getValue('query_log_file_backtrace') === 'yes') { + $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + array_pop($trace); + $postfix .= '; ' . json_encode($trace); + } + + // FIXME: Improve to log the actual target db host + $isPrimary = $this->connections['primary'] === $this->_conn; + $prefix .= ' ' . ($isPrimary === true ? 'primary' : 'replica') . ' '; + $prefix .= ' ' . $this->getTransactionNestingLevel() . ' '; + + file_put_contents( + $this->systemConfig->getValue('query_log_file', ''), + $prefix . $sql . $postfix . "\n", + FILE_APPEND + ); + } } /** @@ -270,14 +513,14 @@ class Connection extends \Doctrine\DBAL\Connection { * * @param string $seqName Name of the sequence object from which the ID should be returned. * - * @return string the last inserted ID. + * @return int the last inserted ID. * @throws Exception */ - public function lastInsertId($seqName = null) { - if ($seqName) { - $seqName = $this->replaceTablePrefix($seqName); + public function lastInsertId($name = null): int { + if ($name) { + $name = $this->replaceTablePrefix($name); } - return $this->adapter->lastInsertId($seqName); + return $this->adapter->lastInsertId($name); } /** @@ -296,18 +539,28 @@ class Connection extends \Doctrine\DBAL\Connection { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws \Doctrine\DBAL\Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 */ - public function insertIfNotExist($table, $input, array $compare = null) { - return $this->adapter->insertIfNotExist($table, $input, $compare); + public function insertIfNotExist($table, $input, ?array $compare = null) { + try { + return $this->adapter->insertIfNotExist($table, $input, $compare); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } public function insertIgnoreConflict(string $table, array $values) : int { - return $this->adapter->insertIgnoreConflict($table, $values); + try { + return $this->adapter->insertIgnoreConflict($table, $values); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } private function getType($value) { @@ -328,10 +581,10 @@ class Connection extends \Doctrine\DBAL\Connection { * @param array $values (column name => value) * @param array $updatePreconditionValues ensure values match preconditions (column name => value) * @return int number of new rows - * @throws \Doctrine\DBAL\Exception + * @throws \OCP\DB\Exception * @throws PreConditionNotMetException */ - public function setValues($table, array $keys, array $values, array $updatePreconditionValues = []) { + public function setValues(string $table, array $keys, array $values, array $updatePreconditionValues = []): int { try { $insertQb = $this->getQueryBuilder(); $insertQb->insert($table) @@ -340,33 +593,39 @@ class Connection extends \Doctrine\DBAL\Connection { return $insertQb->createNamedParameter($value, $this->getType($value)); }, array_merge($keys, $values)) ); - return $insertQb->execute(); - } catch (NotNullConstraintViolationException $e) { - throw $e; - } catch (ConstraintViolationException $e) { + return $insertQb->executeStatement(); + } catch (\OCP\DB\Exception $e) { + if (!in_array($e->getReason(), [ + \OCP\DB\Exception::REASON_CONSTRAINT_VIOLATION, + \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION, + ]) + ) { + throw $e; + } + // value already exists, try update $updateQb = $this->getQueryBuilder(); $updateQb->update($table); foreach ($values as $name => $value) { $updateQb->set($name, $updateQb->createNamedParameter($value, $this->getType($value))); } - $where = $updateQb->expr()->andX(); + $where = []; $whereValues = array_merge($keys, $updatePreconditionValues); foreach ($whereValues as $name => $value) { if ($value === '') { - $where->add($updateQb->expr()->emptyString( + $where[] = $updateQb->expr()->emptyString( $name - )); + ); } else { - $where->add($updateQb->expr()->eq( + $where[] = $updateQb->expr()->eq( $name, $updateQb->createNamedParameter($value, $this->getType($value)), $this->getType($value) - )); + ); } } - $updateQb->where($where); - $affected = $updateQb->execute(); + $updateQb->where($updateQb->expr()->andX(...$where)); + $affected = $updateQb->executeStatement(); if ($affected === 0 && !empty($updatePreconditionValues)) { throw new PreConditionNotMetException(); @@ -415,9 +674,9 @@ class Connection extends \Doctrine\DBAL\Connection { $msg = $this->errorCode() . ': '; $errorInfo = $this->errorInfo(); if (!empty($errorInfo)) { - $msg .= 'SQLSTATE = '.$errorInfo[0] . ', '; - $msg .= 'Driver Code = '.$errorInfo[1] . ', '; - $msg .= 'Driver Message = '.$errorInfo[2]; + $msg .= 'SQLSTATE = ' . $errorInfo[0] . ', '; + $msg .= 'Driver Code = ' . $errorInfo[1] . ', '; + $msg .= 'Driver Message = ' . $errorInfo[2]; } return $msg; } @@ -439,13 +698,26 @@ class Connection extends \Doctrine\DBAL\Connection { */ public function dropTable($table) { $table = $this->tablePrefix . trim($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); if ($schema->tablesExist([$table])) { $schema->dropTable($table); } } /** + * Truncate a table data if it exists + * + * @param string $table table name without the prefix + * @param bool $cascade whether to truncate cascading + * + * @throws Exception + */ + public function truncateTable(string $table, bool $cascade) { + $this->executeStatement($this->getDatabasePlatform() + ->getTruncateTableSQL($this->tablePrefix . trim($table), $cascade)); + } + + /** * Check if a table exists * * @param string $table table name without the prefix @@ -455,10 +727,20 @@ class Connection extends \Doctrine\DBAL\Connection { */ public function tableExists($table) { $table = $this->tablePrefix . trim($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); return $schema->tablesExist([$table]); } + protected function finishQuery(string $statement): string { + $statement = $this->replaceTablePrefix($statement); + $statement = $this->adapter->fixupStatement($statement); + if ($this->logRequestId) { + return $statement . ' /* reqid: ' . $this->requestId . ' */'; + } else { + return $statement; + } + } + // internal use /** * @param string $statement @@ -509,8 +791,7 @@ class Connection extends \Doctrine\DBAL\Connection { * @throws Exception */ public function createSchema() { - $schemaManager = new MDB2SchemaManager($this); - $migrator = $schemaManager->getMigrator(); + $migrator = $this->getMigrator(); return $migrator->createSchema(); } @@ -518,12 +799,172 @@ class Connection extends \Doctrine\DBAL\Connection { * Migrate the database to the given schema * * @param Schema $toSchema + * @param bool $dryRun If true, will return the sql queries instead of running them. * * @throws Exception + * + * @return string|null Returns a string only if $dryRun is true. + */ + public function migrateToSchema(Schema $toSchema, bool $dryRun = false) { + $migrator = $this->getMigrator(); + + if ($dryRun) { + return $migrator->generateChangeScript($toSchema); + } else { + $migrator->migrate($toSchema); + foreach ($this->getShardConnections() as $shardConnection) { + $shardConnection->migrateToSchema($toSchema); + } + } + } + + private function getMigrator() { + // TODO properly inject those dependencies + $random = \OC::$server->get(ISecureRandom::class); + $platform = $this->getDatabasePlatform(); + $config = \OC::$server->getConfig(); + $dispatcher = Server::get(\OCP\EventDispatcher\IEventDispatcher::class); + if ($platform instanceof SqlitePlatform) { + return new SQLiteMigrator($this, $config, $dispatcher); + } elseif ($platform instanceof OraclePlatform) { + return new OracleMigrator($this, $config, $dispatcher); + } else { + return new Migrator($this, $config, $dispatcher); + } + } + + public function beginTransaction() { + if (!$this->inTransaction()) { + $this->transactionBacktrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + $this->transactionActiveSince = microtime(true); + } + return parent::beginTransaction(); + } + + public function commit() { + $result = parent::commit(); + if ($this->getTransactionNestingLevel() === 0) { + $timeTook = microtime(true) - $this->transactionActiveSince; + $this->transactionBacktrace = null; + $this->transactionActiveSince = null; + if ($timeTook > 1) { + $logLevel = match (true) { + $timeTook > 20 * 60 => ILogger::ERROR, + $timeTook > 5 * 60 => ILogger::WARN, + $timeTook > 10 => ILogger::INFO, + default => ILogger::DEBUG, + }; + $this->logger->log( + $logLevel, + 'Transaction took ' . $timeTook . 's', + [ + 'exception' => new \Exception('Transaction took ' . $timeTook . 's'), + 'timeSpent' => $timeTook, + ] + ); + } + } + return $result; + } + + public function rollBack() { + $result = parent::rollBack(); + if ($this->getTransactionNestingLevel() === 0) { + $timeTook = microtime(true) - $this->transactionActiveSince; + $this->transactionBacktrace = null; + $this->transactionActiveSince = null; + if ($timeTook > 1) { + $logLevel = match (true) { + $timeTook > 20 * 60 => ILogger::ERROR, + $timeTook > 5 * 60 => ILogger::WARN, + $timeTook > 10 => ILogger::INFO, + default => ILogger::DEBUG, + }; + $this->logger->log( + $logLevel, + 'Transaction rollback took longer than 1s: ' . $timeTook, + [ + 'exception' => new \Exception('Long running transaction rollback'), + 'timeSpent' => $timeTook, + ] + ); + } + } + return $result; + } + + private function reconnectIfNeeded(): void { + if ( + !isset($this->lastConnectionCheck[$this->getConnectionName()]) + || time() <= $this->lastConnectionCheck[$this->getConnectionName()] + 30 + || $this->isTransactionActive() + ) { + return; + } + + try { + $this->_conn->query($this->getDriver()->getDatabasePlatform()->getDummySelectSQL()); + $this->lastConnectionCheck[$this->getConnectionName()] = time(); + } catch (ConnectionLost|\Exception $e) { + $this->logger->warning('Exception during connectivity check, closing and reconnecting', ['exception' => $e]); + $this->close(); + } + } + + private function getConnectionName(): string { + return $this->isConnectedToPrimary() ? 'primary' : 'replica'; + } + + /** + * @return IDBConnection::PLATFORM_MYSQL|IDBConnection::PLATFORM_ORACLE|IDBConnection::PLATFORM_POSTGRES|IDBConnection::PLATFORM_SQLITE|IDBConnection::PLATFORM_MARIADB + */ + public function getDatabaseProvider(bool $strict = false): string { + $platform = $this->getDatabasePlatform(); + if ($strict && $platform instanceof MariaDBPlatform) { + return IDBConnection::PLATFORM_MARIADB; + } elseif ($platform instanceof MySQLPlatform) { + return IDBConnection::PLATFORM_MYSQL; + } elseif ($platform instanceof OraclePlatform) { + return IDBConnection::PLATFORM_ORACLE; + } elseif ($platform instanceof PostgreSQLPlatform) { + return IDBConnection::PLATFORM_POSTGRES; + } elseif ($platform instanceof SqlitePlatform) { + return IDBConnection::PLATFORM_SQLITE; + } else { + throw new \Exception('Database ' . $platform::class . ' not supported'); + } + } + + /** + * @internal Should only be used inside the QueryBuilder, ExpressionBuilder and FunctionBuilder + * All apps and API code should not need this and instead use provided functionality from the above. */ - public function migrateToSchema(Schema $toSchema) { - $schemaManager = new MDB2SchemaManager($this); - $migrator = $schemaManager->getMigrator(); - $migrator->migrate($toSchema); + public function getServerVersion(): string { + /** @var ServerInfoAwareConnection $this->_conn */ + return $this->_conn->getServerVersion(); + } + + /** + * Log a database exception if enabled + * + * @param \Exception $exception + * @return void + */ + public function logDatabaseException(\Exception $exception): void { + if ($this->logDbException) { + if ($exception instanceof Exception\UniqueConstraintViolationException) { + $this->logger->info($exception->getMessage(), ['exception' => $exception, 'transaction' => $this->transactionBacktrace]); + } else { + $this->logger->error($exception->getMessage(), ['exception' => $exception, 'transaction' => $this->transactionBacktrace]); + } + } + } + + public function getShardDefinition(string $name): ?ShardDefinition { + return $this->shards[$name] ?? null; + } + + public function getCrossShardMoveHelper(): CrossShardMoveHelper { + return new CrossShardMoveHelper($this->shardConnectionManager); } } diff --git a/lib/private/DB/ConnectionAdapter.php b/lib/private/DB/ConnectionAdapter.php index 62c013e4dcf..d9ccb3c54f2 100644 --- a/lib/private/DB/ConnectionAdapter.php +++ b/lib/private/DB/ConnectionAdapter.php @@ -2,33 +2,18 @@ declare(strict_types=1); -/* - * @copyright 2020 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @author 2020 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. +/** + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\Schema; use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\CrossShardMoveHelper; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OCP\DB\IPreparedStatement; use OCP\DB\IResult; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -38,7 +23,6 @@ use OCP\IDBConnection; * Adapts the public API to our internal DBAL connection wrapper */ class ConnectionAdapter implements IDBConnection { - /** @var Connection */ private $inner; @@ -66,7 +50,7 @@ class ConnectionAdapter implements IDBConnection { $this->inner->executeQuery($sql, $params, $types) ); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -74,7 +58,7 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeUpdate($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -82,19 +66,19 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeStatement($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } public function lastInsertId(string $table): int { try { - return (int)$this->inner->lastInsertId($table); + return $this->inner->lastInsertId($table); } catch (Exception $e) { throw DbalException::wrap($e); } } - public function insertIfNotExist(string $table, array $input, array $compare = null) { + public function insertIfNotExist(string $table, array $input, ?array $compare = null) { try { return $this->inner->insertIfNotExist($table, $input, $compare); } catch (Exception $e) { @@ -205,6 +189,14 @@ class ConnectionAdapter implements IDBConnection { } } + public function truncateTable(string $table, bool $cascade): void { + try { + $this->inner->truncateTable($table, $cascade); + } catch (Exception $e) { + throw DbalException::wrap($e); + } + } + public function tableExists(string $table): bool { try { return $this->inner->tableExists($table); @@ -243,4 +235,31 @@ class ConnectionAdapter implements IDBConnection { public function getInner(): Connection { return $this->inner; } + + /** + * @return self::PLATFORM_MYSQL|self::PLATFORM_ORACLE|self::PLATFORM_POSTGRES|self::PLATFORM_SQLITE|self::PLATFORM_MARIADB + */ + public function getDatabaseProvider(bool $strict = false): string { + return $this->inner->getDatabaseProvider($strict); + } + + /** + * @internal Should only be used inside the QueryBuilder, ExpressionBuilder and FunctionBuilder + * All apps and API code should not need this and instead use provided functionality from the above. + */ + public function getServerVersion(): string { + return $this->inner->getServerVersion(); + } + + public function logDatabaseException(\Exception $exception) { + $this->inner->logDatabaseException($exception); + } + + public function getShardDefinition(string $name): ?ShardDefinition { + return $this->inner->getShardDefinition($name); + } + + public function getCrossShardMoveHelper(): CrossShardMoveHelper { + return $this->inner->getCrossShardMoveHelper(); + } } diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index 441e4567dbd..d9b80b81992 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -1,40 +1,21 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\Common\EventManager; use Doctrine\DBAL\Configuration; use Doctrine\DBAL\DriverManager; use Doctrine\DBAL\Event\Listeners\OracleSessionInit; -use Doctrine\DBAL\Event\Listeners\SQLSessionInit; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; use OC\SystemConfig; +use OCP\ICacheFactory; +use OCP\Server; /** * Takes care of creating and configuring Doctrine connections. @@ -77,19 +58,22 @@ class ConnectionFactory { ], ]; - /** @var SystemConfig */ - private $config; + private ShardConnectionManager $shardConnectionManager; + private ICacheFactory $cacheFactory; - /** - * ConnectionFactory constructor. - * - * @param SystemConfig $systemConfig - */ - public function __construct(SystemConfig $systemConfig) { - $this->config = $systemConfig; + public function __construct( + private SystemConfig $config, + ?ICacheFactory $cacheFactory = null, + ) { if ($this->config->getValue('mysql.utf8mb4', false)) { $this->defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; } + $collationOverride = $this->config->getValue('mysql.collation', null); + if ($collationOverride) { + $this->defaultConnectionParams['mysql']['collation'] = $collationOverride; + } + $this->shardConnectionManager = new ShardConnectionManager($this->config, $this); + $this->cacheFactory = $cacheFactory ?? Server::get(ICacheFactory::class); } /** @@ -120,43 +104,37 @@ class ConnectionFactory { * @param array $additionalConnectionParams Additional connection parameters * @return \OC\DB\Connection */ - public function getConnection($type, $additionalConnectionParams) { + public function getConnection(string $type, array $additionalConnectionParams): Connection { $normalizedType = $this->normalizeType($type); $eventManager = new EventManager(); $eventManager->addEventSubscriber(new SetTransactionIsolationLevel()); + $connectionParams = $this->createConnectionParams('', $additionalConnectionParams, $type); switch ($normalizedType) { - case 'mysql': - $eventManager->addEventSubscriber( - new SQLSessionInit("SET SESSION AUTOCOMMIT=1")); + case 'pgsql': + // pg_connect used by Doctrine DBAL does not support URI notation (enclosed in brackets) + $matches = []; + if (preg_match('/^\[([^\]]+)\]$/', $connectionParams['host'], $matches)) { + // Host variable carries a port or socket. + $connectionParams['host'] = $matches[1]; + } break; + case 'oci': $eventManager->addEventSubscriber(new OracleSessionInit); - // the driverOptions are unused in dbal and need to be mapped to the parameters - if (isset($additionalConnectionParams['driverOptions'])) { - $additionalConnectionParams = array_merge($additionalConnectionParams, $additionalConnectionParams['driverOptions']); - } - $host = $additionalConnectionParams['host']; - $port = isset($additionalConnectionParams['port']) ? $additionalConnectionParams['port'] : null; - $dbName = $additionalConnectionParams['dbname']; - - // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string - if ($host === '') { - $additionalConnectionParams['dbname'] = $dbName; // use dbname as easy connect name - } else { - $additionalConnectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : "") . '/' . $dbName; - } - unset($additionalConnectionParams['host']); + $connectionParams = $this->forceConnectionStringOracle($connectionParams); + $connectionParams['primary'] = $this->forceConnectionStringOracle($connectionParams['primary']); + $connectionParams['replica'] = array_map([$this, 'forceConnectionStringOracle'], $connectionParams['replica']); break; case 'sqlite3': - $journalMode = $additionalConnectionParams['sqlite.journal_mode']; - $additionalConnectionParams['platform'] = new OCSqlitePlatform(); + $journalMode = $connectionParams['sqlite.journal_mode']; + $connectionParams['platform'] = new OCSqlitePlatform(); $eventManager->addEventSubscriber(new SQLiteSessionInit(true, $journalMode)); break; } /** @var Connection $connection */ $connection = DriverManager::getConnection( - array_merge($this->getDefaultConnectionParams($type), $additionalConnectionParams), + $connectionParams, new Configuration(), $eventManager ); @@ -185,23 +163,22 @@ class ConnectionFactory { /** * Create the connection parameters for the config - * - * @return array */ - public function createConnectionParams() { - $type = $this->config->getValue('dbtype', 'sqlite'); + public function createConnectionParams(string $configPrefix = '', array $additionalConnectionParams = [], ?string $type = null) { + // use provided type or if null use type from config + $type = $type ?? $this->config->getValue('dbtype', 'sqlite'); - $connectionParams = [ - 'user' => $this->config->getValue('dbuser', ''), - 'password' => $this->config->getValue('dbpassword', ''), - ]; - $name = $this->config->getValue('dbname', self::DEFAULT_DBNAME); + $connectionParams = array_merge($this->getDefaultConnectionParams($type), [ + 'user' => $this->config->getValue($configPrefix . 'dbuser', $this->config->getValue('dbuser', '')), + 'password' => $this->config->getValue($configPrefix . 'dbpassword', $this->config->getValue('dbpassword', '')), + ]); + $name = $this->config->getValue($configPrefix . 'dbname', $this->config->getValue('dbname', self::DEFAULT_DBNAME)); if ($this->normalizeType($type) === 'sqlite3') { - $dataDir = $this->config->getValue("datadirectory", \OC::$SERVERROOT . '/data'); + $dataDir = $this->config->getValue('datadirectory', \OC::$SERVERROOT . '/data'); $connectionParams['path'] = $dataDir . '/' . $name . '.db'; } else { - $host = $this->config->getValue('dbhost', ''); + $host = $this->config->getValue($configPrefix . 'dbhost', $this->config->getValue('dbhost', '')); $connectionParams = array_merge($connectionParams, $this->splitHostFromPortAndSocket($host)); $connectionParams['dbname'] = $name; } @@ -210,7 +187,7 @@ class ConnectionFactory { $connectionParams['sqlite.journal_mode'] = $this->config->getValue('sqlite.journal_mode', 'WAL'); //additional driver options, eg. for mysql ssl - $driverOptions = $this->config->getValue('dbdriveroptions', null); + $driverOptions = $this->config->getValue($configPrefix . 'dbdriveroptions', $this->config->getValue('dbdriveroptions', null)); if ($driverOptions) { $connectionParams['driverOptions'] = $driverOptions; } @@ -221,16 +198,37 @@ class ConnectionFactory { 'tablePrefix' => $connectionParams['tablePrefix'] ]; - if ($this->config->getValue('mysql.utf8mb4', false)) { + if ($type === 'mysql' && $this->config->getValue('mysql.utf8mb4', false)) { $connectionParams['defaultTableOptions'] = [ 'collate' => 'utf8mb4_bin', 'charset' => 'utf8mb4', - 'row_format' => 'compressed', 'tablePrefix' => $connectionParams['tablePrefix'] ]; } - return $connectionParams; + if ($this->config->getValue('dbpersistent', false)) { + $connectionParams['persistent'] = true; + } + + $connectionParams['sharding'] = $this->config->getValue('dbsharding', []); + if (!empty($connectionParams['sharding'])) { + $connectionParams['shard_connection_manager'] = $this->shardConnectionManager; + $connectionParams['auto_increment_handler'] = new AutoIncrementHandler( + $this->cacheFactory, + $this->shardConnectionManager, + ); + } else { + // just in case only the presence could lead to funny behaviour + unset($connectionParams['sharding']); + } + + $connectionParams = array_merge($connectionParams, $additionalConnectionParams); + + $replica = $this->config->getValue($configPrefix . 'dbreplica', $this->config->getValue('dbreplica', [])) ?: [$connectionParams]; + return array_merge($connectionParams, [ + 'primary' => $connectionParams, + 'replica' => $replica, + ]); } /** @@ -247,7 +245,7 @@ class ConnectionFactory { // Host variable carries a port or socket. $params['host'] = $matches[1]; if (is_numeric($matches[2])) { - $params['port'] = (int) $matches[2]; + $params['port'] = (int)$matches[2]; } else { $params['unix_socket'] = $matches[2]; } @@ -255,4 +253,24 @@ class ConnectionFactory { return $params; } + + protected function forceConnectionStringOracle(array $connectionParams): array { + // the driverOptions are unused in dbal and need to be mapped to the parameters + if (isset($connectionParams['driverOptions'])) { + $connectionParams = array_merge($connectionParams, $connectionParams['driverOptions']); + } + $host = $connectionParams['host']; + $port = $connectionParams['port'] ?? null; + $dbName = $connectionParams['dbname']; + + // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string + if ($host === '') { + $connectionParams['dbname'] = $dbName; // use dbname as easy connect name + } else { + $connectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : '') . '/' . $dbName; + } + unset($connectionParams['host']); + + return $connectionParams; + } } diff --git a/lib/private/DB/DbDataCollector.php b/lib/private/DB/DbDataCollector.php new file mode 100644 index 00000000000..e3c7cd35855 --- /dev/null +++ b/lib/private/DB/DbDataCollector.php @@ -0,0 +1,136 @@ +<?php + +declare(strict_types = 1); +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB; + +use Doctrine\DBAL\Types\ConversionException; +use Doctrine\DBAL\Types\Type; +use OC\AppFramework\Http\Request; +use OCP\AppFramework\Http\Response; + +class DbDataCollector extends \OCP\DataCollector\AbstractDataCollector { + protected ?BacktraceDebugStack $debugStack = null; + private Connection $connection; + + /** + * DbDataCollector constructor. + */ + public function __construct(Connection $connection) { + $this->connection = $connection; + } + + public function setDebugStack(BacktraceDebugStack $debugStack, $name = 'default'): void { + $this->debugStack = $debugStack; + } + + /** + * @inheritDoc + */ + public function collect(Request $request, Response $response, ?\Throwable $exception = null): void { + $queries = $this->sanitizeQueries($this->debugStack->queries); + + $this->data = [ + 'queries' => $queries, + ]; + } + + public function getName(): string { + return 'db'; + } + + public function getQueries(): array { + return $this->data['queries']; + } + + private function sanitizeQueries(array $queries): array { + foreach ($queries as $i => $query) { + $queries[$i] = $this->sanitizeQuery($query); + } + + return $queries; + } + + private function sanitizeQuery(array $query): array { + $query['explainable'] = true; + $query['runnable'] = true; + if ($query['params'] === null) { + $query['params'] = []; + } + if (!\is_array($query['params'])) { + $query['params'] = [$query['params']]; + } + if (!\is_array($query['types'])) { + $query['types'] = []; + } + foreach ($query['params'] as $j => $param) { + $e = null; + if (isset($query['types'][$j])) { + // Transform the param according to the type + $type = $query['types'][$j]; + if (\is_string($type)) { + $type = Type::getType($type); + } + if ($type instanceof Type) { + $query['types'][$j] = $type->getBindingType(); + try { + $param = $type->convertToDatabaseValue($param, $this->connection->getDatabasePlatform()); + } catch (\TypeError $e) { + } catch (ConversionException $e) { + } + } + } + + [$query['params'][$j], $explainable, $runnable] = $this->sanitizeParam($param, $e); + if (!$explainable) { + $query['explainable'] = false; + } + + if (!$runnable) { + $query['runnable'] = false; + } + } + + return $query; + } + + /** + * Sanitizes a param. + * + * The return value is an array with the sanitized value and a boolean + * indicating if the original value was kept (allowing to use the sanitized + * value to explain the query). + */ + private function sanitizeParam($var, ?\Throwable $error): array { + if (\is_object($var)) { + return [$o = new ObjectParameter($var, $error), false, $o->isStringable() && !$error]; + } + + if ($error) { + return ['⚠ ' . $error->getMessage(), false, false]; + } + + if (\is_array($var)) { + $a = []; + $explainable = $runnable = true; + foreach ($var as $k => $v) { + [$value, $e, $r] = $this->sanitizeParam($v, null); + $explainable = $explainable && $e; + $runnable = $runnable && $r; + $a[$k] = $value; + } + + return [$a, $explainable, $runnable]; + } + + if (\is_resource($var)) { + return [sprintf('/* Resource(%s) */', get_resource_type($var)), false, false]; + } + + return [$var, true, true]; + } +} diff --git a/lib/private/DB/Exceptions/DbalException.php b/lib/private/DB/Exceptions/DbalException.php index bdb4d3147ae..2ce6ddf80a6 100644 --- a/lib/private/DB/Exceptions/DbalException.php +++ b/lib/private/DB/Exceptions/DbalException.php @@ -2,27 +2,10 @@ declare(strict_types=1); -/* - * @copyright 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @author 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\Exceptions; use Doctrine\DBAL\ConnectionException; @@ -34,8 +17,10 @@ use Doctrine\DBAL\Exception\DriverException; use Doctrine\DBAL\Exception\ForeignKeyConstraintViolationException; use Doctrine\DBAL\Exception\InvalidArgumentException; use Doctrine\DBAL\Exception\InvalidFieldNameException; +use Doctrine\DBAL\Exception\LockWaitTimeoutException; use Doctrine\DBAL\Exception\NonUniqueFieldNameException; use Doctrine\DBAL\Exception\NotNullConstraintViolationException; +use Doctrine\DBAL\Exception\RetryableException; use Doctrine\DBAL\Exception\ServerException; use Doctrine\DBAL\Exception\SyntaxErrorException; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; @@ -48,32 +33,38 @@ use OCP\DB\Exception; * @psalm-immutable */ class DbalException extends Exception { - /** @var \Doctrine\DBAL\Exception */ private $original; + public readonly ?string $query; /** * @param \Doctrine\DBAL\Exception $original * @param int $code * @param string $message */ - private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message) { + private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message, ?string $query = null) { parent::__construct( $message, $code, $original ); $this->original = $original; + $this->query = $query; } - public static function wrap(\Doctrine\DBAL\Exception $original, string $message = ''): self { + public static function wrap(\Doctrine\DBAL\Exception $original, string $message = '', ?string $query = null): self { return new self( $original, is_int($original->getCode()) ? $original->getCode() : 0, - empty($message) ? $original->getMessage() : $message + empty($message) ? $original->getMessage() : $message, + $query, ); } + public function isRetryable(): bool { + return $this->original instanceof RetryableException; + } + public function getReason(): ?int { /** * Constraint errors @@ -95,6 +86,9 @@ class DbalException extends Exception { /** * Other server errors */ + if ($this->original instanceof LockWaitTimeoutException) { + return parent::REASON_LOCK_WAIT_TIMEOUT; + } if ($this->original instanceof DatabaseObjectExistsException) { return parent::REASON_DATABASE_OBJECT_EXISTS; } diff --git a/lib/private/DB/MDB2SchemaManager.php b/lib/private/DB/MDB2SchemaManager.php deleted file mode 100644 index 18549eb1fa2..00000000000 --- a/lib/private/DB/MDB2SchemaManager.php +++ /dev/null @@ -1,161 +0,0 @@ -<?php -/** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * - */ - -namespace OC\DB; - -use Doctrine\DBAL\Platforms\MySQLPlatform; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQL94Platform; -use Doctrine\DBAL\Platforms\SqlitePlatform; -use Doctrine\DBAL\Schema\Schema; - -class MDB2SchemaManager { - /** @var Connection $conn */ - protected $conn; - - /** - * @param Connection $conn - */ - public function __construct($conn) { - $this->conn = $conn; - } - - /** - * Creates tables from XML file - * @param string $file file to read structure from - * @return bool - * - * TODO: write more documentation - */ - public function createDbFromStructure($file) { - $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); - $toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); - $toSchema = $schemaReader->loadSchemaFromFile($file, $toSchema); - return $this->executeSchemaChange($toSchema); - } - - /** - * @return \OC\DB\Migrator - */ - public function getMigrator() { - $random = \OC::$server->getSecureRandom(); - $platform = $this->conn->getDatabasePlatform(); - $config = \OC::$server->getConfig(); - $dispatcher = \OC::$server->getEventDispatcher(); - if ($platform instanceof SqlitePlatform) { - return new SQLiteMigrator($this->conn, $random, $config, $dispatcher); - } elseif ($platform instanceof OraclePlatform) { - return new OracleMigrator($this->conn, $random, $config, $dispatcher); - } elseif ($platform instanceof MySQLPlatform) { - return new MySQLMigrator($this->conn, $random, $config, $dispatcher); - } elseif ($platform instanceof PostgreSQL94Platform) { - return new PostgreSqlMigrator($this->conn, $random, $config, $dispatcher); - } else { - return new Migrator($this->conn, $random, $config, $dispatcher); - } - } - - /** - * Reads database schema from file - * - * @param string $file file to read from - * @return \Doctrine\DBAL\Schema\Schema - */ - private function readSchemaFromFile($file) { - $platform = $this->conn->getDatabasePlatform(); - $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform); - $toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); - return $schemaReader->loadSchemaFromFile($file, $toSchema); - } - - /** - * update the database scheme - * @param string $file file to read structure from - * @param bool $generateSql only return the sql needed for the upgrade - * @return string|boolean - */ - public function updateDbFromStructure($file, $generateSql = false) { - $toSchema = $this->readSchemaFromFile($file); - $migrator = $this->getMigrator(); - - if ($generateSql) { - return $migrator->generateChangeScript($toSchema); - } else { - $migrator->migrate($toSchema); - return true; - } - } - - /** - * @param \Doctrine\DBAL\Schema\Schema $schema - * @return string - */ - public function generateChangeScript($schema) { - $migrator = $this->getMigrator(); - return $migrator->generateChangeScript($schema); - } - - /** - * remove all tables defined in a database structure xml file - * - * @param string $file the xml file describing the tables - */ - public function removeDBStructure($file) { - $schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform()); - $toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig()); - $fromSchema = $schemaReader->loadSchemaFromFile($file, $toSchema); - $toSchema = clone $fromSchema; - foreach ($toSchema->getTables() as $table) { - $toSchema->dropTable($table->getName()); - } - $comparator = new \Doctrine\DBAL\Schema\Comparator(); - $schemaDiff = $comparator->compare($fromSchema, $toSchema); - $this->executeSchemaChange($schemaDiff); - } - - /** - * @param \Doctrine\DBAL\Schema\Schema|\Doctrine\DBAL\Schema\SchemaDiff $schema - * @return bool - */ - private function executeSchemaChange($schema) { - $this->conn->beginTransaction(); - foreach ($schema->toSql($this->conn->getDatabasePlatform()) as $sql) { - $this->conn->query($sql); - } - $this->conn->commit(); - - if ($this->conn->getDatabasePlatform() instanceof SqlitePlatform) { - $this->conn->close(); - $this->conn->connect(); - } - return true; - } -} diff --git a/lib/private/DB/MDB2SchemaReader.php b/lib/private/DB/MDB2SchemaReader.php deleted file mode 100644 index 687438495b1..00000000000 --- a/lib/private/DB/MDB2SchemaReader.php +++ /dev/null @@ -1,355 +0,0 @@ -<?php -/** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Oliver Gasser <oliver.gasser@gmail.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Robin McCorkell <robin@mccorkell.me.uk> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * - */ - -namespace OC\DB; - -use Doctrine\DBAL\Platforms\AbstractPlatform; -use Doctrine\DBAL\Schema\Schema; -use OCP\IConfig; - -class MDB2SchemaReader { - - /** - * @var string $DBTABLEPREFIX - */ - protected $DBTABLEPREFIX; - - /** - * @var \Doctrine\DBAL\Platforms\AbstractPlatform $platform - */ - protected $platform; - - /** @var IConfig */ - protected $config; - - /** - * @param \OCP\IConfig $config - * @param \Doctrine\DBAL\Platforms\AbstractPlatform $platform - */ - public function __construct(IConfig $config, AbstractPlatform $platform) { - $this->platform = $platform; - $this->config = $config; - $this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_'); - } - - /** - * @param string $file - * @param Schema $schema - * @return Schema - * @throws \DomainException - */ - public function loadSchemaFromFile($file, Schema $schema) { - $loadEntities = libxml_disable_entity_loader(false); - $xml = simplexml_load_file($file); - libxml_disable_entity_loader($loadEntities); - foreach ($xml->children() as $child) { - /** - * @var \SimpleXMLElement $child - */ - switch ($child->getName()) { - case 'name': - case 'create': - case 'overwrite': - case 'charset': - break; - case 'table': - $this->loadTable($schema, $child); - break; - default: - throw new \DomainException('Unknown element: ' . $child->getName()); - - } - } - return $schema; - } - - /** - * @param \Doctrine\DBAL\Schema\Schema $schema - * @param \SimpleXMLElement $xml - * @throws \DomainException - */ - private function loadTable($schema, $xml) { - $table = null; - foreach ($xml->children() as $child) { - /** - * @var \SimpleXMLElement $child - */ - switch ($child->getName()) { - case 'name': - $name = (string)$child; - $name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name); - $name = $this->platform->quoteIdentifier($name); - $table = $schema->createTable($name); - break; - case 'create': - case 'overwrite': - case 'charset': - break; - case 'declaration': - if (is_null($table)) { - throw new \DomainException('Table declaration before table name'); - } - $this->loadDeclaration($table, $child); - break; - default: - throw new \DomainException('Unknown element: ' . $child->getName()); - - } - } - } - - /** - * @param \Doctrine\DBAL\Schema\Table $table - * @param \SimpleXMLElement $xml - * @throws \DomainException - */ - private function loadDeclaration($table, $xml) { - foreach ($xml->children() as $child) { - /** - * @var \SimpleXMLElement $child - */ - switch ($child->getName()) { - case 'field': - $this->loadField($table, $child); - break; - case 'index': - $this->loadIndex($table, $child); - break; - default: - throw new \DomainException('Unknown element: ' . $child->getName()); - - } - } - } - - /** - * @param \Doctrine\DBAL\Schema\Table $table - * @param \SimpleXMLElement $xml - * @throws \DomainException - */ - private function loadField($table, $xml) { - $options = [ 'notnull' => false ]; - foreach ($xml->children() as $child) { - /** - * @var \SimpleXMLElement $child - */ - switch ($child->getName()) { - case 'name': - $name = (string)$child; - $name = $this->platform->quoteIdentifier($name); - break; - case 'type': - $type = (string)$child; - switch ($type) { - case 'text': - $type = 'string'; - break; - case 'clob': - $type = 'text'; - break; - case 'timestamp': - $type = 'datetime'; - break; - case 'numeric': - $type = 'decimal'; - break; - } - break; - case 'length': - $length = (string)$child; - $options['length'] = $length; - break; - case 'unsigned': - $unsigned = $this->asBool($child); - $options['unsigned'] = $unsigned; - break; - case 'notnull': - $notnull = $this->asBool($child); - $options['notnull'] = $notnull; - break; - case 'autoincrement': - $autoincrement = $this->asBool($child); - $options['autoincrement'] = $autoincrement; - break; - case 'default': - $default = (string)$child; - $options['default'] = $default; - break; - case 'comments': - $comment = (string)$child; - $options['comment'] = $comment; - break; - case 'primary': - $primary = $this->asBool($child); - $options['primary'] = $primary; - break; - case 'precision': - $precision = (string)$child; - $options['precision'] = $precision; - break; - case 'scale': - $scale = (string)$child; - $options['scale'] = $scale; - break; - default: - throw new \DomainException('Unknown element: ' . $child->getName()); - - } - } - if (isset($name) && isset($type)) { - if (isset($options['default']) && empty($options['default'])) { - if (empty($options['notnull']) || !$options['notnull']) { - unset($options['default']); - $options['notnull'] = false; - } else { - $options['default'] = ''; - } - if ($type == 'integer' || $type == 'decimal') { - $options['default'] = 0; - } elseif ($type == 'boolean') { - $options['default'] = false; - } - if (!empty($options['autoincrement']) && $options['autoincrement']) { - unset($options['default']); - } - } - if ($type === 'integer' && isset($options['default'])) { - $options['default'] = (int)$options['default']; - } - if ($type === 'integer' && isset($options['length'])) { - $length = $options['length']; - if ($length < 4) { - $type = 'smallint'; - } elseif ($length > 4) { - $type = 'bigint'; - } - } - if ($type === 'boolean' && isset($options['default'])) { - $options['default'] = $this->asBool($options['default']); - } - if (!empty($options['autoincrement']) - && !empty($options['notnull']) - ) { - $options['primary'] = true; - } - - # not used anymore in the options argument - # see https://github.com/doctrine/dbal/commit/138eb85234a1faeaa2e6a32cd7bcc66bb51c64e8#diff-300f55366adb50a32a40882ebdc95c163b141f64cba5f45f20bda04a907b3eb3L82 - # therefore it's read before and then unset right before the addColumn call - $setPrimaryKey = false; - if (!empty($options['primary']) && $options['primary']) { - $setPrimaryKey = true; - } - unset($options['primary']); - $table->addColumn($name, $type, $options); - if ($setPrimaryKey) { - $table->setPrimaryKey([$name]); - } - } - } - - /** - * @param \Doctrine\DBAL\Schema\Table $table - * @param \SimpleXMLElement $xml - * @throws \DomainException - */ - private function loadIndex($table, $xml) { - $name = null; - $fields = []; - foreach ($xml->children() as $child) { - /** - * @var \SimpleXMLElement $child - */ - switch ($child->getName()) { - case 'name': - $name = (string)$child; - break; - case 'primary': - $primary = $this->asBool($child); - break; - case 'unique': - $unique = $this->asBool($child); - break; - case 'field': - foreach ($child->children() as $field) { - /** - * @var \SimpleXMLElement $field - */ - switch ($field->getName()) { - case 'name': - $field_name = (string)$field; - $field_name = $this->platform->quoteIdentifier($field_name); - $fields[] = $field_name; - break; - case 'sorting': - break; - default: - throw new \DomainException('Unknown element: ' . $field->getName()); - - } - } - break; - default: - throw new \DomainException('Unknown element: ' . $child->getName()); - - } - } - if (!empty($fields)) { - if (isset($primary) && $primary) { - if ($table->hasPrimaryKey()) { - return; - } - $table->setPrimaryKey($fields, $name); - } else { - if (isset($unique) && $unique) { - $table->addUniqueIndex($fields, $name); - } else { - $table->addIndex($fields, $name); - } - } - } else { - throw new \DomainException('Empty index definition: ' . $name . ' options:' . print_r($fields, true)); - } - } - - /** - * @param \SimpleXMLElement|string $xml - * @return bool - */ - private function asBool($xml) { - $result = (string)$xml; - if ($result == 'true') { - $result = true; - } elseif ($result == 'false') { - $result = false; - } - return (bool)$result; - } -} diff --git a/lib/private/DB/MigrationException.php b/lib/private/DB/MigrationException.php index 76d769e08be..5b50f8ed3a4 100644 --- a/lib/private/DB/MigrationException.php +++ b/lib/private/DB/MigrationException.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class MigrationException extends \Exception { diff --git a/lib/private/DB/MigrationService.php b/lib/private/DB/MigrationService.php index 8bbd077d85d..40579c7a898 100644 --- a/lib/private/DB/MigrationService.php +++ b/lib/private/DB/MigrationService.php @@ -1,80 +1,56 @@ <?php + /** - * @copyright Copyright (c) 2017 Joas Schilling <coding@schilljs.com> - * @copyright Copyright (c) 2017, ownCloud GmbH - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Julius Härtl <jus@bitgrid.net> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2017 ownCloud GmbH + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; -use Doctrine\DBAL\Exception\DriverException; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Schema\SchemaException; use Doctrine\DBAL\Schema\Sequence; use Doctrine\DBAL\Schema\Table; -use Doctrine\DBAL\Types\Types; use OC\App\InfoParser; use OC\IntegrityCheck\Helpers\AppLocator; use OC\Migration\SimpleOutput; use OCP\AppFramework\App; use OCP\AppFramework\QueryException; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\IDBConnection; use OCP\Migration\IMigrationStep; use OCP\Migration\IOutput; +use OCP\Server; +use Psr\Log\LoggerInterface; class MigrationService { - - /** @var boolean */ - private $migrationTableCreated; - /** @var array */ - private $migrations; - /** @var IOutput */ - private $output; - /** @var Connection */ - private $connection; - /** @var string */ - private $appName; - /** @var bool */ - private $checkOracle; + private bool $migrationTableCreated; + private array $migrations; + private string $migrationsPath; + private string $migrationsNamespace; + private IOutput $output; + private LoggerInterface $logger; + private Connection $connection; + private string $appName; + private bool $checkOracle; /** - * MigrationService constructor. - * - * @param $appName - * @param Connection $connection - * @param AppLocator $appLocator - * @param IOutput|null $output * @throws \Exception */ - public function __construct($appName, Connection $connection, IOutput $output = null, AppLocator $appLocator = null) { + public function __construct(string $appName, Connection $connection, ?IOutput $output = null, ?AppLocator $appLocator = null, ?LoggerInterface $logger = null) { $this->appName = $appName; $this->connection = $connection; - $this->output = $output; - if (null === $this->output) { - $this->output = new SimpleOutput(\OC::$server->getLogger(), $appName); + if ($logger === null) { + $this->logger = Server::get(LoggerInterface::class); + } else { + $this->logger = $logger; + } + if ($output === null) { + $this->output = new SimpleOutput($this->logger, $appName); + } else { + $this->output = $output; } if ($appName === 'core') { @@ -82,7 +58,7 @@ class MigrationService { $this->migrationsNamespace = 'OC\\Core\\Migrations'; $this->checkOracle = true; } else { - if (null === $appLocator) { + if ($appLocator === null) { $appLocator = new AppLocator(); } $appPath = $appLocator->getAppPath($appName); @@ -105,22 +81,20 @@ class MigrationService { } } } + $this->migrationTableCreated = false; } /** * Returns the name of the app for which this migration is executed - * - * @return string */ - public function getApp() { + public function getApp(): string { return $this->appName; } /** - * @return bool * @codeCoverageIgnore - this will implicitly tested on installation */ - private function createMigrationTable() { + private function createMigrationTable(): bool { if ($this->migrationTableCreated) { return false; } @@ -185,7 +159,7 @@ class MigrationService { /** * Returns all versions which have already been applied * - * @return string[] + * @return list<string> * @codeCoverageIgnore - no need to test this */ public function getMigratedVersions() { @@ -197,24 +171,44 @@ class MigrationService { ->where($qb->expr()->eq('app', $qb->createNamedParameter($this->getApp()))) ->orderBy('version'); - $result = $qb->execute(); + $result = $qb->executeQuery(); $rows = $result->fetchAll(\PDO::FETCH_COLUMN); $result->closeCursor(); + usort($rows, $this->sortMigrations(...)); + return $rows; } /** * Returns all versions which are available in the migration folder - * - * @return array + * @return list<string> */ - public function getAvailableVersions() { + public function getAvailableVersions(): array { $this->ensureMigrationsAreLoaded(); - return array_map('strval', array_keys($this->migrations)); + $versions = array_map('strval', array_keys($this->migrations)); + usort($versions, $this->sortMigrations(...)); + return $versions; + } + + protected function sortMigrations(string $a, string $b): int { + preg_match('/(\d+)Date(\d+)/', basename($a), $matchA); + preg_match('/(\d+)Date(\d+)/', basename($b), $matchB); + if (!empty($matchA) && !empty($matchB)) { + $versionA = (int)$matchA[1]; + $versionB = (int)$matchB[1]; + if ($versionA !== $versionB) { + return ($versionA < $versionB) ? -1 : 1; + } + return strnatcmp($matchA[2], $matchB[2]); + } + return strnatcmp(basename($a), basename($b)); } - protected function findMigrations() { + /** + * @return array<string, string> + */ + protected function findMigrations(): array { $directory = realpath($this->migrationsPath); if ($directory === false || !file_exists($directory) || !is_dir($directory)) { return []; @@ -229,23 +223,13 @@ class MigrationService { \RegexIterator::GET_MATCH); $files = array_keys(iterator_to_array($iterator)); - uasort($files, function ($a, $b) { - preg_match('/^Version(\d+)Date(\d+)\\.php$/', basename($a), $matchA); - preg_match('/^Version(\d+)Date(\d+)\\.php$/', basename($b), $matchB); - if (!empty($matchA) && !empty($matchB)) { - if ($matchA[1] !== $matchB[1]) { - return ($matchA[1] < $matchB[1]) ? -1 : 1; - } - return ($matchA[2] < $matchB[2]) ? -1 : 1; - } - return (basename($a) < basename($b)) ? -1 : 1; - }); + usort($files, $this->sortMigrations(...)); $migrations = []; foreach ($files as $file) { $className = basename($file, '.php'); - $version = (string) substr($className, 7); + $version = (string)substr($className, 7); if ($version === '0') { throw new \InvalidArgumentException( "Cannot load a migrations with the name '$version' because it is a reserved number" @@ -267,7 +251,7 @@ class MigrationService { $toBeExecuted = []; foreach ($availableMigrations as $v) { - if ($to !== 'latest' && $v > $to) { + if ($to !== 'latest' && ($this->sortMigrations($v, $to) > 0)) { continue; } if ($this->shallBeExecuted($v, $knownMigrations)) { @@ -331,10 +315,9 @@ class MigrationService { /** * Return the explicit version for the aliases; current, next, prev, latest * - * @param string $alias * @return mixed|null|string */ - public function getMigration($alias) { + public function getMigration(string $alias) { switch ($alias) { case 'current': return $this->getCurrentVersion(); @@ -351,29 +334,22 @@ class MigrationService { return '0'; } - /** - * @param string $version - * @param int $delta - * @return null|string - */ - private function getRelativeVersion($version, $delta) { + private function getRelativeVersion(string $version, int $delta): ?string { $this->ensureMigrationsAreLoaded(); $versions = $this->getAvailableVersions(); - array_unshift($versions, 0); + array_unshift($versions, '0'); + /** @var int $offset */ $offset = array_search($version, $versions, true); if ($offset === false || !isset($versions[$offset + $delta])) { // Unknown version or delta out of bounds. return null; } - return (string) $versions[$offset + $delta]; + return (string)$versions[$offset + $delta]; } - /** - * @return string - */ - private function getCurrentVersion() { + private function getCurrentVersion(): string { $m = $this->getMigratedVersions(); if (count($m) === 0) { return '0'; @@ -383,11 +359,9 @@ class MigrationService { } /** - * @param string $version - * @return string * @throws \InvalidArgumentException */ - private function getClass($version) { + private function getClass(string $version): string { $this->ensureMigrationsAreLoaded(); if (isset($this->migrations[$version])) { @@ -399,22 +373,18 @@ class MigrationService { /** * Allows to set an IOutput implementation which is used for logging progress and messages - * - * @param IOutput $output */ - public function setOutput(IOutput $output) { + public function setOutput(IOutput $output): void { $this->output = $output; } /** * Applies all not yet applied versions up to $to - * - * @param string $to - * @param bool $schemaOnly * @throws \InvalidArgumentException */ - public function migrate($to = 'latest', $schemaOnly = false) { + public function migrate(string $to = 'latest', bool $schemaOnly = false): void { if ($schemaOnly) { + $this->output->debug('Migrating schema only'); $this->migrateSchemaOnly($to); return; } @@ -424,21 +394,19 @@ class MigrationService { foreach ($toBeExecuted as $version) { try { $this->executeStep($version, $schemaOnly); - } catch (DriverException $e) { + } catch (\Exception $e) { // The exception itself does not contain the name of the migration, // so we wrap it here, to make debugging easier. - throw new \Exception('Database error when running migration ' . $to . ' for app ' . $this->getApp(), 0, $e); + throw new \Exception('Database error when running migration ' . $version . ' for app ' . $this->getApp() . PHP_EOL . $e->getMessage(), 0, $e); } } } /** * Applies all not yet applied versions up to $to - * - * @param string $to * @throws \InvalidArgumentException */ - public function migrateSchemaOnly($to = 'latest') { + public function migrateSchemaOnly(string $to = 'latest'): void { // read known migrations $toBeExecuted = $this->getMigrationsToExecute($to); @@ -448,24 +416,32 @@ class MigrationService { $toSchema = null; foreach ($toBeExecuted as $version) { + $this->output->debug('- Reading ' . $version); $instance = $this->createInstance($version); - $toSchema = $instance->changeSchema($this->output, function () use ($toSchema) { + $toSchema = $instance->changeSchema($this->output, function () use ($toSchema): ISchemaWrapper { return $toSchema ?: new SchemaWrapper($this->connection); }, ['tablePrefix' => $this->connection->getPrefix()]) ?: $toSchema; - - $this->markAsExecuted($version); } if ($toSchema instanceof SchemaWrapper) { + $this->output->debug('- Checking target database schema'); $targetSchema = $toSchema->getWrappedSchema(); + $this->ensureUniqueNamesConstraints($targetSchema, true); if ($this->checkOracle) { $beforeSchema = $this->connection->createSchema(); - $this->ensureOracleIdentifierLengthLimit($beforeSchema, $targetSchema, strlen($this->connection->getPrefix())); + $this->ensureOracleConstraints($beforeSchema, $targetSchema, strlen($this->connection->getPrefix())); } + + $this->output->debug('- Migrate database schema'); $this->connection->migrateToSchema($targetSchema); $toSchema->performDropTableCalls(); } + + $this->output->debug('- Mark migrations as executed'); + foreach ($toBeExecuted as $version) { + $this->markAsExecuted($version); + } } /** @@ -491,10 +467,10 @@ class MigrationService { * @return IMigrationStep * @throws \InvalidArgumentException */ - protected function createInstance($version) { + public function createInstance($version) { $class = $this->getClass($version); try { - $s = \OC::$server->query($class); + $s = \OCP\Server::get($class); if (!$s instanceof IMigrationStep) { throw new \InvalidArgumentException('Not a valid migration'); @@ -521,27 +497,28 @@ class MigrationService { $instance = $this->createInstance($version); if (!$schemaOnly) { - $instance->preSchemaChange($this->output, function () { + $instance->preSchemaChange($this->output, function (): ISchemaWrapper { return new SchemaWrapper($this->connection); }, ['tablePrefix' => $this->connection->getPrefix()]); } - $toSchema = $instance->changeSchema($this->output, function () { + $toSchema = $instance->changeSchema($this->output, function (): ISchemaWrapper { return new SchemaWrapper($this->connection); }, ['tablePrefix' => $this->connection->getPrefix()]); if ($toSchema instanceof SchemaWrapper) { $targetSchema = $toSchema->getWrappedSchema(); + $this->ensureUniqueNamesConstraints($targetSchema, $schemaOnly); if ($this->checkOracle) { $sourceSchema = $this->connection->createSchema(); - $this->ensureOracleIdentifierLengthLimit($sourceSchema, $targetSchema, strlen($this->connection->getPrefix())); + $this->ensureOracleConstraints($sourceSchema, $targetSchema, strlen($this->connection->getPrefix())); } $this->connection->migrateToSchema($targetSchema); $toSchema->performDropTableCalls(); } if (!$schemaOnly) { - $instance->postSchemaChange($this->output, function () { + $instance->postSchemaChange($this->output, function (): ISchemaWrapper { return new SchemaWrapper($this->connection); }, ['tablePrefix' => $this->connection->getPrefix()]); } @@ -549,7 +526,29 @@ class MigrationService { $this->markAsExecuted($version); } - public function ensureOracleIdentifierLengthLimit(Schema $sourceSchema, Schema $targetSchema, int $prefixLength) { + /** + * Naming constraints: + * - Tables names must be 30 chars or shorter (27 + oc_ prefix) + * - Column names must be 30 chars or shorter + * - Index names must be 30 chars or shorter + * - Sequence names must be 30 chars or shorter + * - Primary key names must be set or the table name 23 chars or shorter + * + * Data constraints: + * - Tables need a primary key (Not specific to Oracle, but required for performant clustering support) + * - Columns with "NotNull" can not have empty string as default value + * - Columns with "NotNull" can not have number 0 as default value + * - Columns with type "bool" (which is in fact integer of length 1) can not be "NotNull" as it can not store 0/false + * - Columns with type "string" can not be longer than 4.000 characters, use "text" instead + * + * @see https://github.com/nextcloud/documentation/blob/master/developer_manual/basics/storage/database.rst + * + * @param Schema $sourceSchema + * @param Schema $targetSchema + * @param int $prefixLength + * @throws \Doctrine\DBAL\Exception + */ + public function ensureOracleConstraints(Schema $sourceSchema, Schema $targetSchema, int $prefixLength) { $sequences = $targetSchema->getSequences(); foreach ($targetSchema->getTables() as $table) { @@ -563,13 +562,31 @@ class MigrationService { } foreach ($table->getColumns() as $thing) { - if ((!$sourceTable instanceof Table || !$sourceTable->hasColumn($thing->getName())) && \strlen($thing->getName()) > 30) { - throw new \InvalidArgumentException('Column name "' . $table->getName() . '"."' . $thing->getName() . '" is too long.'); + // If the table doesn't exist OR if the column doesn't exist in the table + if (!$sourceTable instanceof Table || !$sourceTable->hasColumn($thing->getName())) { + if (\strlen($thing->getName()) > 30) { + throw new \InvalidArgumentException('Column name "' . $table->getName() . '"."' . $thing->getName() . '" is too long.'); + } + + if ($thing->getNotnull() && $thing->getDefault() === '' + && $sourceTable instanceof Table && !$sourceTable->hasColumn($thing->getName())) { + throw new \InvalidArgumentException('Column "' . $table->getName() . '"."' . $thing->getName() . '" is NotNull, but has empty string or null as default.'); + } + + if ($thing->getNotnull() && $thing->getType()->getName() === Types::BOOLEAN) { + throw new \InvalidArgumentException('Column "' . $table->getName() . '"."' . $thing->getName() . '" is type Bool and also NotNull, so it can not store "false".'); + } + + $sourceColumn = null; + } else { + $sourceColumn = $sourceTable->getColumn($thing->getName()); } - if ($thing->getNotnull() && $thing->getDefault() === '' - && $sourceTable instanceof Table && !$sourceTable->hasColumn($thing->getName())) { - throw new \InvalidArgumentException('Column name "' . $table->getName() . '"."' . $thing->getName() . '" is NotNull, but has empty string or null as default.'); + // If the column was just created OR the length changed OR the type changed + // we will NOT detect invalid length if the column is not modified + if (($sourceColumn === null || $sourceColumn->getLength() !== $thing->getLength() || $sourceColumn->getType()->getName() !== Types::STRING) + && $thing->getLength() > 4000 && $thing->getType()->getName() === Types::STRING) { + throw new \InvalidArgumentException('Column "' . $table->getName() . '"."' . $thing->getName() . '" is type String, but exceeding the 4.000 length limit.'); } } @@ -581,7 +598,7 @@ class MigrationService { foreach ($table->getForeignKeys() as $thing) { if ((!$sourceTable instanceof Table || !$sourceTable->hasForeignKey($thing->getName())) && \strlen($thing->getName()) > 30) { - throw new \InvalidArgumentException('Foreign key name "' . $table->getName() . '"."' . $thing->getName() . '" is too long.'); + throw new \InvalidArgumentException('Foreign key name "' . $table->getName() . '"."' . $thing->getName() . '" is too long.'); } } @@ -590,7 +607,7 @@ class MigrationService { $indexName = strtolower($primaryKey->getName()); $isUsingDefaultName = $indexName === 'primary'; - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { + if ($this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_POSTGRES) { $defaultName = $table->getName() . '_pkey'; $isUsingDefaultName = strtolower($defaultName) === $indexName; @@ -600,7 +617,7 @@ class MigrationService { return $sequence->getName() !== $sequenceName; }); } - } elseif ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { + } elseif ($this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_ORACLE) { $defaultName = $table->getName() . '_seq'; $isUsingDefaultName = strtolower($defaultName) === $indexName; } @@ -611,6 +628,11 @@ class MigrationService { if ($isUsingDefaultName && \strlen($table->getName()) - $prefixLength >= 23) { throw new \InvalidArgumentException('Primary index name on "' . $table->getName() . '" is too long.'); } + } elseif (!$primaryKey instanceof Index && !$sourceTable instanceof Table) { + /** @var LoggerInterface $logger */ + $logger = \OC::$server->get(LoggerInterface::class); + $logger->error('Table "' . $table->getName() . '" has no primary key and therefor will not behave sane in clustered setups. This will throw an exception and not be installable in a future version of Nextcloud.'); + // throw new \InvalidArgumentException('Table "' . $table->getName() . '" has no primary key and therefor will not behave sane in clustered setups.'); } } @@ -621,6 +643,91 @@ class MigrationService { } } + /** + * Ensure naming constraints + * + * Naming constraints: + * - Index, sequence and primary key names must be unique within a Postgres Schema + * + * Only on installation we want to break hard, so that all developers notice + * the bugs when installing the app on any database or CI, and can work on + * fixing their migrations before releasing a version incompatible with Postgres. + * + * In case of updates we might be running on production instances and the + * administrators being faced with the error would not know how to resolve it + * anyway. This can also happen with instances, that had the issue before the + * current update, so we don't want to make their life more complicated + * than needed. + * + * @param Schema $targetSchema + * @param bool $isInstalling + */ + public function ensureUniqueNamesConstraints(Schema $targetSchema, bool $isInstalling): void { + $constraintNames = []; + $sequences = $targetSchema->getSequences(); + + foreach ($targetSchema->getTables() as $table) { + foreach ($table->getIndexes() as $thing) { + $indexName = strtolower($thing->getName()); + if ($indexName === 'primary' || $thing->isPrimary()) { + continue; + } + + if (isset($constraintNames[$thing->getName()])) { + if ($isInstalling) { + throw new \InvalidArgumentException('Index name "' . $thing->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $this->logErrorOrWarning('Index name "' . $thing->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $constraintNames[$thing->getName()] = $table->getName(); + } + + foreach ($table->getForeignKeys() as $thing) { + if (isset($constraintNames[$thing->getName()])) { + if ($isInstalling) { + throw new \InvalidArgumentException('Foreign key name "' . $thing->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $this->logErrorOrWarning('Foreign key name "' . $thing->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $constraintNames[$thing->getName()] = $table->getName(); + } + + $primaryKey = $table->getPrimaryKey(); + if ($primaryKey instanceof Index) { + $indexName = strtolower($primaryKey->getName()); + if ($indexName === 'primary') { + continue; + } + + if (isset($constraintNames[$indexName])) { + if ($isInstalling) { + throw new \InvalidArgumentException('Primary index name "' . $indexName . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $this->logErrorOrWarning('Primary index name "' . $indexName . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $constraintNames[$indexName] = $table->getName(); + } + } + + foreach ($sequences as $sequence) { + if (isset($constraintNames[$sequence->getName()])) { + if ($isInstalling) { + throw new \InvalidArgumentException('Sequence name "' . $sequence->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $this->logErrorOrWarning('Sequence name "' . $sequence->getName() . '" for table "' . $table->getName() . '" collides with the constraint on table "' . $constraintNames[$thing->getName()] . '".'); + } + $constraintNames[$sequence->getName()] = 'sequence'; + } + } + + protected function logErrorOrWarning(string $log): void { + if ($this->output instanceof SimpleOutput) { + $this->output->warning($log); + } else { + $this->logger->error($log); + } + } + private function ensureMigrationsAreLoaded() { if (empty($this->migrations)) { $this->migrations = $this->findMigrations(); diff --git a/lib/private/DB/Migrator.php b/lib/private/DB/Migrator.php index dcf0db89f72..40f8ad9676a 100644 --- a/lib/private/DB/Migrator.php +++ b/lib/private/DB/Migrator.php @@ -1,88 +1,45 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author martin-rueegg <martin.rueegg@metaworx.ch> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author tbelau666 <thomas.belau@gmx.de> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; +use Doctrine\DBAL\Connection; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\Schema\AbstractAsset; -use Doctrine\DBAL\Schema\Comparator; -use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Schema; -use Doctrine\DBAL\Schema\SchemaConfig; -use Doctrine\DBAL\Schema\Table; +use Doctrine\DBAL\Schema\SchemaDiff; use Doctrine\DBAL\Types\StringType; use Doctrine\DBAL\Types\Type; +use OCP\EventDispatcher\IEventDispatcher; use OCP\IConfig; -use OCP\Security\ISecureRandom; -use Symfony\Component\EventDispatcher\EventDispatcherInterface; -use Symfony\Component\EventDispatcher\GenericEvent; use function preg_match; class Migrator { - - /** @var \Doctrine\DBAL\Connection */ + /** @var Connection */ protected $connection; - /** @var ISecureRandom */ - private $random; - /** @var IConfig */ protected $config; - /** @var EventDispatcherInterface */ - private $dispatcher; + private ?IEventDispatcher $dispatcher; /** @var bool */ private $noEmit = false; - /** - * @param \Doctrine\DBAL\Connection $connection - * @param ISecureRandom $random - * @param IConfig $config - * @param EventDispatcherInterface $dispatcher - */ - public function __construct(\Doctrine\DBAL\Connection $connection, - ISecureRandom $random, - IConfig $config, - EventDispatcherInterface $dispatcher = null) { + public function __construct(Connection $connection, + IConfig $config, + ?IEventDispatcher $dispatcher = null) { $this->connection = $connection; - $this->random = $random; $this->config = $config; $this->dispatcher = $dispatcher; } /** - * @param \Doctrine\DBAL\Schema\Schema $targetSchema - * * @throws Exception */ public function migrate(Schema $targetSchema) { @@ -91,14 +48,13 @@ class Migrator { } /** - * @param \Doctrine\DBAL\Schema\Schema $targetSchema * @return string */ public function generateChangeScript(Schema $targetSchema) { $schemaDiff = $this->getDiff($targetSchema, $this->connection); $script = ''; - $sqls = $schemaDiff->toSql($this->connection->getDatabasePlatform()); + $sqls = $this->connection->getDatabasePlatform()->getAlterSchemaSQL($schemaDiff); foreach ($sqls as $sql) { $script .= $this->convertStatementToScript($sql); } @@ -107,73 +63,6 @@ class Migrator { } /** - * Create a unique name for the temporary table - * - * @param string $name - * @return string - */ - protected function generateTemporaryTableName($name) { - return $this->config->getSystemValue('dbtableprefix', 'oc_') . $name . '_' . $this->random->generate(13, ISecureRandom::CHAR_LOWER . ISecureRandom::CHAR_DIGITS); - } - - /** - * Check the migration of a table on a copy so we can detect errors before messing with the real table - * - * @param \Doctrine\DBAL\Schema\Table $table - * @throws \OC\DB\MigrationException - */ - protected function checkTableMigrate(Table $table) { - $name = $table->getName(); - $tmpName = $this->generateTemporaryTableName($name); - - $this->copyTable($name, $tmpName); - - //create the migration schema for the temporary table - $tmpTable = $this->renameTableSchema($table, $tmpName); - $schemaConfig = new SchemaConfig(); - $schemaConfig->setName($this->connection->getDatabase()); - $schema = new Schema([$tmpTable], [], $schemaConfig); - - try { - $this->applySchema($schema); - $this->dropTable($tmpName); - } catch (Exception $e) { - // pgsql needs to commit it's failed transaction before doing anything else - if ($this->connection->isTransactionActive()) { - $this->connection->commit(); - } - $this->dropTable($tmpName); - throw new MigrationException($table->getName(), $e->getMessage()); - } - } - - /** - * @param \Doctrine\DBAL\Schema\Table $table - * @param string $newName - * @return \Doctrine\DBAL\Schema\Table - */ - protected function renameTableSchema(Table $table, $newName) { - /** - * @var \Doctrine\DBAL\Schema\Index[] $indexes - */ - $indexes = $table->getIndexes(); - $newIndexes = []; - foreach ($indexes as $index) { - if ($index->isPrimary()) { - // do not rename primary key - $indexName = $index->getName(); - } else { - // avoid conflicts in index names - $indexName = $this->config->getSystemValue('dbtableprefix', 'oc_') . $this->random->generate(13, ISecureRandom::CHAR_LOWER); - } - $newIndexes[] = new Index($indexName, $index->getColumns(), $index->isUnique(), $index->isPrimary()); - } - - // foreign keys are not supported so we just set it to an empty array - return new Table($newName, $table->getColumns(), $newIndexes, [], [], $table->getOptions()); - } - - /** * @throws Exception */ public function createSchema() { @@ -181,24 +70,24 @@ class Migrator { /** @var string|AbstractAsset $asset */ $filterExpression = $this->getFilterExpression(); if ($asset instanceof AbstractAsset) { - return preg_match($filterExpression, $asset->getName()) !== false; + return preg_match($filterExpression, $asset->getName()) === 1; } - return preg_match($filterExpression, $asset) !== false; + return preg_match($filterExpression, $asset) === 1; }); - return $this->connection->getSchemaManager()->createSchema(); + return $this->connection->createSchemaManager()->introspectSchema(); } /** - * @param Schema $targetSchema - * @param \Doctrine\DBAL\Connection $connection - * @return \Doctrine\DBAL\Schema\SchemaDiff + * @return SchemaDiff */ - protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { - // adjust varchar columns with a length higher then getVarcharMaxLength to clob + protected function getDiff(Schema $targetSchema, Connection $connection) { + // Adjust STRING columns with a length higher than 4000 to TEXT (clob) + // for consistency between the supported databases and + // old vs. new installations. foreach ($targetSchema->getTables() as $table) { foreach ($table->getColumns() as $column) { if ($column->getType() instanceof StringType) { - if ($column->getLength() > $connection->getDatabasePlatform()->getVarcharMaxLength()) { + if ($column->getLength() > 4000) { $column->setType(Type::getType('text')); $column->setLength(null); } @@ -210,11 +99,11 @@ class Migrator { /** @var string|AbstractAsset $asset */ $filterExpression = $this->getFilterExpression(); if ($asset instanceof AbstractAsset) { - return preg_match($filterExpression, $asset->getName()) !== false; + return preg_match($filterExpression, $asset->getName()) === 1; } - return preg_match($filterExpression, $asset) !== false; + return preg_match($filterExpression, $asset) === 1; }); - $sourceSchema = $connection->getSchemaManager()->createSchema(); + $sourceSchema = $connection->createSchemaManager()->introspectSchema(); // remove tables we don't know about foreach ($sourceSchema->getTables() as $table) { @@ -229,17 +118,14 @@ class Migrator { } } - $comparator = new Comparator(); - return $comparator->compare($sourceSchema, $targetSchema); + $comparator = $connection->createSchemaManager()->createComparator(); + return $comparator->compareSchemas($sourceSchema, $targetSchema); } /** - * @param \Doctrine\DBAL\Schema\Schema $targetSchema - * @param \Doctrine\DBAL\Connection $connection - * * @throws Exception */ - protected function applySchema(Schema $targetSchema, \Doctrine\DBAL\Connection $connection = null) { + protected function applySchema(Schema $targetSchema, ?Connection $connection = null) { if (is_null($connection)) { $connection = $this->connection; } @@ -249,11 +135,11 @@ class Migrator { if (!$connection->getDatabasePlatform() instanceof MySQLPlatform) { $connection->beginTransaction(); } - $sqls = $schemaDiff->toSql($connection->getDatabasePlatform()); + $sqls = $connection->getDatabasePlatform()->getAlterSchemaSQL($schemaDiff); $step = 0; foreach ($sqls as $sql) { $this->emit($sql, $step++, count($sqls)); - $connection->query($sql); + $connection->executeStatement($sql); } if (!$connection->getDatabasePlatform() instanceof MySQLPlatform) { $connection->commit(); @@ -261,25 +147,6 @@ class Migrator { } /** - * @param string $sourceName - * @param string $targetName - */ - protected function copyTable($sourceName, $targetName) { - $quotedSource = $this->connection->quoteIdentifier($sourceName); - $quotedTarget = $this->connection->quoteIdentifier($targetName); - - $this->connection->exec('CREATE TABLE ' . $quotedTarget . ' (LIKE ' . $quotedSource . ')'); - $this->connection->exec('INSERT INTO ' . $quotedTarget . ' SELECT * FROM ' . $quotedSource); - } - - /** - * @param string $name - */ - protected function dropTable($name) { - $this->connection->exec('DROP TABLE ' . $this->connection->quoteIdentifier($name)); - } - - /** * @param $statement * @return string */ @@ -291,23 +158,16 @@ class Migrator { } protected function getFilterExpression() { - return '/^' . preg_quote($this->config->getSystemValue('dbtableprefix', 'oc_')) . '/'; + return '/^' . preg_quote($this->config->getSystemValueString('dbtableprefix', 'oc_'), '/') . '/'; } - protected function emit($sql, $step, $max) { + protected function emit(string $sql, int $step, int $max): void { if ($this->noEmit) { return; } if (is_null($this->dispatcher)) { return; } - $this->dispatcher->dispatch('\OC\DB\Migrator::executeSql', new GenericEvent($sql, [$step + 1, $max])); - } - - private function emitCheckStep($tableName, $step, $max) { - if (is_null($this->dispatcher)) { - return; - } - $this->dispatcher->dispatch('\OC\DB\Migrator::checkTable', new GenericEvent($tableName, [$step + 1, $max])); + $this->dispatcher->dispatchTyped(new MigratorExecuteSqlEvent($sql, $step, $max)); } } diff --git a/lib/private/DB/MigratorExecuteSqlEvent.php b/lib/private/DB/MigratorExecuteSqlEvent.php new file mode 100644 index 00000000000..340cd636300 --- /dev/null +++ b/lib/private/DB/MigratorExecuteSqlEvent.php @@ -0,0 +1,39 @@ +<?php + +declare(strict_types=1); + +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ +namespace OC\DB; + +use OCP\EventDispatcher\Event; + +class MigratorExecuteSqlEvent extends Event { + private string $sql; + private int $current; + private int $max; + + public function __construct( + string $sql, + int $current, + int $max, + ) { + $this->sql = $sql; + $this->current = $current; + $this->max = $max; + } + + public function getSql(): string { + return $this->sql; + } + + public function getCurrentStep(): int { + return $this->current; + } + + public function getMaxStep(): int { + return $this->max; + } +} diff --git a/lib/private/DB/MissingColumnInformation.php b/lib/private/DB/MissingColumnInformation.php index d1c81c2e199..5e8402d394e 100644 --- a/lib/private/DB/MissingColumnInformation.php +++ b/lib/private/DB/MissingColumnInformation.php @@ -3,31 +3,13 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2020 Joas Schilling <coding@schilljs.com> - * - * @author Joas Schilling <coding@schilljs.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; class MissingColumnInformation { - private $listOfMissingColumns = []; + private array $listOfMissingColumns = []; public function addHintForMissingColumn(string $tableName, string $columnName): void { $this->listOfMissingColumns[] = [ diff --git a/lib/private/DB/MissingIndexInformation.php b/lib/private/DB/MissingIndexInformation.php index 04853dcac2d..99a81782858 100644 --- a/lib/private/DB/MissingIndexInformation.php +++ b/lib/private/DB/MissingIndexInformation.php @@ -3,41 +3,22 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Morris Jobke <hey@morrisjobke.de> - * - * @author Morris Jobke <hey@morrisjobke.de> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; class MissingIndexInformation { - private $listOfMissingIndexes = []; + private array $listOfMissingIndices = []; - public function addHintForMissingSubject(string $tableName, string $indexName) { - $this->listOfMissingIndexes[] = [ + public function addHintForMissingIndex(string $tableName, string $indexName): void { + $this->listOfMissingIndices[] = [ 'tableName' => $tableName, 'indexName' => $indexName ]; } - public function getListOfMissingIndexes(): array { - return $this->listOfMissingIndexes; + public function getListOfMissingIndices(): array { + return $this->listOfMissingIndices; } } diff --git a/lib/private/DB/MissingPrimaryKeyInformation.php b/lib/private/DB/MissingPrimaryKeyInformation.php index 62b5dd2c631..355ce86423a 100644 --- a/lib/private/DB/MissingPrimaryKeyInformation.php +++ b/lib/private/DB/MissingPrimaryKeyInformation.php @@ -3,33 +3,15 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2018 Morris Jobke <hey@morrisjobke.de> - * - * @author Joas Schilling <coding@schilljs.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; class MissingPrimaryKeyInformation { - private $listOfMissingPrimaryKeys = []; + private array $listOfMissingPrimaryKeys = []; - public function addHintForMissingSubject(string $tableName) { + public function addHintForMissingPrimaryKey(string $tableName): void { $this->listOfMissingPrimaryKeys[] = [ 'tableName' => $tableName, ]; diff --git a/lib/private/DB/MySQLMigrator.php b/lib/private/DB/MySQLMigrator.php deleted file mode 100644 index 58c54683a3a..00000000000 --- a/lib/private/DB/MySQLMigrator.php +++ /dev/null @@ -1,76 +0,0 @@ -<?php -/** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * - */ - -namespace OC\DB; - -use Doctrine\DBAL\Schema\Schema; -use Doctrine\DBAL\Schema\Table; - -class MySQLMigrator extends Migrator { - /** - * @param Schema $targetSchema - * @param \Doctrine\DBAL\Connection $connection - * @return \Doctrine\DBAL\Schema\SchemaDiff - */ - protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { - $platform = $connection->getDatabasePlatform(); - $platform->registerDoctrineTypeMapping('enum', 'string'); - $platform->registerDoctrineTypeMapping('bit', 'string'); - - $schemaDiff = parent::getDiff($targetSchema, $connection); - - // identifiers need to be quoted for mysql - foreach ($schemaDiff->changedTables as $tableDiff) { - $tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name); - foreach ($tableDiff->changedColumns as $column) { - $column->oldColumnName = $this->connection->quoteIdentifier($column->oldColumnName); - } - } - - return $schemaDiff; - } - - /** - * Speed up migration test by disabling autocommit and unique indexes check - * - * @param \Doctrine\DBAL\Schema\Table $table - * @throws \OC\DB\MigrationException - */ - protected function checkTableMigrate(Table $table) { - $this->connection->exec('SET autocommit=0'); - $this->connection->exec('SET unique_checks=0'); - - try { - parent::checkTableMigrate($table); - } catch (\Exception $e) { - $this->connection->exec('SET unique_checks=1'); - $this->connection->exec('SET autocommit=1'); - throw new MigrationException($table->getName(), $e->getMessage()); - } - $this->connection->exec('SET unique_checks=1'); - $this->connection->exec('SET autocommit=1'); - } -} diff --git a/lib/private/DB/MySqlTools.php b/lib/private/DB/MySqlTools.php index e738a2bd94a..3413be43417 100644 --- a/lib/private/DB/MySqlTools.php +++ b/lib/private/DB/MySqlTools.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2017, ownCloud GmbH - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2017 ownCloud GmbH + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use OCP\IDBConnection; @@ -30,7 +12,6 @@ use OCP\IDBConnection; * Various MySQL specific helper functions. */ class MySqlTools { - /** * @param IDBConnection $connection * @return bool @@ -65,7 +46,7 @@ class MySqlTools { return false; } - return strpos($row, 'maria') && version_compare($row, '10.3', '>=') || - strpos($row, 'maria') === false && version_compare($row, '8.0', '>='); + return str_contains($row, 'maria') && version_compare($row, '10.3', '>=') + || !str_contains($row, 'maria') && version_compare($row, '8.0', '>='); } } diff --git a/lib/private/DB/OCSqlitePlatform.php b/lib/private/DB/OCSqlitePlatform.php index be33629e885..3e2b10e6e40 100644 --- a/lib/private/DB/OCSqlitePlatform.php +++ b/lib/private/DB/OCSqlitePlatform.php @@ -1,25 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class OCSqlitePlatform extends \Doctrine\DBAL\Platforms\SqlitePlatform { diff --git a/lib/private/DB/ObjectParameter.php b/lib/private/DB/ObjectParameter.php new file mode 100644 index 00000000000..1b013734c95 --- /dev/null +++ b/lib/private/DB/ObjectParameter.php @@ -0,0 +1,47 @@ +<?php + +declare(strict_types = 1); +/** + * This file is part of the Symfony package. + * + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2022 Fabien Potencier <fabien@symfony.com> + * SPDX-License-Identifier: AGPL-3.0-or-later AND MIT + */ +namespace OC\DB; + +final class ObjectParameter { + private $object; + private $error; + private $stringable; + private $class; + + /** + * @param object $object + */ + public function __construct($object, ?\Throwable $error) { + $this->object = $object; + $this->error = $error; + $this->stringable = \is_callable([$object, '__toString']); + $this->class = \get_class($object); + } + + /** + * @return object + */ + public function getObject() { + return $this->object; + } + + public function getError(): ?\Throwable { + return $this->error; + } + + public function isStringable(): bool { + return $this->stringable; + } + + public function getClass(): string { + return $this->class; + } +} diff --git a/lib/private/DB/OracleConnection.php b/lib/private/DB/OracleConnection.php index 8198f3bd5e5..5ffb65d801d 100644 --- a/lib/private/DB/OracleConnection.php +++ b/lib/private/DB/OracleConnection.php @@ -1,36 +1,17 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Bart Visscher <bartv@thisnet.nl> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; class OracleConnection extends Connection { /** * Quote the keys of the array + * @param array<string, string> $data + * @return array<string, string> */ private function quoteKeys(array $data) { $return = []; @@ -87,7 +68,7 @@ class OracleConnection extends Connection { public function dropTable($table) { $table = $this->tablePrefix . trim($table); $table = $this->quoteIdentifier($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); if ($schema->tablesExist([$table])) { $schema->dropTable($table); } @@ -102,7 +83,7 @@ class OracleConnection extends Connection { public function tableExists($table) { $table = $this->tablePrefix . trim($table); $table = $this->quoteIdentifier($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); return $schema->tablesExist([$table]); } } diff --git a/lib/private/DB/OracleMigrator.php b/lib/private/DB/OracleMigrator.php index 7a327a42797..c5a7646dbb2 100644 --- a/lib/private/DB/OracleMigrator.php +++ b/lib/private/DB/OracleMigrator.php @@ -1,215 +1,122 @@ <?php + +declare(strict_types=1); /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * @author Jörn Friedrich Dreyer <jfd@butonic.de> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Piotr Mrowczynski <mrow4a@yahoo.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\DBAL\Exception; -use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\ColumnDiff; -use Doctrine\DBAL\Schema\ForeignKeyConstraint; -use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Schema; -use Doctrine\DBAL\Schema\Table; class OracleMigrator extends Migrator { - - /** - * Quote a column's name but changing the name requires recreating - * the column instance and copying over all properties. - * - * @param Column $column old column - * @return Column new column instance with new name - */ - protected function quoteColumn(Column $column) { - $newColumn = new Column( - $this->connection->quoteIdentifier($column->getName()), - $column->getType() - ); - $newColumn->setAutoincrement($column->getAutoincrement()); - $newColumn->setColumnDefinition($column->getColumnDefinition()); - $newColumn->setComment($column->getComment()); - $newColumn->setDefault($column->getDefault()); - $newColumn->setFixed($column->getFixed()); - $newColumn->setLength($column->getLength()); - $newColumn->setNotnull($column->getNotnull()); - $newColumn->setPrecision($column->getPrecision()); - $newColumn->setScale($column->getScale()); - $newColumn->setUnsigned($column->getUnsigned()); - $newColumn->setPlatformOptions($column->getPlatformOptions()); - $newColumn->setCustomSchemaOptions($column->getPlatformOptions()); - return $newColumn; - } - - /** - * Quote an index's name but changing the name requires recreating - * the index instance and copying over all properties. - * - * @param Index $index old index - * @return Index new index instance with new name - */ - protected function quoteIndex($index) { - return new Index( - //TODO migrate existing uppercase indexes, then $this->connection->quoteIdentifier($index->getName()), - $index->getName(), - array_map(function ($columnName) { - return $this->connection->quoteIdentifier($columnName); - }, $index->getColumns()), - $index->isUnique(), - $index->isPrimary(), - $index->getFlags(), - $index->getOptions() - ); - } - - /** - * Quote an ForeignKeyConstraint's name but changing the name requires recreating - * the ForeignKeyConstraint instance and copying over all properties. - * - * @param ForeignKeyConstraint $fkc old fkc - * @return ForeignKeyConstraint new fkc instance with new name - */ - protected function quoteForeignKeyConstraint($fkc) { - return new ForeignKeyConstraint( - array_map(function ($columnName) { - return $this->connection->quoteIdentifier($columnName); - }, $fkc->getLocalColumns()), - $this->connection->quoteIdentifier($fkc->getForeignTableName()), - array_map(function ($columnName) { - return $this->connection->quoteIdentifier($columnName); - }, $fkc->getForeignColumns()), - $fkc->getName(), - $fkc->getOptions() - ); - } - /** * @param Schema $targetSchema * @param \Doctrine\DBAL\Connection $connection * @return \Doctrine\DBAL\Schema\SchemaDiff * @throws Exception */ - protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { - $schemaDiff = parent::getDiff($targetSchema, $connection); - + protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection): \Doctrine\DBAL\Schema\SchemaDiff { // oracle forces us to quote the identifiers - $schemaDiff->newTables = array_map(function (Table $table) { - return new Table( - $this->connection->quoteIdentifier($table->getName()), - array_map(function (Column $column) { - return $this->quoteColumn($column); - }, $table->getColumns()), - array_map(function (Index $index) { - return $this->quoteIndex($index); - }, $table->getIndexes()), - [], - array_map(function (ForeignKeyConstraint $fck) { - return $this->quoteForeignKeyConstraint($fck); - }, $table->getForeignKeys()), - $table->getOptions() - ); - }, $schemaDiff->newTables); - - $schemaDiff->removedTables = array_map(function (Table $table) { - return new Table( + $quotedSchema = new Schema(); + foreach ($targetSchema->getTables() as $table) { + $quotedTable = $quotedSchema->createTable( $this->connection->quoteIdentifier($table->getName()), - $table->getColumns(), - $table->getIndexes(), - [], - $table->getForeignKeys(), - $table->getOptions() ); - }, $schemaDiff->removedTables); - - foreach ($schemaDiff->changedTables as $tableDiff) { - $tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name); - - $tableDiff->addedColumns = array_map(function (Column $column) { - return $this->quoteColumn($column); - }, $tableDiff->addedColumns); - foreach ($tableDiff->changedColumns as $column) { - $column->oldColumnName = $this->connection->quoteIdentifier($column->oldColumnName); - // auto increment is not relevant for oracle and can anyhow not be applied on change - $column->changedProperties = array_diff($column->changedProperties, ['autoincrement', 'unsigned']); + foreach ($table->getColumns() as $column) { + $newColumn = $quotedTable->addColumn( + $this->connection->quoteIdentifier($column->getName()), + $column->getType()->getTypeRegistry()->lookupName($column->getType()), + ); + $newColumn->setAutoincrement($column->getAutoincrement()); + $newColumn->setColumnDefinition($column->getColumnDefinition()); + $newColumn->setComment($column->getComment()); + $newColumn->setDefault($column->getDefault()); + $newColumn->setFixed($column->getFixed()); + $newColumn->setLength($column->getLength()); + $newColumn->setNotnull($column->getNotnull()); + $newColumn->setPrecision($column->getPrecision()); + $newColumn->setScale($column->getScale()); + $newColumn->setUnsigned($column->getUnsigned()); + $newColumn->setPlatformOptions($column->getPlatformOptions()); } - // remove columns that no longer have changed (because autoincrement and unsigned are not supported) - $tableDiff->changedColumns = array_filter($tableDiff->changedColumns, function (ColumnDiff $column) { - return count($column->changedProperties) > 0; - }); - - $tableDiff->removedColumns = array_map(function (Column $column) { - return $this->quoteColumn($column); - }, $tableDiff->removedColumns); - - $tableDiff->renamedColumns = array_map(function (Column $column) { - return $this->quoteColumn($column); - }, $tableDiff->renamedColumns); - $tableDiff->addedIndexes = array_map(function (Index $index) { - return $this->quoteIndex($index); - }, $tableDiff->addedIndexes); - - $tableDiff->changedIndexes = array_map(function (Index $index) { - return $this->quoteIndex($index); - }, $tableDiff->changedIndexes); - - $tableDiff->removedIndexes = array_map(function (Index $index) { - return $this->quoteIndex($index); - }, $tableDiff->removedIndexes); - - $tableDiff->renamedIndexes = array_map(function (Index $index) { - return $this->quoteIndex($index); - }, $tableDiff->renamedIndexes); + foreach ($table->getIndexes() as $index) { + if ($index->isPrimary()) { + $quotedTable->setPrimaryKey( + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $index->getColumns()), + //TODO migrate existing uppercase indexes, then $this->connection->quoteIdentifier($index->getName()), + $index->getName(), + ); + } elseif ($index->isUnique()) { + $quotedTable->addUniqueIndex( + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $index->getColumns()), + //TODO migrate existing uppercase indexes, then $this->connection->quoteIdentifier($index->getName()), + $index->getName(), + $index->getOptions(), + ); + } else { + $quotedTable->addIndex( + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $index->getColumns()), + //TODO migrate existing uppercase indexes, then $this->connection->quoteIdentifier($index->getName()), + $index->getName(), + $index->getFlags(), + $index->getOptions(), + ); + } + } - $tableDiff->addedForeignKeys = array_map(function (ForeignKeyConstraint $fkc) { - return $this->quoteForeignKeyConstraint($fkc); - }, $tableDiff->addedForeignKeys); + foreach ($table->getUniqueConstraints() as $constraint) { + $quotedTable->addUniqueConstraint( + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $constraint->getColumns()), + $this->connection->quoteIdentifier($constraint->getName()), + $constraint->getFlags(), + $constraint->getOptions(), + ); + } - $tableDiff->changedForeignKeys = array_map(function (ForeignKeyConstraint $fkc) { - return $this->quoteForeignKeyConstraint($fkc); - }, $tableDiff->changedForeignKeys); + foreach ($table->getForeignKeys() as $foreignKey) { + $quotedTable->addForeignKeyConstraint( + $this->connection->quoteIdentifier($foreignKey->getForeignTableName()), + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $foreignKey->getLocalColumns()), + array_map(function ($columnName) { + return $this->connection->quoteIdentifier($columnName); + }, $foreignKey->getForeignColumns()), + $foreignKey->getOptions(), + $this->connection->quoteIdentifier($foreignKey->getName()), + ); + } - $tableDiff->removedForeignKeys = array_map(function (ForeignKeyConstraint $fkc) { - return $this->quoteForeignKeyConstraint($fkc); - }, $tableDiff->removedForeignKeys); + foreach ($table->getOptions() as $option => $value) { + $quotedTable->addOption( + $option, + $value, + ); + } } - return $schemaDiff; - } + foreach ($targetSchema->getSequences() as $sequence) { + $quotedSchema->createSequence( + $sequence->getName(), + $sequence->getAllocationSize(), + $sequence->getInitialValue(), + ); + } - /** - * @param string $name - * @return string - */ - protected function generateTemporaryTableName($name) { - return 'oc_' . uniqid(); + return parent::getDiff($quotedSchema, $connection); } /** @@ -217,7 +124,7 @@ class OracleMigrator extends Migrator { * @return string */ protected function convertStatementToScript($statement) { - if (substr($statement, -1) === ';') { + if (str_ends_with($statement, ';')) { return $statement . PHP_EOL . '/' . PHP_EOL; } $script = $statement . ';'; @@ -227,6 +134,6 @@ class OracleMigrator extends Migrator { } protected function getFilterExpression() { - return '/^"' . preg_quote($this->config->getSystemValue('dbtableprefix', 'oc_')) . '/'; + return '/^"' . preg_quote($this->config->getSystemValueString('dbtableprefix', 'oc_')) . '/'; } } diff --git a/lib/private/DB/PgSqlTools.php b/lib/private/DB/PgSqlTools.php index d555bfb391b..d529cb26b09 100644 --- a/lib/private/DB/PgSqlTools.php +++ b/lib/private/DB/PgSqlTools.php @@ -1,29 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Andreas Fischer <bantu@owncloud.com> - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author tbelau666 <thomas.belau@gmx.de> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\DBAL\Schema\AbstractAsset; @@ -35,7 +16,6 @@ use function preg_quote; * Various PostgreSQL specific helper functions. */ class PgSqlTools { - /** @var \OCP\IConfig */ private $config; @@ -56,14 +36,14 @@ class PgSqlTools { $databaseName = $conn->getDatabase(); $conn->getConfiguration()->setSchemaAssetsFilter(function ($asset) { /** @var string|AbstractAsset $asset */ - $filterExpression = '/^' . preg_quote($this->config->getSystemValue('dbtableprefix', 'oc_')) . '/'; + $filterExpression = '/^' . preg_quote($this->config->getSystemValueString('dbtableprefix', 'oc_')) . '/'; if ($asset instanceof AbstractAsset) { return preg_match($filterExpression, $asset->getName()) !== false; } return preg_match($filterExpression, $asset) !== false; }); - foreach ($conn->getSchemaManager()->listSequences() as $sequence) { + foreach ($conn->createSchemaManager()->listSequences() as $sequence) { $sequenceName = $sequence->getName(); $sqlInfo = 'SELECT table_schema, table_name, column_name FROM information_schema.columns diff --git a/lib/private/DB/PostgreSqlMigrator.php b/lib/private/DB/PostgreSqlMigrator.php deleted file mode 100644 index 0c3b7956f47..00000000000 --- a/lib/private/DB/PostgreSqlMigrator.php +++ /dev/null @@ -1,56 +0,0 @@ -<?php -/** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * - */ - -namespace OC\DB; - -use Doctrine\DBAL\Schema\Schema; - -class PostgreSqlMigrator extends Migrator { - /** - * @param Schema $targetSchema - * @param \Doctrine\DBAL\Connection $connection - * @return \Doctrine\DBAL\Schema\SchemaDiff - */ - protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { - $schemaDiff = parent::getDiff($targetSchema, $connection); - - foreach ($schemaDiff->changedTables as $tableDiff) { - // fix default value in brackets - pg 9.4 is returning a negative default value in () - // see https://github.com/doctrine/dbal/issues/2427 - foreach ($tableDiff->changedColumns as $column) { - $column->changedProperties = array_filter($column->changedProperties, function ($changedProperties) use ($column) { - if ($changedProperties !== 'default') { - return true; - } - $fromDefault = $column->fromColumn->getDefault(); - $toDefault = $column->column->getDefault(); - $fromDefault = trim($fromDefault, "()"); - - // by intention usage of != - return $fromDefault != $toDefault; - }); - } - } - - return $schemaDiff; - } -} diff --git a/lib/private/DB/PreparedStatement.php b/lib/private/DB/PreparedStatement.php index f679576a428..54561ed96cd 100644 --- a/lib/private/DB/PreparedStatement.php +++ b/lib/private/DB/PreparedStatement.php @@ -2,27 +2,10 @@ declare(strict_types=1); -/* - * @copyright 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @author 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; use Doctrine\DBAL\Exception; @@ -42,7 +25,6 @@ use PDO; * methods without much magic. */ class PreparedStatement implements IPreparedStatement { - /** @var Statement */ private $statement; @@ -96,6 +78,6 @@ class PreparedStatement implements IPreparedStatement { return $this->result; } - throw new Exception("You have to execute the prepared statement before accessing the results"); + throw new Exception('You have to execute the prepared statement before accessing the results'); } } diff --git a/lib/private/DB/QueryBuilder/CompositeExpression.php b/lib/private/DB/QueryBuilder/CompositeExpression.php index 512343662ad..122998036ae 100644 --- a/lib/private/DB/QueryBuilder/CompositeExpression.php +++ b/lib/private/DB/QueryBuilder/CompositeExpression.php @@ -1,24 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ namespace OC\DB\QueryBuilder; @@ -26,16 +11,13 @@ namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\ICompositeExpression; class CompositeExpression implements ICompositeExpression, \Countable { - /** @var \Doctrine\DBAL\Query\Expression\CompositeExpression */ - protected $compositeExpression; + public const TYPE_AND = 'AND'; + public const TYPE_OR = 'OR'; - /** - * Constructor. - * - * @param \Doctrine\DBAL\Query\Expression\CompositeExpression $compositeExpression - */ - public function __construct(\Doctrine\DBAL\Query\Expression\CompositeExpression $compositeExpression) { - $this->compositeExpression = $compositeExpression; + public function __construct( + private string $type, + private array $parts = [], + ) { } /** @@ -45,8 +27,10 @@ class CompositeExpression implements ICompositeExpression, \Countable { * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ - public function addMultiple(array $parts = []) { - $this->compositeExpression->addMultiple($parts); + public function addMultiple(array $parts = []): ICompositeExpression { + foreach ($parts as $part) { + $this->add($part); + } return $this; } @@ -58,8 +42,16 @@ class CompositeExpression implements ICompositeExpression, \Countable { * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ - public function add($part) { - $this->compositeExpression->add($part); + public function add($part): ICompositeExpression { + if ($part === null) { + return $this; + } + + if ($part instanceof self && count($part) === 0) { + return $this; + } + + $this->parts[] = $part; return $this; } @@ -69,8 +61,8 @@ class CompositeExpression implements ICompositeExpression, \Countable { * * @return integer */ - public function count() { - return $this->compositeExpression->count(); + public function count(): int { + return count($this->parts); } /** @@ -78,8 +70,8 @@ class CompositeExpression implements ICompositeExpression, \Countable { * * @return string */ - public function getType() { - return $this->compositeExpression->getType(); + public function getType(): string { + return $this->type; } /** @@ -87,7 +79,14 @@ class CompositeExpression implements ICompositeExpression, \Countable { * * @return string */ - public function __toString() { - return (string) $this->compositeExpression; + public function __toString(): string { + if ($this->count() === 1) { + return (string)$this->parts[0]; + } + return '(' . implode(') ' . $this->type . ' (', $this->parts) . ')'; + } + + public function getParts(): array { + return $this->parts; } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php index d4c1a9db881..b922c861630 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php @@ -1,29 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder\ExpressionBuilder; use Doctrine\DBAL\Query\Expression\ExpressionBuilder as DoctrineExpressionBuilder; @@ -33,12 +14,14 @@ use OC\DB\QueryBuilder\FunctionBuilder\FunctionBuilder; use OC\DB\QueryBuilder\Literal; use OC\DB\QueryBuilder\QueryFunction; use OC\DB\QueryBuilder\QuoteHelper; +use OCP\DB\QueryBuilder\ICompositeExpression; use OCP\DB\QueryBuilder\IExpressionBuilder; use OCP\DB\QueryBuilder\ILiteral; use OCP\DB\QueryBuilder\IParameter; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; use OCP\IDBConnection; +use Psr\Log\LoggerInterface; class ExpressionBuilder implements IExpressionBuilder { /** @var \Doctrine\DBAL\Query\Expression\ExpressionBuilder */ @@ -50,17 +33,15 @@ class ExpressionBuilder implements IExpressionBuilder { /** @var IDBConnection */ protected $connection; + /** @var LoggerInterface */ + protected $logger; + /** @var FunctionBuilder */ protected $functionBuilder; - /** - * Initializes a new <tt>ExpressionBuilder</tt>. - * - * @param ConnectionAdapter $connection - * @param IQueryBuilder $queryBuilder - */ - public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder) { + public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder, LoggerInterface $logger) { $this->connection = $connection; + $this->logger = $logger; $this->helper = new QuoteHelper(); $this->expressionBuilder = new DoctrineExpressionBuilder($connection->getInner()); $this->functionBuilder = $queryBuilder->func(); @@ -76,13 +57,15 @@ class ExpressionBuilder implements IExpressionBuilder { * $expr->andX('u.type = ?', 'u.role = ?')); * * @param mixed ...$x Optional clause. Defaults = null, but requires - * at least one defined when converting to string. + * at least one defined when converting to string. * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ - public function andX(...$x) { - $compositeExpression = call_user_func_array([$this->expressionBuilder, 'andX'], $x); - return new CompositeExpression($compositeExpression); + public function andX(...$x): ICompositeExpression { + if (empty($x)) { + $this->logger->debug('Calling ' . IQueryBuilder::class . '::' . __FUNCTION__ . ' without parameters is deprecated and will throw soon.', ['exception' => new \Exception('No parameters in call to ' . __METHOD__)]); + } + return new CompositeExpression(CompositeExpression::TYPE_AND, $x); } /** @@ -95,13 +78,15 @@ class ExpressionBuilder implements IExpressionBuilder { * $qb->where($qb->expr()->orX('u.type = ?', 'u.role = ?')); * * @param mixed ...$x Optional clause. Defaults = null, but requires - * at least one defined when converting to string. + * at least one defined when converting to string. * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ - public function orX(...$x) { - $compositeExpression = call_user_func_array([$this->expressionBuilder, 'orX'], $x); - return new CompositeExpression($compositeExpression); + public function orX(...$x): ICompositeExpression { + if (empty($x)) { + $this->logger->debug('Calling ' . IQueryBuilder::class . '::' . __FUNCTION__ . ' without parameters is deprecated and will throw soon.', ['exception' => new \Exception('No parameters in call to ' . __METHOD__)]); + } + return new CompositeExpression(CompositeExpression::TYPE_OR, $x); } /** @@ -111,13 +96,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param string $operator One of the IExpressionBuilder::* constants. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function comparison($x, $operator, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function comparison($x, string $operator, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->comparison($x, $operator, $y); } @@ -134,13 +119,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function eq($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function eq($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->eq($x, $y); } @@ -156,13 +141,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function neq($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function neq($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->neq($x, $y); } @@ -178,13 +163,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function lt($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function lt($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->lt($x, $y); } @@ -200,13 +185,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function lte($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function lte($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->lte($x, $y); } @@ -222,13 +207,13 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function gt($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function gt($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->gt($x, $y); } @@ -244,24 +229,24 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function gte($x, $y, $type = null) { - $x = $this->helper->quoteColumnName($x); - $y = $this->helper->quoteColumnName($y); + public function gte($x, $y, $type = null): string { + $x = $this->prepareColumn($x, $type); + $y = $this->prepareColumn($y, $type); return $this->expressionBuilder->gte($x, $y); } /** * Creates an IS NULL expression with the given arguments. * - * @param string $x The field in string format to be restricted by IS NULL. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be restricted by IS NULL. * * @return string */ - public function isNull($x) { + public function isNull($x): string { $x = $this->helper->quoteColumnName($x); return $this->expressionBuilder->isNull($x); } @@ -269,11 +254,11 @@ class ExpressionBuilder implements IExpressionBuilder { /** * Creates an IS NOT NULL expression with the given arguments. * - * @param string $x The field in string format to be restricted by IS NOT NULL. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be restricted by IS NOT NULL. * * @return string */ - public function isNotNull($x) { + public function isNotNull($x): string { $x = $this->helper->quoteColumnName($x); return $this->expressionBuilder->isNotNull($x); } @@ -284,11 +269,11 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x Field in string format to be inspected by LIKE() comparison. * @param mixed $y Argument to be used in LIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function like($x, $y, $type = null) { + public function like($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnName($y); return $this->expressionBuilder->like($x, $y); @@ -300,12 +285,12 @@ class ExpressionBuilder implements IExpressionBuilder { * @param string $x Field in string format to be inspected by ILIKE() comparison. * @param mixed $y Argument to be used in ILIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string * @since 9.0.0 */ - public function iLike($x, $y, $type = null) { + public function iLike($x, $y, $type = null): string { return $this->expressionBuilder->like($this->functionBuilder->lower($x), $this->functionBuilder->lower($y)); } @@ -315,11 +300,11 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x Field in string format to be inspected by NOT LIKE() comparison. * @param mixed $y Argument to be used in NOT LIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function notLike($x, $y, $type = null) { + public function notLike($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnName($y); return $this->expressionBuilder->notLike($x, $y); @@ -331,11 +316,11 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x The field in string format to be inspected by IN() comparison. * @param ILiteral|IParameter|IQueryFunction|string|array $y The placeholder or the array of values to be used by IN() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function in($x, $y, $type = null) { + public function in($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnNames($y); return $this->expressionBuilder->in($x, $y); @@ -347,11 +332,11 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x The field in string format to be inspected by NOT IN() comparison. * @param ILiteral|IParameter|IQueryFunction|string|array $y The placeholder or the array of values to be used by NOT IN() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ - public function notIn($x, $y, $type = null) { + public function notIn($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnNames($y); return $this->expressionBuilder->notIn($x, $y); @@ -360,22 +345,22 @@ class ExpressionBuilder implements IExpressionBuilder { /** * Creates a $x = '' statement, because Oracle needs a different check * - * @param string $x The field in string format to be inspected by the comparison. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be inspected by the comparison. * @return string * @since 13.0.0 */ - public function emptyString($x) { + public function emptyString($x): string { return $this->eq($x, $this->literal('', IQueryBuilder::PARAM_STR)); } /** * Creates a `$x <> ''` statement, because Oracle needs a different check * - * @param string $x The field in string format to be inspected by the comparison. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be inspected by the comparison. * @return string * @since 13.0.0 */ - public function nonEmptyString($x) { + public function nonEmptyString($x): string { return $this->neq($x, $this->literal('', IQueryBuilder::PARAM_STR)); } @@ -387,7 +372,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @return IQueryFunction * @since 12.0.0 */ - public function bitwiseAnd($x, $y) { + public function bitwiseAnd($x, int $y): IQueryFunction { return new QueryFunction($this->connection->getDatabasePlatform()->getBitAndComparisonExpression( $this->helper->quoteColumnName($x), $y @@ -402,7 +387,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @return IQueryFunction * @since 12.0.0 */ - public function bitwiseOr($x, $y) { + public function bitwiseOr($x, int $y): IQueryFunction { return new QueryFunction($this->connection->getDatabasePlatform()->getBitOrComparisonExpression( $this->helper->quoteColumnName($x), $y @@ -413,24 +398,34 @@ class ExpressionBuilder implements IExpressionBuilder { * Quotes a given input parameter. * * @param mixed $input The parameter to be quoted. - * @param mixed|null $type One of the IQueryBuilder::PARAM_* constants + * @param int $type One of the IQueryBuilder::PARAM_* constants * * @return ILiteral */ - public function literal($input, $type = null) { + public function literal($input, $type = IQueryBuilder::PARAM_STR): ILiteral { return new Literal($this->expressionBuilder->literal($input, $type)); } /** * Returns a IQueryFunction that casts the column to the given type * - * @param string $column + * @param string|IQueryFunction $column * @param mixed $type One of IQueryBuilder::PARAM_* - * @return string + * @psalm-param IQueryBuilder::PARAM_* $type + * @return IQueryFunction */ - public function castColumn($column, $type) { + public function castColumn($column, $type): IQueryFunction { return new QueryFunction( $this->helper->quoteColumnName($column) ); } + + /** + * @param mixed $column + * @param mixed|null $type + * @return array|IQueryFunction|string + */ + protected function prepareColumn($column, $type) { + return $this->helper->quoteColumnNames($column); + } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php index 3e4119bb11c..7216fd8807b 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php @@ -1,54 +1,51 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder\ExpressionBuilder; use OC\DB\ConnectionAdapter; +use OC\DB\QueryBuilder\QueryFunction; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\IQueryFunction; +use Psr\Log\LoggerInterface; class MySqlExpressionBuilder extends ExpressionBuilder { + protected string $collation; - /** @var string */ - protected $charset; - - /** - * @param ConnectionAdapter $connection - * @param IQueryBuilder $queryBuilder - */ - public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder) { - parent::__construct($connection, $queryBuilder); + public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder, LoggerInterface $logger) { + parent::__construct($connection, $queryBuilder, $logger); $params = $connection->getInner()->getParams(); - $this->charset = isset($params['charset']) ? $params['charset'] : 'utf8'; + $this->collation = $params['collation'] ?? (($params['charset'] ?? 'utf8') . '_general_ci'); } /** * @inheritdoc */ - public function iLike($x, $y, $type = null) { + public function iLike($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnName($y); - return $this->expressionBuilder->comparison($x, ' COLLATE ' . $this->charset . '_general_ci LIKE', $y); + return $this->expressionBuilder->comparison($x, ' COLLATE ' . $this->collation . ' LIKE', $y); + } + + /** + * Returns a IQueryFunction that casts the column to the given type + * + * @param string|IQueryFunction $column + * @param mixed $type One of IQueryBuilder::PARAM_* + * @psalm-param IQueryBuilder::PARAM_* $type + * @return IQueryFunction + */ + public function castColumn($column, $type): IQueryFunction { + switch ($type) { + case IQueryBuilder::PARAM_STR: + return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS CHAR)'); + default: + return parent::castColumn($column, $type); + } } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php index f41242fdc60..542e8d62ede 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php @@ -1,27 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder\ExpressionBuilder; use OC\DB\QueryBuilder\QueryFunction; @@ -31,7 +14,6 @@ use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; class OCIExpressionBuilder extends ExpressionBuilder { - /** * @param mixed $column * @param mixed|null $type @@ -40,86 +22,15 @@ class OCIExpressionBuilder extends ExpressionBuilder { protected function prepareColumn($column, $type) { if ($type === IQueryBuilder::PARAM_STR && !is_array($column) && !($column instanceof IParameter) && !($column instanceof ILiteral)) { $column = $this->castColumn($column, $type); - } else { - $column = $this->helper->quoteColumnNames($column); } - return $column; - } - - /** - * @inheritdoc - */ - public function comparison($x, $operator, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->comparison($x, $operator, $y); - } - - /** - * @inheritdoc - */ - public function eq($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->eq($x, $y); - } - - /** - * @inheritdoc - */ - public function neq($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->neq($x, $y); - } - - /** - * @inheritdoc - */ - public function lt($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->lt($x, $y); - } - - /** - * @inheritdoc - */ - public function lte($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->lte($x, $y); - } - - /** - * @inheritdoc - */ - public function gt($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - - return $this->expressionBuilder->gt($x, $y); - } - - /** - * @inheritdoc - */ - public function gte($x, $y, $type = null) { - $x = $this->prepareColumn($x, $type); - $y = $this->prepareColumn($y, $type); - return $this->expressionBuilder->gte($x, $y); + return parent::prepareColumn($column, $type); } /** * @inheritdoc */ - public function in($x, $y, $type = null) { + public function in($x, $y, $type = null): string { $x = $this->prepareColumn($x, $type); $y = $this->prepareColumn($y, $type); @@ -129,7 +40,7 @@ class OCIExpressionBuilder extends ExpressionBuilder { /** * @inheritdoc */ - public function notIn($x, $y, $type = null) { + public function notIn($x, $y, $type = null): string { $x = $this->prepareColumn($x, $type); $y = $this->prepareColumn($y, $type); @@ -139,33 +50,34 @@ class OCIExpressionBuilder extends ExpressionBuilder { /** * Creates a $x = '' statement, because Oracle needs a different check * - * @param string $x The field in string format to be inspected by the comparison. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be inspected by the comparison. * @return string * @since 13.0.0 */ - public function emptyString($x) { + public function emptyString($x): string { return $this->isNull($x); } /** * Creates a `$x <> ''` statement, because Oracle needs a different check * - * @param string $x The field in string format to be inspected by the comparison. + * @param string|ILiteral|IParameter|IQueryFunction $x The field in string format to be inspected by the comparison. * @return string * @since 13.0.0 */ - public function nonEmptyString($x) { + public function nonEmptyString($x): string { return $this->isNotNull($x); } /** * Returns a IQueryFunction that casts the column to the given type * - * @param string $column + * @param string|IQueryFunction $column * @param mixed $type One of IQueryBuilder::PARAM_* + * @psalm-param IQueryBuilder::PARAM_* $type * @return IQueryFunction */ - public function castColumn($column, $type) { + public function castColumn($column, $type): IQueryFunction { if ($type === IQueryBuilder::PARAM_STR) { $column = $this->helper->quoteColumnName($column); return new QueryFunction('to_char(' . $column . ')'); @@ -181,14 +93,14 @@ class OCIExpressionBuilder extends ExpressionBuilder { /** * @inheritdoc */ - public function like($x, $y, $type = null) { + public function like($x, $y, $type = null): string { return parent::like($x, $y, $type) . " ESCAPE '\\'"; } /** * @inheritdoc */ - public function iLike($x, $y, $type = null) { + public function iLike($x, $y, $type = null): string { return $this->like($this->functionBuilder->lower($x), $this->functionBuilder->lower($y)); } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php index 141a93ff75a..53a566a7eb6 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php @@ -1,45 +1,29 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder\ExpressionBuilder; use OC\DB\QueryBuilder\QueryFunction; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\IQueryFunction; class PgSqlExpressionBuilder extends ExpressionBuilder { - /** * Returns a IQueryFunction that casts the column to the given type * - * @param string $column + * @param string|IQueryFunction $column * @param mixed $type One of IQueryBuilder::PARAM_* - * @return string + * @psalm-param IQueryBuilder::PARAM_* $type + * @return IQueryFunction */ - public function castColumn($column, $type) { + public function castColumn($column, $type): IQueryFunction { switch ($type) { case IQueryBuilder::PARAM_INT: - return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS INT)'); + return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS BIGINT)'); case IQueryBuilder::PARAM_STR: return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS TEXT)'); default: @@ -50,7 +34,7 @@ class PgSqlExpressionBuilder extends ExpressionBuilder { /** * @inheritdoc */ - public function iLike($x, $y, $type = null) { + public function iLike($x, $y, $type = null): string { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnName($y); return $this->expressionBuilder->comparison($x, 'ILIKE', $y); diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php index 1fa0d79663a..52f82db2232 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php @@ -1,37 +1,71 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\QueryBuilder\ExpressionBuilder; +use OC\DB\QueryBuilder\QueryFunction; +use OCP\DB\QueryBuilder\ILiteral; +use OCP\DB\QueryBuilder\IParameter; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\IQueryFunction; + class SqliteExpressionBuilder extends ExpressionBuilder { /** * @inheritdoc */ - public function like($x, $y, $type = null) { + public function like($x, $y, $type = null): string { return parent::like($x, $y, $type) . " ESCAPE '\\'"; } - public function iLike($x, $y, $type = null) { + public function iLike($x, $y, $type = null): string { return $this->like($this->functionBuilder->lower($x), $this->functionBuilder->lower($y), $type); } + + /** + * @param mixed $column + * @param mixed|null $type + * @return array|IQueryFunction|string + */ + protected function prepareColumn($column, $type) { + if ($type !== null + && !is_array($column) + && !($column instanceof IParameter) + && !($column instanceof ILiteral) + && (str_starts_with($type, 'date') || str_starts_with($type, 'time'))) { + return $this->castColumn($column, $type); + } + + return parent::prepareColumn($column, $type); + } + + /** + * Returns a IQueryFunction that casts the column to the given type + * + * @param string $column + * @param mixed $type One of IQueryBuilder::PARAM_* + * @return IQueryFunction + */ + public function castColumn($column, $type): IQueryFunction { + switch ($type) { + case IQueryBuilder::PARAM_DATE_MUTABLE: + case IQueryBuilder::PARAM_DATE_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('DATE(' . $column . ')'); + case IQueryBuilder::PARAM_DATETIME_MUTABLE: + case IQueryBuilder::PARAM_DATETIME_IMMUTABLE: + case IQueryBuilder::PARAM_DATETIME_TZ_MUTABLE: + case IQueryBuilder::PARAM_DATETIME_TZ_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('DATETIME(' . $column . ')'); + case IQueryBuilder::PARAM_TIME_MUTABLE: + case IQueryBuilder::PARAM_TIME_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('TIME(' . $column . ')'); + } + + return parent::castColumn($column, $type); + } } diff --git a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php new file mode 100644 index 00000000000..f928132a123 --- /dev/null +++ b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php @@ -0,0 +1,309 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder; + +use OC\DB\Exceptions\DbalException; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Base class for creating classes that extend the builtin query builder + */ +abstract class ExtendedQueryBuilder implements IQueryBuilder { + public function __construct( + protected IQueryBuilder $builder, + ) { + } + + public function automaticTablePrefix($enabled) { + $this->builder->automaticTablePrefix($enabled); + return $this; + } + + public function expr() { + return $this->builder->expr(); + } + + public function func() { + return $this->builder->func(); + } + + public function getType() { + return $this->builder->getType(); + } + + public function getConnection() { + return $this->builder->getConnection(); + } + + public function getState() { + return $this->builder->getState(); + } + + public function execute(?IDBConnection $connection = null) { + try { + if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + return $this->executeQuery($connection); + } else { + return $this->executeStatement($connection); + } + } catch (DBALException $e) { + // `IQueryBuilder->execute` never wrapped the exception, but `executeQuery` and `executeStatement` do + /** @var \Doctrine\DBAL\Exception $previous */ + $previous = $e->getPrevious(); + throw $previous; + } + } + + public function getSQL() { + return $this->builder->getSQL(); + } + + public function setParameter($key, $value, $type = null) { + $this->builder->setParameter($key, $value, $type); + return $this; + } + + public function setParameters(array $params, array $types = []) { + $this->builder->setParameters($params, $types); + return $this; + } + + public function getParameters() { + return $this->builder->getParameters(); + } + + public function getParameter($key) { + return $this->builder->getParameter($key); + } + + public function getParameterTypes() { + return $this->builder->getParameterTypes(); + } + + public function getParameterType($key) { + return $this->builder->getParameterType($key); + } + + public function setFirstResult($firstResult) { + $this->builder->setFirstResult($firstResult); + return $this; + } + + public function getFirstResult() { + return $this->builder->getFirstResult(); + } + + public function setMaxResults($maxResults) { + $this->builder->setMaxResults($maxResults); + return $this; + } + + public function getMaxResults() { + return $this->builder->getMaxResults(); + } + + public function select(...$selects) { + $this->builder->select(...$selects); + return $this; + } + + public function selectAlias($select, $alias) { + $this->builder->selectAlias($select, $alias); + return $this; + } + + public function selectDistinct($select) { + $this->builder->selectDistinct($select); + return $this; + } + + public function addSelect(...$select) { + $this->builder->addSelect(...$select); + return $this; + } + + public function delete($delete = null, $alias = null) { + $this->builder->delete($delete, $alias); + return $this; + } + + public function update($update = null, $alias = null) { + $this->builder->update($update, $alias); + return $this; + } + + public function insert($insert = null) { + $this->builder->insert($insert); + return $this; + } + + public function from($from, $alias = null) { + $this->builder->from($from, $alias); + return $this; + } + + public function join($fromAlias, $join, $alias, $condition = null) { + $this->builder->join($fromAlias, $join, $alias, $condition); + return $this; + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->innerJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->leftJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function rightJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->rightJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function set($key, $value) { + $this->builder->set($key, $value); + return $this; + } + + public function where(...$predicates) { + $this->builder->where(...$predicates); + return $this; + } + + public function andWhere(...$where) { + $this->builder->andWhere(...$where); + return $this; + } + + public function orWhere(...$where) { + $this->builder->orWhere(...$where); + return $this; + } + + public function groupBy(...$groupBys) { + $this->builder->groupBy(...$groupBys); + return $this; + } + + public function addGroupBy(...$groupBy) { + $this->builder->addGroupBy(...$groupBy); + return $this; + } + + public function setValue($column, $value) { + $this->builder->setValue($column, $value); + return $this; + } + + public function values(array $values) { + $this->builder->values($values); + return $this; + } + + public function having(...$having) { + $this->builder->having(...$having); + return $this; + } + + public function andHaving(...$having) { + $this->builder->andHaving(...$having); + return $this; + } + + public function orHaving(...$having) { + $this->builder->orHaving(...$having); + return $this; + } + + public function orderBy($sort, $order = null) { + $this->builder->orderBy($sort, $order); + return $this; + } + + public function addOrderBy($sort, $order = null) { + $this->builder->addOrderBy($sort, $order); + return $this; + } + + public function getQueryPart($queryPartName) { + return $this->builder->getQueryPart($queryPartName); + } + + public function getQueryParts() { + return $this->builder->getQueryParts(); + } + + public function resetQueryParts($queryPartNames = null) { + $this->builder->resetQueryParts($queryPartNames); + return $this; + } + + public function resetQueryPart($queryPartName) { + $this->builder->resetQueryPart($queryPartName); + return $this; + } + + public function createNamedParameter($value, $type = self::PARAM_STR, $placeHolder = null) { + return $this->builder->createNamedParameter($value, $type, $placeHolder); + } + + public function createPositionalParameter($value, $type = self::PARAM_STR) { + return $this->builder->createPositionalParameter($value, $type); + } + + public function createParameter($name) { + return $this->builder->createParameter($name); + } + + public function createFunction($call) { + return $this->builder->createFunction($call); + } + + public function getLastInsertId(): int { + return $this->builder->getLastInsertId(); + } + + public function getTableName($table) { + return $this->builder->getTableName($table); + } + + public function getColumnName($column, $tableAlias = '') { + return $this->builder->getColumnName($column, $tableAlias); + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + return $this->builder->executeQuery($connection); + } + + public function executeStatement(?IDBConnection $connection = null): int { + return $this->builder->executeStatement($connection); + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + $this->builder->hintShardKey($column, $value, $overwrite); + return $this; + } + + public function runAcrossAllShards(): self { + $this->builder->runAcrossAllShards(); + return $this; + } + + public function getOutputColumns(): array { + return $this->builder->getOutputColumns(); + } + + public function prefixTableName(string $table): string { + return $this->builder->prefixTableName($table); + } +} diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php index b8c54546b78..48dc1da6330 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php @@ -1,44 +1,32 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\QueryBuilder\FunctionBuilder; +use OC\DB\Connection; use OC\DB\QueryBuilder\QueryFunction; use OC\DB\QueryBuilder\QuoteHelper; use OCP\DB\QueryBuilder\IFunctionBuilder; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; +use OCP\IDBConnection; class FunctionBuilder implements IFunctionBuilder { + /** @var IDBConnection|Connection */ + protected $connection; + + /** @var IQueryBuilder */ + protected $queryBuilder; + /** @var QuoteHelper */ protected $helper; - /** - * ExpressionBuilder constructor. - * - * @param QuoteHelper $helper - */ - public function __construct(QuoteHelper $helper) { + public function __construct(IDBConnection $connection, IQueryBuilder $queryBuilder, QuoteHelper $helper) { + $this->connection = $connection; + $this->queryBuilder = $queryBuilder; $this->helper = $helper; } @@ -46,8 +34,18 @@ class FunctionBuilder implements IFunctionBuilder { return new QueryFunction('MD5(' . $this->helper->quoteColumnName($input) . ')'); } - public function concat($x, $y): IQueryFunction { - return new QueryFunction('CONCAT(' . $this->helper->quoteColumnName($x) . ', ' . $this->helper->quoteColumnName($y) . ')'); + public function concat($x, ...$expr): IQueryFunction { + $args = func_get_args(); + $list = []; + foreach ($args as $item) { + $list[] = $this->helper->quoteColumnName($item); + } + return new QueryFunction(sprintf('CONCAT(%s)', implode(', ', $list))); + } + + public function groupConcat($expr, ?string $separator = ','): IQueryFunction { + $separator = $this->connection->quote($separator); + return new QueryFunction('GROUP_CONCAT(' . $this->helper->quoteColumnName($expr) . ' SEPARATOR ' . $separator . ')'); } public function substring($input, $start, $length = null): IQueryFunction { @@ -80,6 +78,18 @@ class FunctionBuilder implements IFunctionBuilder { return new QueryFunction('COUNT(' . $quotedName . ')' . $alias); } + public function octetLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('OCTET_LENGTH(' . $quotedName . ')' . $alias); + } + + public function charLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('CHAR_LENGTH(' . $quotedName . ')' . $alias); + } + public function max($field): IQueryFunction { return new QueryFunction('MAX(' . $this->helper->quoteColumnName($field) . ')'); } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php index c269fc03c8a..47a8eaa6fd0 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php @@ -1,27 +1,9 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\QueryBuilder\FunctionBuilder; use OC\DB\QueryBuilder\QueryFunction; @@ -31,7 +13,10 @@ use OCP\DB\QueryBuilder\IQueryFunction; class OCIFunctionBuilder extends FunctionBuilder { public function md5($input): IQueryFunction { - return new QueryFunction('LOWER(DBMS_OBFUSCATION_TOOLKIT.md5 (input => UTL_RAW.cast_to_raw(' . $this->helper->quoteColumnName($input) .')))'); + if (version_compare($this->connection->getServerVersion(), '20', '>=')) { + return new QueryFunction('LOWER(STANDARD_HASH(' . $this->helper->quoteColumnName($input) . ", 'MD5'))"); + } + return new QueryFunction('LOWER(DBMS_OBFUSCATION_TOOLKIT.md5 (input => UTL_RAW.cast_to_raw(' . $this->helper->quoteColumnName($input) . ')))'); } /** @@ -73,4 +58,35 @@ class OCIFunctionBuilder extends FunctionBuilder { return parent::least($x, $y); } + + public function concat($x, ...$expr): IQueryFunction { + $args = func_get_args(); + $list = []; + foreach ($args as $item) { + $list[] = $this->helper->quoteColumnName($item); + } + return new QueryFunction(sprintf('(%s)', implode(' || ', $list))); + } + + public function groupConcat($expr, ?string $separator = ','): IQueryFunction { + $orderByClause = ' WITHIN GROUP(ORDER BY NULL)'; + if (is_null($separator)) { + return new QueryFunction('LISTAGG(' . $this->helper->quoteColumnName($expr) . ')' . $orderByClause); + } + + $separator = $this->connection->quote($separator); + return new QueryFunction('LISTAGG(' . $this->helper->quoteColumnName($expr) . ', ' . $separator . ')' . $orderByClause); + } + + public function octetLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias); + } + + public function charLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias); + } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php index a44b80bbaaf..354a2b126d7 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -1,34 +1,33 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\QueryBuilder\FunctionBuilder; use OC\DB\QueryBuilder\QueryFunction; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; class PgSqlFunctionBuilder extends FunctionBuilder { - public function concat($x, $y): IQueryFunction { - return new QueryFunction('(' . $this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y) . ')'); + public function concat($x, ...$expr): IQueryFunction { + $args = func_get_args(); + $list = []; + foreach ($args as $item) { + $list[] = $this->queryBuilder->expr()->castColumn($item, IQueryBuilder::PARAM_STR); + } + return new QueryFunction(sprintf('(%s)', implode(' || ', $list))); + } + + public function groupConcat($expr, ?string $separator = ','): IQueryFunction { + $castedExpression = $this->queryBuilder->expr()->castColumn($expr, IQueryBuilder::PARAM_STR); + + if (is_null($separator)) { + return new QueryFunction('string_agg(' . $castedExpression . ')'); + } + + $separator = $this->connection->quote($separator); + return new QueryFunction('string_agg(' . $castedExpression . ', ' . $separator . ')'); } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php index 8e490c15a3c..53aa530054b 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -1,35 +1,27 @@ <?php + /** - * @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl> - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB\QueryBuilder\FunctionBuilder; use OC\DB\QueryBuilder\QueryFunction; use OCP\DB\QueryBuilder\IQueryFunction; class SqliteFunctionBuilder extends FunctionBuilder { - public function concat($x, $y): IQueryFunction { - return new QueryFunction('(' . $this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y) . ')'); + public function concat($x, ...$expr): IQueryFunction { + $args = func_get_args(); + $list = []; + foreach ($args as $item) { + $list[] = $this->helper->quoteColumnName($item); + } + return new QueryFunction(sprintf('(%s)', implode(' || ', $list))); + } + + public function groupConcat($expr, ?string $separator = ','): IQueryFunction { + $separator = $this->connection->quote($separator); + return new QueryFunction('GROUP_CONCAT(' . $this->helper->quoteColumnName($expr) . ', ' . $separator . ')'); } public function greatest($x, $y): IQueryFunction { @@ -39,4 +31,16 @@ class SqliteFunctionBuilder extends FunctionBuilder { public function least($x, $y): IQueryFunction { return new QueryFunction('MIN(' . $this->helper->quoteColumnName($x) . ', ' . $this->helper->quoteColumnName($y) . ')'); } + + public function octetLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('LENGTH(CAST(' . $quotedName . ' as BLOB))' . $alias); + } + + public function charLength($field, $alias = ''): IQueryFunction { + $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; + $quotedName = $this->helper->quoteColumnName($field); + return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias); + } } diff --git a/lib/private/DB/QueryBuilder/Literal.php b/lib/private/DB/QueryBuilder/Literal.php index 02fd232ff0b..3fb897328e5 100644 --- a/lib/private/DB/QueryBuilder/Literal.php +++ b/lib/private/DB/QueryBuilder/Literal.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\ILiteral; @@ -33,10 +17,7 @@ class Literal implements ILiteral { $this->literal = $literal; } - /** - * @return string - */ - public function __toString() { - return (string) $this->literal; + public function __toString(): string { + return (string)$this->literal; } } diff --git a/lib/private/DB/QueryBuilder/Parameter.php b/lib/private/DB/QueryBuilder/Parameter.php index b64b0cf5fcc..a272c744d62 100644 --- a/lib/private/DB/QueryBuilder/Parameter.php +++ b/lib/private/DB/QueryBuilder/Parameter.php @@ -1,25 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\IParameter; @@ -32,10 +17,7 @@ class Parameter implements IParameter { $this->name = $name; } - /** - * @return string - */ - public function __toString() { - return (string) $this->name; + public function __toString(): string { + return (string)$this->name; } } diff --git a/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php b/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php new file mode 100644 index 00000000000..3a5aa2f3e0e --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php @@ -0,0 +1,79 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +/** + * Partitioned queries impose limitations that queries have to follow: + * + * 1. Any reference to columns not in the "main table" (the table referenced by "FROM"), needs to explicitly include the + * table or alias the column belongs to. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("root_id", "fileid")); + * ``` + * will not work, as the query builder doesn't know that the `mimetype` column belongs to the "filecache partition". + * Instead, you need to do + * ``` + * $query->select("mount_point", "f.mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid")); + * ``` + * + * 2. The "ON" condition for the join can only perform a comparison between both sides of the join once. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->andX($query->expr()->eq("m.root_id", "f.fileid"), $query->expr()->eq("m.storage_id", "f.storage"))); + * ``` + * will not work. + * + * 3. An "OR" expression in the "WHERE" cannot mention both sides of the join, this does not apply to "AND" expressions. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid"))) + * ->where($query->expr()->orX( + * $query->expr()-eq("m.user_id", $query->createNamedParameter("test"))), + * $query->expr()-eq("f.name", $query->createNamedParameter("test"))), + * )); + * ``` + * will not work, but. + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid"))) + * ->where($query->expr()->andX( + * $query->expr()-eq("m.user_id", $query->createNamedParameter("test"))), + * $query->expr()-eq("f.name", $query->createNamedParameter("test"))), + * )); + * ``` + * will. + * + * 4. Queries that join cross-partition cannot use position parameters, only named parameters are allowed + * 5. The "ON" condition of a join cannot contain and "OR" expression. + * 6. Right-joins are not allowed. + * 7. Update, delete and insert statements aren't allowed to contain cross-partition joins. + * 8. Queries that "GROUP BY" a column from the joined partition are not allowed. + * 9. Any `join` call needs to be made before any `where` call. + * 10. Queries that join cross-partition with an "INNER JOIN" or "LEFT JOIN" with a condition on the left side + * cannot use "LIMIT" or "OFFSET" in queries. + * + * The part of the query running on the sharded table has some additional limitations, + * see the `InvalidShardedQueryException` documentation for more information. + */ +class InvalidPartitionedQueryException extends \Exception { + +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php b/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php new file mode 100644 index 00000000000..a08858d1d6b --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php @@ -0,0 +1,173 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\QueryFunction; +use OCP\DB\QueryBuilder\IQueryFunction; + +/** + * Utility class for working with join conditions + */ +class JoinCondition { + public function __construct( + public string|IQueryFunction $fromColumn, + public ?string $fromAlias, + public string|IQueryFunction $toColumn, + public ?string $toAlias, + public array $fromConditions, + public array $toConditions, + ) { + if (is_string($this->fromColumn) && str_starts_with($this->fromColumn, '(')) { + $this->fromColumn = new QueryFunction($this->fromColumn); + } + if (is_string($this->toColumn) && str_starts_with($this->toColumn, '(')) { + $this->toColumn = new QueryFunction($this->toColumn); + } + } + + /** + * @param JoinCondition[] $conditions + * @return JoinCondition + */ + public static function merge(array $conditions): JoinCondition { + $fromColumn = ''; + $toColumn = ''; + $fromAlias = null; + $toAlias = null; + $fromConditions = []; + $toConditions = []; + foreach ($conditions as $condition) { + if (($condition->fromColumn && $fromColumn) || ($condition->toColumn && $toColumn)) { + throw new InvalidPartitionedQueryException("Can't join from {$condition->fromColumn} to {$condition->toColumn} as it already join froms {$fromColumn} to {$toColumn}"); + } + if ($condition->fromColumn) { + $fromColumn = $condition->fromColumn; + } + if ($condition->toColumn) { + $toColumn = $condition->toColumn; + } + if ($condition->fromAlias) { + $fromAlias = $condition->fromAlias; + } + if ($condition->toAlias) { + $toAlias = $condition->toAlias; + } + $fromConditions = array_merge($fromConditions, $condition->fromConditions); + $toConditions = array_merge($toConditions, $condition->toConditions); + } + return new JoinCondition($fromColumn, $fromAlias, $toColumn, $toAlias, $fromConditions, $toConditions); + } + + /** + * @param null|string|CompositeExpression $condition + * @param string $join + * @param string $alias + * @param string $fromAlias + * @return JoinCondition + * @throws InvalidPartitionedQueryException + */ + public static function parse($condition, string $join, string $alias, string $fromAlias): JoinCondition { + if ($condition === null) { + throw new InvalidPartitionedQueryException("Can't join on $join without a condition"); + } + + $result = self::parseSubCondition($condition, $join, $alias, $fromAlias); + if (!$result->fromColumn || !$result->toColumn) { + throw new InvalidPartitionedQueryException("No join condition found from $fromAlias to $alias"); + } + return $result; + } + + private static function parseSubCondition($condition, string $join, string $alias, string $fromAlias): JoinCondition { + if ($condition instanceof CompositeExpression) { + if ($condition->getType() === CompositeExpression::TYPE_OR) { + throw new InvalidPartitionedQueryException("Cannot join on $join with an OR expression"); + } + return self::merge(array_map(function ($subCondition) use ($join, $alias, $fromAlias) { + return self::parseSubCondition($subCondition, $join, $alias, $fromAlias); + }, $condition->getParts())); + } + + $condition = (string)$condition; + $isSubCondition = self::isExtraCondition($condition); + if ($isSubCondition) { + if (self::mentionsAlias($condition, $fromAlias)) { + return new JoinCondition('', null, '', null, [$condition], []); + } else { + return new JoinCondition('', null, '', null, [], [$condition]); + } + } + + $condition = str_replace('`', '', $condition); + + // expect a condition in the form of 'alias1.column1 = alias2.column2' + if (!str_contains($condition, ' = ')) { + throw new InvalidPartitionedQueryException("Can only join on $join with an `eq` condition"); + } + $parts = explode(' = ', $condition, 2); + $parts = array_map(function (string $part) { + return self::clearConditionPart($part); + }, $parts); + + if (!self::isSingleCondition($parts[0]) || !self::isSingleCondition($parts[1])) { + throw new InvalidPartitionedQueryException("Can only join on $join with a single condition"); + } + + + if (self::mentionsAlias($parts[0], $fromAlias)) { + return new JoinCondition($parts[0], self::getAliasForPart($parts[0]), $parts[1], self::getAliasForPart($parts[1]), [], []); + } elseif (self::mentionsAlias($parts[1], $fromAlias)) { + return new JoinCondition($parts[1], self::getAliasForPart($parts[1]), $parts[0], self::getAliasForPart($parts[0]), [], []); + } else { + throw new InvalidPartitionedQueryException("join condition for $join needs to explicitly refer to the table by alias"); + } + } + + private static function isSingleCondition(string $condition): bool { + return !(str_contains($condition, ' OR ') || str_contains($condition, ' AND ')); + } + + private static function getAliasForPart(string $part): ?string { + if (str_contains($part, ' ')) { + return uniqid('join_alias_'); + } else { + return null; + } + } + + private static function clearConditionPart(string $part): string { + if (str_starts_with($part, 'CAST(')) { + // pgsql/mysql cast + $part = substr($part, strlen('CAST(')); + [$part] = explode(' AS ', $part); + } elseif (str_starts_with($part, 'to_number(to_char(')) { + // oracle cast to int + $part = substr($part, strlen('to_number(to_char('), -2); + } elseif (str_starts_with($part, 'to_number(to_char(')) { + // oracle cast to string + $part = substr($part, strlen('to_char('), -1); + } + return $part; + } + + /** + * Check that a condition is an extra limit on the from/to part, and not the join condition + * + * This is done by checking that only one of the halves of the condition references a column + */ + private static function isExtraCondition(string $condition): bool { + $parts = explode(' ', $condition, 2); + return str_contains($parts[0], '`') xor str_contains($parts[1], '`'); + } + + private static function mentionsAlias(string $condition, string $alias): bool { + return str_contains($condition, "$alias."); + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php new file mode 100644 index 00000000000..a5024b478d3 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php @@ -0,0 +1,75 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OCP\DB\QueryBuilder\IQueryBuilder; + +/** + * A sub-query from a partitioned join + */ +class PartitionQuery { + public const JOIN_MODE_INNER = 'inner'; + public const JOIN_MODE_LEFT = 'left'; + // left-join where the left side IS NULL + public const JOIN_MODE_LEFT_NULL = 'left_null'; + + public const JOIN_MODE_RIGHT = 'right'; + + public function __construct( + public IQueryBuilder $query, + public string $joinFromColumn, + public string $joinToColumn, + public string $joinMode, + ) { + if ($joinMode !== self::JOIN_MODE_LEFT && $joinMode !== self::JOIN_MODE_INNER) { + throw new InvalidPartitionedQueryException("$joinMode joins aren't allowed in partitioned queries"); + } + } + + public function mergeWith(array $rows): array { + if (empty($rows)) { + return []; + } + // strip table/alias from column names + $joinFromColumn = preg_replace('/\w+\./', '', $this->joinFromColumn); + $joinToColumn = preg_replace('/\w+\./', '', $this->joinToColumn); + + $joinFromValues = array_map(function (array $row) use ($joinFromColumn) { + return $row[$joinFromColumn]; + }, $rows); + $joinFromValues = array_filter($joinFromValues, function ($value) { + return $value !== null; + }); + $this->query->andWhere($this->query->expr()->in($this->joinToColumn, $this->query->createNamedParameter($joinFromValues, IQueryBuilder::PARAM_STR_ARRAY, ':' . uniqid()))); + + $s = $this->query->getSQL(); + $partitionedRows = $this->query->executeQuery()->fetchAll(); + + $columns = $this->query->getOutputColumns(); + $nullResult = array_combine($columns, array_fill(0, count($columns), null)); + + $partitionedRowsByKey = []; + foreach ($partitionedRows as $partitionedRow) { + $partitionedRowsByKey[$partitionedRow[$joinToColumn]][] = $partitionedRow; + } + $result = []; + foreach ($rows as $row) { + if (isset($partitionedRowsByKey[$row[$joinFromColumn]])) { + if ($this->joinMode !== self::JOIN_MODE_LEFT_NULL) { + foreach ($partitionedRowsByKey[$row[$joinFromColumn]] as $partitionedRow) { + $result[] = array_merge($row, $partitionedRow); + } + } + } elseif ($this->joinMode === self::JOIN_MODE_LEFT || $this->joinMode === self::JOIN_MODE_LEFT_NULL) { + $result[] = array_merge($nullResult, $row); + } + } + return $result; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php new file mode 100644 index 00000000000..ad4c0fab055 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php @@ -0,0 +1,74 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +/** + * Information about a database partition, containing the tables in the partition and any active alias + */ +class PartitionSplit { + /** @var array<string, string> */ + public array $aliases = []; + + /** + * @param string[] $tables + */ + public function __construct( + public string $name, + public array $tables, + ) { + } + + public function addAlias(string $table, string $alias): void { + if ($this->containsTable($table)) { + $this->aliases[$alias] = $table; + } + } + + public function addTable(string $table): void { + if (!$this->containsTable($table)) { + $this->tables[] = $table; + } + } + + public function containsTable(string $table): bool { + return in_array($table, $this->tables); + } + + public function containsAlias(string $alias): bool { + return array_key_exists($alias, $this->aliases); + } + + private function getTablesAndAliases(): array { + return array_keys($this->aliases) + $this->tables; + } + + /** + * Check if a query predicate mentions a table or alias from this partition + * + * @param string $predicate + * @return bool + */ + public function checkPredicateForTable(string $predicate): bool { + foreach ($this->getTablesAndAliases() as $name) { + if (str_contains($predicate, "`$name`.`")) { + return true; + } + } + return false; + } + + public function isColumnInPartition(string $column): bool { + foreach ($this->getTablesAndAliases() as $name) { + if (str_starts_with($column, "$name.")) { + return true; + } + } + return false; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php new file mode 100644 index 00000000000..d748c791321 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php @@ -0,0 +1,462 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\QuoteHelper; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; +use OC\DB\QueryBuilder\Sharded\ShardedQueryBuilder; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\IQueryFunction; +use OCP\IDBConnection; + +/** + * A special query builder that automatically splits queries that span across multiple database partitions[1]. + * + * This is done by inspecting the query as it's being built, and when a cross-partition join is detected, + * the part of the query that touches the partition is split of into a different sub-query. + * Then, when the query is executed, the results from the sub-queries are automatically merged. + * + * This whole process is intended to be transparent to any code using the query builder, however it does impose some extra + * limitation for queries that work cross-partition. See the documentation from `InvalidPartitionedQueryException` for more details. + * + * When a join is created in the query, this builder checks if it belongs to the same partition as the table from the + * original FROM/UPDATE/DELETE/INSERT and if not, creates a new "sub query" for the partition. + * Then for every part that is added the query, the part is analyzed to determine which partition the query part is referencing + * and the query part is added to the sub query for that partition. + * + * [1]: A set of tables which can't be queried together with the rest of the tables, such as when sharding is used. + */ +class PartitionedQueryBuilder extends ShardedQueryBuilder { + /** @var array<string, PartitionQuery> $splitQueries */ + private array $splitQueries = []; + /** @var list<PartitionSplit> */ + private array $partitions = []; + + /** @var array{'select': string|array, 'alias': ?string}[] */ + private array $selects = []; + private ?PartitionSplit $mainPartition = null; + private bool $hasPositionalParameter = false; + private QuoteHelper $quoteHelper; + private ?int $limit = null; + private ?int $offset = null; + + public function __construct( + IQueryBuilder $builder, + array $shardDefinitions, + ShardConnectionManager $shardConnectionManager, + AutoIncrementHandler $autoIncrementHandler, + ) { + parent::__construct($builder, $shardDefinitions, $shardConnectionManager, $autoIncrementHandler); + $this->quoteHelper = new QuoteHelper(); + } + + private function newQuery(): IQueryBuilder { + // get a fresh, non-partitioning query builder + $builder = $this->builder->getConnection()->getQueryBuilder(); + if ($builder instanceof PartitionedQueryBuilder) { + $builder = $builder->builder; + } + + return new ShardedQueryBuilder( + $builder, + $this->shardDefinitions, + $this->shardConnectionManager, + $this->autoIncrementHandler, + ); + } + + // we need to save selects until we know all the table aliases + public function select(...$selects) { + if (count($selects) === 1 && is_array($selects[0])) { + $selects = $selects[0]; + } + $this->selects = []; + $this->addSelect(...$selects); + return $this; + } + + public function addSelect(...$select) { + $select = array_map(function ($select) { + return ['select' => $select, 'alias' => null]; + }, $select); + $this->selects = array_merge($this->selects, $select); + return $this; + } + + public function selectAlias($select, $alias) { + $this->selects[] = ['select' => $select, 'alias' => $alias]; + return $this; + } + + /** + * Ensure that a column is being selected by the query + * + * This is mainly used to ensure that the returned rows from both sides of a partition contains the columns of the join predicate + * + * @param string|IQueryFunction $column + * @return void + */ + private function ensureSelect(string|IQueryFunction $column, ?string $alias = null): void { + $checkColumn = $alias ?: $column; + if (str_contains($checkColumn, '.')) { + [$table, $checkColumn] = explode('.', $checkColumn); + $partition = $this->getPartition($table); + } else { + $partition = null; + } + foreach ($this->selects as $select) { + $select = $select['select']; + if (!is_string($select)) { + continue; + } + + if (str_contains($select, '.')) { + [$table, $select] = explode('.', $select); + $selectPartition = $this->getPartition($table); + } else { + $selectPartition = null; + } + if ( + ($select === $checkColumn || $select === '*') + && $selectPartition === $partition + ) { + return; + } + } + if ($alias) { + $this->selectAlias($column, $alias); + } else { + $this->addSelect($column); + } + } + + /** + * Distribute the select statements to the correct partition + * + * This is done at the end instead of when the `select` call is made, because the `select` calls are generally done + * before we know what tables are involved in the query + * + * @return void + */ + private function applySelects(): void { + foreach ($this->selects as $select) { + foreach ($this->partitions as $partition) { + if (is_string($select['select']) && ( + $select['select'] === '*' + || $partition->isColumnInPartition($select['select'])) + ) { + if (isset($this->splitQueries[$partition->name])) { + if ($select['alias']) { + $this->splitQueries[$partition->name]->query->selectAlias($select['select'], $select['alias']); + } else { + $this->splitQueries[$partition->name]->query->addSelect($select['select']); + } + if ($select['select'] !== '*') { + continue 2; + } + } + } + } + + if ($select['alias']) { + parent::selectAlias($select['select'], $select['alias']); + } else { + parent::addSelect($select['select']); + } + } + $this->selects = []; + } + + + public function addPartition(PartitionSplit $partition): void { + $this->partitions[] = $partition; + } + + private function getPartition(string $table): ?PartitionSplit { + foreach ($this->partitions as $partition) { + if ($partition->containsTable($table) || $partition->containsAlias($table)) { + return $partition; + } + } + return null; + } + + public function from($from, $alias = null) { + if (is_string($from) && $partition = $this->getPartition($from)) { + $this->mainPartition = $partition; + if ($alias) { + $this->mainPartition->addAlias($from, $alias); + } + } + return parent::from($from, $alias); + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null): self { + return $this->join($fromAlias, $join, $alias, $condition); + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null): self { + return $this->join($fromAlias, (string)$join, $alias, $condition, PartitionQuery::JOIN_MODE_LEFT); + } + + public function join($fromAlias, $join, $alias, $condition = null, $joinMode = PartitionQuery::JOIN_MODE_INNER): self { + $partition = $this->getPartition($join); + $fromPartition = $this->getPartition($fromAlias); + if ($partition && $partition !== $this->mainPartition) { + // join from the main db to a partition + + $joinCondition = JoinCondition::parse($condition, $join, $alias, $fromAlias); + $partition->addAlias($join, $alias); + + if (!isset($this->splitQueries[$partition->name])) { + $this->splitQueries[$partition->name] = new PartitionQuery( + $this->newQuery(), + $joinCondition->fromAlias ?? $joinCondition->fromColumn, $joinCondition->toAlias ?? $joinCondition->toColumn, + $joinMode + ); + $this->splitQueries[$partition->name]->query->from($join, $alias); + $this->ensureSelect($joinCondition->fromColumn, $joinCondition->fromAlias); + $this->ensureSelect($joinCondition->toColumn, $joinCondition->toAlias); + } else { + $query = $this->splitQueries[$partition->name]->query; + if ($partition->containsAlias($fromAlias)) { + $query->innerJoin($fromAlias, $join, $alias, $condition); + } else { + throw new InvalidPartitionedQueryException("Can't join across partition boundaries more than once"); + } + } + $this->splitQueries[$partition->name]->query->andWhere(...$joinCondition->toConditions); + parent::andWhere(...$joinCondition->fromConditions); + return $this; + } elseif ($fromPartition && $fromPartition !== $partition) { + // join from partition, to the main db + + $joinCondition = JoinCondition::parse($condition, $join, $alias, $fromAlias); + if (str_starts_with($fromPartition->name, 'from_')) { + $partitionName = $fromPartition->name; + } else { + $partitionName = 'from_' . $fromPartition->name; + } + + if (!isset($this->splitQueries[$partitionName])) { + $newPartition = new PartitionSplit($partitionName, [$join]); + $newPartition->addAlias($join, $alias); + $this->partitions[] = $newPartition; + + $this->splitQueries[$partitionName] = new PartitionQuery( + $this->newQuery(), + $joinCondition->fromAlias ?? $joinCondition->fromColumn, $joinCondition->toAlias ?? $joinCondition->toColumn, + $joinMode + ); + $this->ensureSelect($joinCondition->fromColumn, $joinCondition->fromAlias); + $this->ensureSelect($joinCondition->toColumn, $joinCondition->toAlias); + $this->splitQueries[$partitionName]->query->from($join, $alias); + $this->splitQueries[$partitionName]->query->andWhere(...$joinCondition->toConditions); + parent::andWhere(...$joinCondition->fromConditions); + } else { + $fromPartition->addTable($join); + $fromPartition->addAlias($join, $alias); + + $query = $this->splitQueries[$partitionName]->query; + $query->innerJoin($fromAlias, $join, $alias, $condition); + } + return $this; + } else { + // join within the main db or a partition + if ($joinMode === PartitionQuery::JOIN_MODE_INNER) { + return parent::innerJoin($fromAlias, $join, $alias, $condition); + } elseif ($joinMode === PartitionQuery::JOIN_MODE_LEFT) { + return parent::leftJoin($fromAlias, $join, $alias, $condition); + } elseif ($joinMode === PartitionQuery::JOIN_MODE_RIGHT) { + return parent::rightJoin($fromAlias, $join, $alias, $condition); + } else { + throw new \InvalidArgumentException("Invalid join mode: $joinMode"); + } + } + } + + /** + * Flatten a list of predicates by merging the parts of any "AND" expression into the list of predicates + * + * @param array $predicates + * @return array + */ + private function flattenPredicates(array $predicates): array { + $result = []; + foreach ($predicates as $predicate) { + if ($predicate instanceof CompositeExpression && $predicate->getType() === CompositeExpression::TYPE_AND) { + $result = array_merge($result, $this->flattenPredicates($predicate->getParts())); + } else { + $result[] = $predicate; + } + } + return $result; + } + + /** + * Split an array of predicates (WHERE query parts) by the partition they reference + * + * @param array $predicates + * @return array<string, array> + */ + private function splitPredicatesByParts(array $predicates): array { + $predicates = $this->flattenPredicates($predicates); + + $partitionPredicates = []; + foreach ($predicates as $predicate) { + $partition = $this->getPartitionForPredicate((string)$predicate); + if ($this->mainPartition === $partition) { + $partitionPredicates[''][] = $predicate; + } elseif ($partition) { + $partitionPredicates[$partition->name][] = $predicate; + } else { + $partitionPredicates[''][] = $predicate; + } + } + return $partitionPredicates; + } + + public function where(...$predicates) { + return $this->andWhere(...$predicates); + } + + public function andWhere(...$where) { + if ($where) { + foreach ($this->splitPredicatesByParts($where) as $alias => $predicates) { + if (isset($this->splitQueries[$alias])) { + // when there is a condition on a table being left-joined it starts to behave as if it's an inner join + // since any joined column that doesn't have the left part will not match the condition + // when there the condition is `$joinToColumn IS NULL` we instead mark the query as excluding the left half + if ($this->splitQueries[$alias]->joinMode === PartitionQuery::JOIN_MODE_LEFT) { + $this->splitQueries[$alias]->joinMode = PartitionQuery::JOIN_MODE_INNER; + + $column = $this->quoteHelper->quoteColumnName($this->splitQueries[$alias]->joinToColumn); + foreach ($predicates as $predicate) { + if ((string)$predicate === "$column IS NULL") { + $this->splitQueries[$alias]->joinMode = PartitionQuery::JOIN_MODE_LEFT_NULL; + } else { + $this->splitQueries[$alias]->query->andWhere($predicate); + } + } + } else { + $this->splitQueries[$alias]->query->andWhere(...$predicates); + } + } else { + parent::andWhere(...$predicates); + } + } + } + return $this; + } + + + private function getPartitionForPredicate(string $predicate): ?PartitionSplit { + foreach ($this->partitions as $partition) { + + if (str_contains($predicate, '?')) { + $this->hasPositionalParameter = true; + } + if ($partition->checkPredicateForTable($predicate)) { + return $partition; + } + } + return null; + } + + public function update($update = null, $alias = null) { + return parent::update($update, $alias); + } + + public function insert($insert = null) { + return parent::insert($insert); + } + + public function delete($delete = null, $alias = null) { + return parent::delete($delete, $alias); + } + + public function setMaxResults($maxResults) { + if ($maxResults > 0) { + $this->limit = (int)$maxResults; + } + return parent::setMaxResults($maxResults); + } + + public function setFirstResult($firstResult) { + if ($firstResult > 0) { + $this->offset = (int)$firstResult; + } + return parent::setFirstResult($firstResult); + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + $this->applySelects(); + if ($this->splitQueries && $this->hasPositionalParameter) { + throw new InvalidPartitionedQueryException("Partitioned queries aren't allowed to to positional arguments"); + } + foreach ($this->splitQueries as $split) { + $split->query->setParameters($this->getParameters(), $this->getParameterTypes()); + } + if (count($this->splitQueries) > 0) { + $hasNonLeftJoins = array_reduce($this->splitQueries, function (bool $hasNonLeftJoins, PartitionQuery $query) { + return $hasNonLeftJoins || $query->joinMode !== PartitionQuery::JOIN_MODE_LEFT; + }, false); + if ($hasNonLeftJoins) { + if (is_int($this->limit)) { + throw new InvalidPartitionedQueryException('Limit is not allowed in partitioned queries'); + } + if (is_int($this->offset)) { + throw new InvalidPartitionedQueryException('Offset is not allowed in partitioned queries'); + } + } + } + + $s = $this->getSQL(); + $result = parent::executeQuery($connection); + if (count($this->splitQueries) > 0) { + return new PartitionedResult($this->splitQueries, $result); + } else { + return $result; + } + } + + public function executeStatement(?IDBConnection $connection = null): int { + if (count($this->splitQueries)) { + throw new InvalidPartitionedQueryException("Partitioning write queries isn't supported"); + } + return parent::executeStatement($connection); + } + + public function getSQL() { + $this->applySelects(); + return parent::getSQL(); + } + + public function getPartitionCount(): int { + return count($this->splitQueries) + 1; + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if (str_contains($column, '.')) { + [$alias, $column] = explode('.', $column); + $partition = $this->getPartition($alias); + if ($partition) { + $this->splitQueries[$partition->name]->query->hintShardKey($column, $value, $overwrite); + } else { + parent::hintShardKey($column, $value, $overwrite); + } + } else { + parent::hintShardKey($column, $value, $overwrite); + } + return $this; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php new file mode 100644 index 00000000000..b3b59e26298 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\ArrayResult; +use OCP\DB\IResult; +use PDO; + +/** + * Combine the results of multiple join parts into a single result + */ +class PartitionedResult extends ArrayResult { + private bool $fetched = false; + + /** + * @param PartitionQuery[] $splitOfParts + * @param IResult $result + */ + public function __construct( + private array $splitOfParts, + private IResult $result, + ) { + parent::__construct([]); + } + + public function closeCursor(): bool { + return $this->result->closeCursor(); + } + + public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { + $this->fetchRows(); + return parent::fetch($fetchMode); + } + + public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { + $this->fetchRows(); + return parent::fetchAll($fetchMode); + } + + public function rowCount(): int { + $this->fetchRows(); + return parent::rowCount(); + } + + private function fetchRows(): void { + if (!$this->fetched) { + $this->fetched = true; + $this->rows = $this->result->fetchAll(); + foreach ($this->splitOfParts as $part) { + $this->rows = $part->mergeWith($this->rows); + } + $this->count = count($this->rows); + } + } +} diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index fb28fa28649..1d44c049793 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -1,42 +1,15 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * @author J0WI <J0WI@users.noreply.github.com> - * @author Joas Schilling <coding@schilljs.com> - * @author Lukas Reschke <lukas@statuscode.ch> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Vincent Petry <vincent@nextcloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder; -use Doctrine\DBAL\Platforms\MySQLPlatform; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQL94Platform; -use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Query\QueryException; use OC\DB\ConnectionAdapter; -use OC\DB\QueryBuilder\ExpressionBuilder\ExpressionBuilder; +use OC\DB\Exceptions\DbalException; use OC\DB\QueryBuilder\ExpressionBuilder\MySqlExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\OCIExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\PgSqlExpressionBuilder; @@ -45,25 +18,24 @@ use OC\DB\QueryBuilder\FunctionBuilder\FunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\OCIFunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\PgSqlFunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\SqliteFunctionBuilder; -use OC\DB\ResultAdapter; use OC\SystemConfig; use OCP\DB\IResult; +use OCP\DB\QueryBuilder\ICompositeExpression; use OCP\DB\QueryBuilder\ILiteral; use OCP\DB\QueryBuilder\IParameter; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; -use OCP\ILogger; +use OCP\IDBConnection; +use Psr\Log\LoggerInterface; class QueryBuilder implements IQueryBuilder { - /** @var ConnectionAdapter */ private $connection; /** @var SystemConfig */ private $systemConfig; - /** @var ILogger */ - private $logger; + private LoggerInterface $logger; /** @var \Doctrine\DBAL\Query\QueryBuilder */ private $queryBuilder; @@ -73,18 +45,19 @@ class QueryBuilder implements IQueryBuilder { /** @var bool */ private $automaticTablePrefix = true; + private bool $nonEmptyWhere = false; /** @var string */ protected $lastInsertedTable; + private array $selectedColumns = []; /** * Initializes a new QueryBuilder. * * @param ConnectionAdapter $connection * @param SystemConfig $systemConfig - * @param ILogger $logger */ - public function __construct(ConnectionAdapter $connection, SystemConfig $systemConfig, ILogger $logger) { + public function __construct(ConnectionAdapter $connection, SystemConfig $systemConfig, LoggerInterface $logger) { $this->connection = $connection; $this->systemConfig = $systemConfig; $this->logger = $logger; @@ -96,11 +69,11 @@ class QueryBuilder implements IQueryBuilder { * Enable/disable automatic prefixing of table names with the oc_ prefix * * @param bool $enabled If set to true table names will be prefixed with the - * owncloud database prefix automatically. + * owncloud database prefix automatically. * @since 8.2.0 */ public function automaticTablePrefix($enabled) { - $this->automaticTablePrefix = (bool) $enabled; + $this->automaticTablePrefix = (bool)$enabled; } /** @@ -120,20 +93,13 @@ class QueryBuilder implements IQueryBuilder { * @return \OCP\DB\QueryBuilder\IExpressionBuilder */ public function expr() { - if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { - return new OCIExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { - return new PgSqlExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof MySQLPlatform) { - return new MySqlExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { - return new SqliteExpressionBuilder($this->connection, $this); - } - - return new ExpressionBuilder($this->connection, $this); + return match($this->connection->getDatabaseProvider()) { + IDBConnection::PLATFORM_ORACLE => new OCIExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_POSTGRES => new PgSqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_MARIADB, + IDBConnection::PLATFORM_MYSQL => new MySqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_SQLITE => new SqliteExpressionBuilder($this->connection, $this, $this->logger), + }; } /** @@ -153,17 +119,13 @@ class QueryBuilder implements IQueryBuilder { * @return \OCP\DB\QueryBuilder\IFunctionBuilder */ public function func() { - if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { - return new OCIFunctionBuilder($this->helper); - } - if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { - return new SqliteFunctionBuilder($this->helper); - } - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { - return new PgSqlFunctionBuilder($this->helper); - } - - return new FunctionBuilder($this->helper); + return match($this->connection->getDatabaseProvider()) { + IDBConnection::PLATFORM_ORACLE => new OCIFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_POSTGRES => new PgSqlFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_MARIADB, + IDBConnection::PLATFORM_MYSQL => new FunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_SQLITE => new SqliteFunctionBuilder($this->connection, $this, $this->helper), + }; } /** @@ -187,26 +149,21 @@ class QueryBuilder implements IQueryBuilder { /** * Gets the state of this query builder instance. * - * @return integer Either QueryBuilder::STATE_DIRTY or QueryBuilder::STATE_CLEAN. + * @return int Always returns 0 which is former `QueryBuilder::STATE_DIRTY` + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. */ public function getState() { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getState(); } - /** - * Executes this query using the bound parameters and their types. - * - * Uses {@see Connection::executeQuery} for select statements and {@see Connection::executeUpdate} - * for insert, update and delete statements. - * - * @return IResult|int - */ - public function execute() { + private function prepareForExecute() { if ($this->systemConfig->getValue('log_query', false)) { try { $params = []; foreach ($this->getParameters() as $placeholder => $value) { - if ($value instanceof \DateTime) { + if ($value instanceof \DateTimeInterface) { $params[] = $placeholder . ' => DateTime:\'' . $value->format('c') . '\''; } elseif (is_array($value)) { $params[] = $placeholder . ' => (\'' . implode('\', \'', $value) . '\')'; @@ -228,31 +185,46 @@ class QueryBuilder implements IQueryBuilder { } } catch (\Error $e) { // likely an error during conversion of $value to string - $this->logger->debug('DB QueryBuilder: error trying to log SQL query'); - $this->logger->logException($e); + $this->logger->error('DB QueryBuilder: error trying to log SQL query', ['exception' => $e]); } } - if (!empty($this->getQueryPart('select'))) { - $select = $this->getQueryPart('select'); - $hasSelectAll = array_filter($select, static function ($s) { - return $s === '*'; - }); - $hasSelectSpecific = array_filter($select, static function ($s) { - return $s !== '*'; - }); - - if (empty($hasSelectAll) === empty($hasSelectSpecific)) { - $exception = new QueryException('Query is selecting * and specific values in the same query. This is not supported in Oracle.'); - $this->logger->logException($exception, [ - 'message' => 'Query is selecting * and specific values in the same query. This is not supported in Oracle.', - 'query' => $this->getSQL(), - 'level' => ILogger::ERROR, - 'app' => 'core', - ]); + // if (!empty($this->getQueryPart('select'))) { + // $select = $this->getQueryPart('select'); + // $hasSelectAll = array_filter($select, static function ($s) { + // return $s === '*'; + // }); + // $hasSelectSpecific = array_filter($select, static function ($s) { + // return $s !== '*'; + // }); + + // if (empty($hasSelectAll) === empty($hasSelectSpecific)) { + // $exception = new QueryException('Query is selecting * and specific values in the same query. This is not supported in Oracle.'); + // $this->logger->error($exception->getMessage(), [ + // 'query' => $this->getSQL(), + // 'app' => 'core', + // 'exception' => $exception, + // ]); + // } + // } + + $tooLongOutputColumns = []; + foreach ($this->getOutputColumns() as $column) { + if (strlen($column) > 30) { + $tooLongOutputColumns[] = $column; } } + if (!empty($tooLongOutputColumns)) { + $exception = new QueryException('More than 30 characters for an output column name are not allowed on Oracle.'); + $this->logger->error($exception->getMessage(), [ + 'query' => $this->getSQL(), + 'columns' => $tooLongOutputColumns, + 'app' => 'core', + 'exception' => $exception, + ]); + } + $numberOfParameters = 0; $hasTooLargeArrayParameter = false; foreach ($this->getParameters() as $parameter) { @@ -265,31 +237,82 @@ class QueryBuilder implements IQueryBuilder { if ($hasTooLargeArrayParameter) { $exception = new QueryException('More than 1000 expressions in a list are not allowed on Oracle.'); - $this->logger->logException($exception, [ - 'message' => 'More than 1000 expressions in a list are not allowed on Oracle.', + $this->logger->error($exception->getMessage(), [ 'query' => $this->getSQL(), - 'level' => ILogger::ERROR, 'app' => 'core', + 'exception' => $exception, ]); } if ($numberOfParameters > 65535) { $exception = new QueryException('The number of parameters must not exceed 65535. Restriction by PostgreSQL.'); - $this->logger->logException($exception, [ - 'message' => 'The number of parameters must not exceed 65535. Restriction by PostgreSQL.', + $this->logger->error($exception->getMessage(), [ 'query' => $this->getSQL(), - 'level' => ILogger::ERROR, 'app' => 'core', + 'exception' => $exception, ]); } - - $result = $this->queryBuilder->execute(); - if (is_int($result)) { - return $result; + } + + /** + * Executes this query using the bound parameters and their types. + * + * Uses {@see Connection::executeQuery} for select statements and {@see Connection::executeUpdate} + * for insert, update and delete statements. + * + * @return IResult|int + */ + public function execute(?IDBConnection $connection = null) { + try { + if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + return $this->executeQuery($connection); + } else { + return $this->executeStatement($connection); + } + } catch (DBALException $e) { + // `IQueryBuilder->execute` never wrapped the exception, but `executeQuery` and `executeStatement` do + /** @var \Doctrine\DBAL\Exception $previous */ + $previous = $e->getPrevious(); + + throw $previous; + } + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + if ($this->getType() !== \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + throw new \RuntimeException('Invalid query type, expected SELECT query'); + } + + $this->prepareForExecute(); + if (!$connection) { + $connection = $this->connection; + } + + return $connection->executeQuery( + $this->getSQL(), + $this->getParameters(), + $this->getParameterTypes(), + ); + } + + public function executeStatement(?IDBConnection $connection = null): int { + if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + throw new \RuntimeException('Invalid query type, expected INSERT, DELETE or UPDATE statement'); } - return new ResultAdapter($result); + + $this->prepareForExecute(); + if (!$connection) { + $connection = $this->connection; + } + + return $connection->executeStatement( + $this->getSQL(), + $this->getParameters(), + $this->getParameterTypes(), + ); } + /** * Gets the complete SQL string formed by the current specifications of this QueryBuilder. * @@ -397,21 +420,21 @@ class QueryBuilder implements IQueryBuilder { /** * Sets the position of the first result to retrieve (the "offset"). * - * @param integer $firstResult The first result to return. + * @param int $firstResult The first result to return. * * @return $this This QueryBuilder instance. */ public function setFirstResult($firstResult) { - $this->queryBuilder->setFirstResult($firstResult); + $this->queryBuilder->setFirstResult((int)$firstResult); return $this; } /** * Gets the position of the first result the query object was set to retrieve (the "offset"). - * Returns NULL if {@link setFirstResult} was not applied to this QueryBuilder. + * Returns 0 if {@link setFirstResult} was not applied to this QueryBuilder. * - * @return integer The position of the first result. + * @return int The position of the first result. */ public function getFirstResult() { return $this->queryBuilder->getFirstResult(); @@ -424,12 +447,16 @@ class QueryBuilder implements IQueryBuilder { * of the databases will just return an empty result set, Oracle will return * all entries. * - * @param integer $maxResults The maximum number of results to retrieve. + * @param int|null $maxResults The maximum number of results to retrieve. * * @return $this This QueryBuilder instance. */ public function setMaxResults($maxResults) { - $this->queryBuilder->setMaxResults($maxResults); + if ($maxResults === null) { + $this->queryBuilder->setMaxResults($maxResults); + } else { + $this->queryBuilder->setMaxResults((int)$maxResults); + } return $this; } @@ -463,6 +490,7 @@ class QueryBuilder implements IQueryBuilder { if (count($selects) === 1 && is_array($selects[0])) { $selects = $selects[0]; } + $this->addOutputColumns($selects); $this->queryBuilder->select( $this->helper->quoteColumnNames($selects) @@ -490,6 +518,7 @@ class QueryBuilder implements IQueryBuilder { $this->queryBuilder->addSelect( $this->helper->quoteColumnName($select) . ' AS ' . $this->helper->quoteColumnName($alias) ); + $this->addOutputColumns([$alias]); return $this; } @@ -511,6 +540,7 @@ class QueryBuilder implements IQueryBuilder { if (!is_array($select)) { $select = [$select]; } + $this->addOutputColumns($select); $quotedSelect = $this->helper->quoteColumnNames($select); @@ -540,6 +570,7 @@ class QueryBuilder implements IQueryBuilder { if (count($selects) === 1 && is_array($selects[0])) { $selects = $selects[0]; } + $this->addOutputColumns($selects); $this->queryBuilder->addSelect( $this->helper->quoteColumnNames($selects) @@ -548,6 +579,26 @@ class QueryBuilder implements IQueryBuilder { return $this; } + private function addOutputColumns(array $columns): void { + foreach ($columns as $column) { + if (is_array($column)) { + $this->addOutputColumns($column); + } elseif (is_string($column) && !str_contains($column, '*')) { + if (str_contains(strtolower($column), ' as ')) { + [, $column] = preg_split('/ as /i', $column); + } + if (str_contains($column, '.')) { + [, $column] = explode('.', $column); + } + $this->selectedColumns[] = $column; + } + } + } + + public function getOutputColumns(): array { + return array_unique($this->selectedColumns); + } + /** * Turns the query being built into a bulk delete query that ranges over * a certain table. @@ -563,8 +614,13 @@ class QueryBuilder implements IQueryBuilder { * @param string $alias The table alias used in the constructed query. * * @return $this This QueryBuilder instance. + * @since 30.0.0 Alias is deprecated and will no longer be used with the next Doctrine/DBAL update */ public function delete($delete = null, $alias = null) { + if ($alias !== null) { + $this->logger->debug('DELETE queries with alias are no longer supported and the provided alias is ignored', ['exception' => new \InvalidArgumentException('Table alias provided for DELETE query')]); + } + $this->queryBuilder->delete( $this->getTableName($delete), $alias @@ -588,8 +644,13 @@ class QueryBuilder implements IQueryBuilder { * @param string $alias The table alias used in the constructed query. * * @return $this This QueryBuilder instance. + * @since 30.0.0 Alias is deprecated and will no longer be used with the next Doctrine/DBAL update */ public function update($update = null, $alias = null) { + if ($alias !== null) { + $this->logger->debug('UPDATE queries with alias are no longer supported and the provided alias is ignored', ['exception' => new \InvalidArgumentException('Table alias provided for UPDATE query')]); + } + $this->queryBuilder->update( $this->getTableName($update), $alias @@ -637,7 +698,7 @@ class QueryBuilder implements IQueryBuilder { * ->from('users', 'u') * </code> * - * @param string $from The table. + * @param string|IQueryFunction $from The table. * @param string|null $alias The alias of the table. * * @return $this This QueryBuilder instance. @@ -664,7 +725,7 @@ class QueryBuilder implements IQueryBuilder { * @param string $fromAlias The alias that points to a from clause. * @param string $join The table name to join. * @param string $alias The alias of the join table. - * @param string $condition The condition for the join. + * @param string|ICompositeExpression|null $condition The condition for the join. * * @return $this This QueryBuilder instance. */ @@ -692,7 +753,7 @@ class QueryBuilder implements IQueryBuilder { * @param string $fromAlias The alias that points to a from clause. * @param string $join The table name to join. * @param string $alias The alias of the join table. - * @param string $condition The condition for the join. + * @param string|ICompositeExpression|null $condition The condition for the join. * * @return $this This QueryBuilder instance. */ @@ -718,9 +779,9 @@ class QueryBuilder implements IQueryBuilder { * </code> * * @param string $fromAlias The alias that points to a from clause. - * @param string $join The table name to join. + * @param string|IQueryFunction $join The table name or sub-query to join. * @param string $alias The alias of the join table. - * @param string $condition The condition for the join. + * @param string|ICompositeExpression|null $condition The condition for the join. * * @return $this This QueryBuilder instance. */ @@ -748,7 +809,7 @@ class QueryBuilder implements IQueryBuilder { * @param string $fromAlias The alias that points to a from clause. * @param string $join The table name to join. * @param string $alias The alias of the join table. - * @param string $condition The condition for the join. + * @param string|ICompositeExpression|null $condition The condition for the join. * * @return $this This QueryBuilder instance. */ @@ -797,12 +858,13 @@ class QueryBuilder implements IQueryBuilder { * ->from('users', 'u') * ->where('u.id = ?'); * - * // You can optionally programatically build and/or expressions + * // You can optionally programmatically build and/or expressions * $qb = $conn->getQueryBuilder(); * - * $or = $qb->expr()->orx(); - * $or->add($qb->expr()->eq('u.id', 1)); - * $or->add($qb->expr()->eq('u.id', 2)); + * $or = $qb->expr()->orx( + * $qb->expr()->eq('u.id', 1), + * $qb->expr()->eq('u.id', 2), + * ); * * $qb->update('users', 'u') * ->set('u.password', md5('password')) @@ -814,6 +876,14 @@ class QueryBuilder implements IQueryBuilder { * @return $this This QueryBuilder instance. */ public function where(...$predicates) { + if ($this->nonEmptyWhere && $this->systemConfig->getValue('debug', false)) { + // Only logging a warning, not throwing for now. + $e = new QueryException('Using where() on non-empty WHERE part, please verify it is intentional to not call andWhere() or orWhere() instead. Otherwise consider creating a new query builder object or call resetQueryPart(\'where\') first.'); + $this->logger->warning($e->getMessage(), ['exception' => $e]); + } + + $this->nonEmptyWhere = true; + call_user_func_array( [$this->queryBuilder, 'where'], $predicates @@ -841,6 +911,7 @@ class QueryBuilder implements IQueryBuilder { * @see where() */ public function andWhere(...$where) { + $this->nonEmptyWhere = true; call_user_func_array( [$this->queryBuilder, 'andWhere'], $where @@ -868,6 +939,7 @@ class QueryBuilder implements IQueryBuilder { * @see where() */ public function orWhere(...$where) { + $this->nonEmptyWhere = true; call_user_func_array( [$this->queryBuilder, 'orWhere'], $where @@ -919,14 +991,10 @@ class QueryBuilder implements IQueryBuilder { * * @return $this This QueryBuilder instance. */ - public function addGroupBy(...$groupBys) { - if (count($groupBys) === 1 && is_array($groupBys[0])) { - $$groupBys = $groupBys[0]; - } - + public function addGroupBy(...$groupBy) { call_user_func_array( [$this->queryBuilder, 'addGroupBy'], - $this->helper->quoteColumnNames($groupBys) + $this->helper->quoteColumnNames($groupBy) ); return $this; @@ -954,7 +1022,7 @@ class QueryBuilder implements IQueryBuilder { public function setValue($column, $value) { $this->queryBuilder->setValue( $this->helper->quoteColumnName($column), - (string) $value + (string)$value ); return $this; @@ -1045,7 +1113,7 @@ class QueryBuilder implements IQueryBuilder { * Specifies an ordering for the query results. * Replaces any previously specified orderings, if any. * - * @param string $sort The ordering expression. + * @param string|IQueryFunction|ILiteral|IParameter $sort The ordering expression. * @param string $order The ordering direction. * * @return $this This QueryBuilder instance. @@ -1062,7 +1130,7 @@ class QueryBuilder implements IQueryBuilder { /** * Adds an ordering to the query results. * - * @param string $sort The ordering expression. + * @param string|ILiteral|IParameter|IQueryFunction $sort The ordering expression. * @param string $order The ordering direction. * * @return $this This QueryBuilder instance. @@ -1082,8 +1150,11 @@ class QueryBuilder implements IQueryBuilder { * @param string $queryPartName * * @return mixed + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please track the details you need, outside the object. */ public function getQueryPart($queryPartName) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getQueryPart($queryPartName); } @@ -1091,8 +1162,11 @@ class QueryBuilder implements IQueryBuilder { * Gets all query parts. * * @return array + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please track the details you need, outside the object. */ public function getQueryParts() { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getQueryParts(); } @@ -1102,8 +1176,11 @@ class QueryBuilder implements IQueryBuilder { * @param array|null $queryPartNames * * @return $this This QueryBuilder instance. + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please create a new IQueryBuilder instead. */ public function resetQueryParts($queryPartNames = null) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); $this->queryBuilder->resetQueryParts($queryPartNames); return $this; @@ -1115,8 +1192,11 @@ class QueryBuilder implements IQueryBuilder { * @param string $queryPartName * * @return $this This QueryBuilder instance. + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please create a new IQueryBuilder instead. */ public function resetQueryPart($queryPartName) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); $this->queryBuilder->resetQueryPart($queryPartName); return $this; @@ -1146,7 +1226,7 @@ class QueryBuilder implements IQueryBuilder { * @link http://www.zetacomponents.org * * @param mixed $value - * @param mixed $type + * @param IQueryBuilder::PARAM_* $type * @param string $placeHolder The name to bind with. The string must start with a colon ':'. * * @return IParameter the placeholder name used. @@ -1173,7 +1253,7 @@ class QueryBuilder implements IQueryBuilder { * </code> * * @param mixed $value - * @param integer $type + * @param IQueryBuilder::PARAM_* $type * * @return IParameter */ @@ -1233,11 +1313,11 @@ class QueryBuilder implements IQueryBuilder { * @return int * @throws \BadMethodCallException When being called before an insert query has been run. */ - public function getLastInsertId() { + public function getLastInsertId(): int { if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::INSERT && $this->lastInsertedTable) { // lastInsertId() needs the prefix but no quotes $table = $this->prefixTableName($this->lastInsertedTable); - return (int) $this->connection->lastInsertId($table); + return $this->connection->lastInsertId($table); } throw new \BadMethodCallException('Invalid call to getLastInsertId without using insert() before.'); @@ -1246,12 +1326,12 @@ class QueryBuilder implements IQueryBuilder { /** * Returns the table name quoted and with database prefix as needed by the implementation * - * @param string $table + * @param string|IQueryFunction $table * @return string */ public function getTableName($table) { if ($table instanceof IQueryFunction) { - return (string) $table; + return (string)$table; } $table = $this->prefixTableName($table); @@ -1261,11 +1341,13 @@ class QueryBuilder implements IQueryBuilder { /** * Returns the table name with database prefix as needed by the implementation * + * Was protected until version 30. + * * @param string $table * @return string */ - protected function prefixTableName($table) { - if ($this->automaticTablePrefix === false || strpos($table, '*PREFIX*') === 0) { + public function prefixTableName(string $table): string { + if ($this->automaticTablePrefix === false || str_starts_with($table, '*PREFIX*')) { return $table; } @@ -1300,4 +1382,18 @@ class QueryBuilder implements IQueryBuilder { return $this->helper->quoteColumnName($alias); } + + public function escapeLikeParameter(string $parameter): string { + return $this->connection->escapeLikeParameter($parameter); + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + return $this; + } + + public function runAcrossAllShards(): self { + // noop + return $this; + } + } diff --git a/lib/private/DB/QueryBuilder/QueryFunction.php b/lib/private/DB/QueryBuilder/QueryFunction.php index 60f2bf12383..7f2ab584a73 100644 --- a/lib/private/DB/QueryBuilder/QueryFunction.php +++ b/lib/private/DB/QueryBuilder/QueryFunction.php @@ -1,25 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; @@ -32,10 +17,7 @@ class QueryFunction implements IQueryFunction { $this->function = $function; } - /** - * @return string - */ - public function __toString() { - return (string) $this->function; + public function __toString(): string { + return (string)$this->function; } } diff --git a/lib/private/DB/QueryBuilder/QuoteHelper.php b/lib/private/DB/QueryBuilder/QuoteHelper.php index 9f31e69fa7e..7ce4b359638 100644 --- a/lib/private/DB/QueryBuilder/QuoteHelper.php +++ b/lib/private/DB/QueryBuilder/QuoteHelper.php @@ -1,26 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Joas Schilling <coding@schilljs.com> - * @author Robin Appelman <robin@icewind.nl> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\ILiteral; @@ -51,7 +35,7 @@ class QuoteHelper { */ public function quoteColumnName($string) { if ($string instanceof IParameter || $string instanceof ILiteral || $string instanceof IQueryFunction) { - return (string) $string; + return (string)$string; } if ($string === null || $string === 'null' || $string === '*') { @@ -68,7 +52,7 @@ class QuoteHelper { } if (substr_count($string, '.')) { - list($alias, $columnName) = explode('.', $string, 2); + [$alias, $columnName] = explode('.', $string, 2); if ($columnName === '*') { return '`' . $alias . '`.*'; diff --git a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php new file mode 100644 index 00000000000..3a230ea544d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php @@ -0,0 +1,155 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\ICacheFactory; +use OCP\IMemcache; +use OCP\IMemcacheTTL; + +/** + * A helper to atomically determine the next auto increment value for a sharded table + * + * Since we can't use the database's auto-increment (since each db doesn't know about the keys in the other shards) + * we need external logic for doing the auto increment + */ +class AutoIncrementHandler { + public const MIN_VALID_KEY = 1000; + public const TTL = 365 * 24 * 60 * 60; + + private ?IMemcache $cache = null; + + public function __construct( + private ICacheFactory $cacheFactory, + private ShardConnectionManager $shardConnectionManager, + ) { + if (PHP_INT_SIZE < 8) { + throw new \Exception('sharding is only supported with 64bit php'); + } + } + + private function getCache(): IMemcache { + if (is_null($this->cache)) { + $cache = $this->cacheFactory->createDistributed('shared_autoincrement'); + if ($cache instanceof IMemcache) { + $this->cache = $cache; + } else { + throw new \Exception('Distributed cache ' . get_class($cache) . ' is not suitable'); + } + } + return $this->cache; + } + + /** + * Get the next value for the given shard definition + * + * The returned key is unique and incrementing, but not sequential. + * The shard id is encoded in the first byte of the returned value + * + * @param ShardDefinition $shardDefinition + * @return int + * @throws \Exception + */ + public function getNextPrimaryKey(ShardDefinition $shardDefinition, int $shard): int { + $retries = 0; + while ($retries < 5) { + $next = $this->getNextInner($shardDefinition); + if ($next !== null) { + if ($next > ShardDefinition::MAX_PRIMARY_KEY) { + throw new \Exception('Max primary key of ' . ShardDefinition::MAX_PRIMARY_KEY . ' exceeded'); + } + // we encode the shard the primary key was originally inserted into to allow guessing the shard by primary key later on + return ($next << 8) | $shard; + } else { + $retries++; + } + } + throw new \Exception('Failed to get next primary key'); + } + + /** + * auto increment logic without retry + * + * @param ShardDefinition $shardDefinition + * @return int|null either the next primary key or null if the call needs to be retried + */ + private function getNextInner(ShardDefinition $shardDefinition): ?int { + $cache = $this->getCache(); + // because this function will likely be called concurrently from different requests + // the implementation needs to ensure that the cached value can be cleared, invalidated or re-calculated at any point between our cache calls + // care must be taken that the logic remains fully resilient against race conditions + + // in the ideal case, the last primary key is stored in the cache and we can just do an `inc` + // if that is not the case we find the highest used id in the database increment it, and save it in the cache + + // prevent inc from returning `1` if the key doesn't exist by setting it to a non-numeric value + $cache->add($shardDefinition->table, 'empty-placeholder', self::TTL); + $next = $cache->inc($shardDefinition->table); + + if ($cache instanceof IMemcacheTTL) { + $cache->setTTL($shardDefinition->table, self::TTL); + } + + // the "add + inc" trick above isn't strictly atomic, so as a safety we reject any result that to small + // to handle the edge case of the stored value disappearing between the add and inc + if (is_int($next) && $next >= self::MIN_VALID_KEY) { + return $next; + } elseif (is_int($next)) { + // we hit the edge case, so invalidate the cached value + if (!$cache->cas($shardDefinition->table, $next, 'empty-placeholder')) { + // someone else is changing the value concurrently, give up and retry + return null; + } + } + + // discard the encoded initial shard + $current = $this->getMaxFromDb($shardDefinition); + $next = max($current, self::MIN_VALID_KEY) + 1; + if ($cache->cas($shardDefinition->table, 'empty-placeholder', $next)) { + return $next; + } + + // another request set the cached value before us, so we should just be able to inc + $next = $cache->inc($shardDefinition->table); + if (is_int($next) && $next >= self::MIN_VALID_KEY) { + return $next; + } elseif (is_int($next)) { + // key got cleared, invalidate and retry + $cache->cas($shardDefinition->table, $next, 'empty-placeholder'); + return null; + } else { + // cleanup any non-numeric value other than the placeholder if that got stored somehow + $cache->ncad($shardDefinition->table, 'empty-placeholder'); + // retry + return null; + } + } + + /** + * Get the maximum primary key value from the shards, note that this has already stripped any embedded shard id + */ + private function getMaxFromDb(ShardDefinition $shardDefinition): int { + $max = $shardDefinition->fromFileId; + $query = $this->shardConnectionManager->getConnection($shardDefinition, 0)->getQueryBuilder(); + $query->select($shardDefinition->primaryKey) + ->from($shardDefinition->table) + ->orderBy($shardDefinition->primaryKey, 'DESC') + ->setMaxResults(1); + foreach ($shardDefinition->getAllShards() as $shard) { + $connection = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + $result = $query->executeQuery($connection)->fetchOne(); + if ($result) { + if ($result > $shardDefinition->fromFileId) { + $result = $result >> 8; + } + $max = max($max, $result); + } + } + return $max; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php b/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php new file mode 100644 index 00000000000..81530b56725 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php @@ -0,0 +1,162 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Utility methods for implementing logic that moves data across shards + */ +class CrossShardMoveHelper { + public function __construct( + private ShardConnectionManager $connectionManager, + ) { + } + + public function getConnection(ShardDefinition $shardDefinition, int $shardKey): IDBConnection { + return $this->connectionManager->getConnection($shardDefinition, $shardDefinition->getShardForKey($shardKey)); + } + + /** + * Update the shard key of a set of rows, moving them to a different shard if needed + * + * @param ShardDefinition $shardDefinition + * @param string $table + * @param string $shardColumn + * @param int $sourceShardKey + * @param int $targetShardKey + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function moveCrossShards(ShardDefinition $shardDefinition, string $table, string $shardColumn, int $sourceShardKey, int $targetShardKey, string $primaryColumn, array $primaryKeys): void { + $sourceShard = $shardDefinition->getShardForKey($sourceShardKey); + $targetShard = $shardDefinition->getShardForKey($targetShardKey); + $sourceConnection = $this->connectionManager->getConnection($shardDefinition, $sourceShard); + if ($sourceShard === $targetShard) { + $this->updateItems($sourceConnection, $table, $shardColumn, $targetShardKey, $primaryColumn, $primaryKeys); + + return; + } + $targetConnection = $this->connectionManager->getConnection($shardDefinition, $targetShard); + + $sourceItems = $this->loadItems($sourceConnection, $table, $primaryColumn, $primaryKeys); + foreach ($sourceItems as &$sourceItem) { + $sourceItem[$shardColumn] = $targetShardKey; + } + if (!$sourceItems) { + return; + } + + $sourceConnection->beginTransaction(); + $targetConnection->beginTransaction(); + try { + $this->saveItems($targetConnection, $table, $sourceItems); + $this->deleteItems($sourceConnection, $table, $primaryColumn, $primaryKeys); + + $targetConnection->commit(); + $sourceConnection->commit(); + } catch (\Exception $e) { + $sourceConnection->rollback(); + $targetConnection->rollback(); + throw $e; + } + } + + /** + * Load rows from a table to move + * + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return array[] + */ + public function loadItems(IDBConnection $connection, string $table, string $primaryColumn, array $primaryKeys): array { + $query = $connection->getQueryBuilder(); + $query->select('*') + ->from($table) + ->where($query->expr()->in($primaryColumn, $query->createParameter('keys'))); + + $chunks = array_chunk($primaryKeys, 1000); + + $results = []; + foreach ($chunks as $chunk) { + $query->setParameter('keys', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $results = array_merge($results, $query->execute()->fetchAll()); + } + + return $results; + } + + /** + * Save modified rows + * + * @param IDBConnection $connection + * @param string $table + * @param array[] $items + * @return void + */ + public function saveItems(IDBConnection $connection, string $table, array $items): void { + if (count($items) === 0) { + return; + } + $query = $connection->getQueryBuilder(); + $query->insert($table); + foreach ($items[0] as $column => $value) { + $query->setValue($column, $query->createParameter($column)); + } + + foreach ($items as $item) { + foreach ($item as $column => $value) { + if (is_int($column)) { + $query->setParameter($column, $value, IQueryBuilder::PARAM_INT); + } else { + $query->setParameter($column, $value); + } + } + $query->executeStatement(); + } + } + + /** + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function updateItems(IDBConnection $connection, string $table, string $shardColumn, int $targetShardKey, string $primaryColumn, array $primaryKeys): void { + $query = $connection->getQueryBuilder(); + $query->update($table) + ->set($shardColumn, $query->createNamedParameter($targetShardKey, IQueryBuilder::PARAM_INT)) + ->where($query->expr()->in($primaryColumn, $query->createNamedParameter($primaryKeys, IQueryBuilder::PARAM_INT_ARRAY))); + $query->executeQuery()->fetchAll(); + } + + /** + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function deleteItems(IDBConnection $connection, string $table, string $primaryColumn, array $primaryKeys): void { + $query = $connection->getQueryBuilder(); + $query->delete($table) + ->where($query->expr()->in($primaryColumn, $query->createParameter('keys'))); + $chunks = array_chunk($primaryKeys, 1000); + + foreach ($chunks as $chunk) { + $query->setParameter('keys', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php b/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php new file mode 100644 index 00000000000..af778489a2d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php @@ -0,0 +1,21 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Map string key to an int-range by hashing the key + */ +class HashShardMapper implements IShardMapper { + public function getShardForKey(int $key, int $count): int { + $int = unpack('L', substr(md5((string)$key, true), 0, 4))[1]; + return $int % $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php b/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php new file mode 100644 index 00000000000..733a6acaf9d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php @@ -0,0 +1,29 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Sharded; + +/** + * Queries on sharded table has the following limitations: + * + * 1. Either the shard key (e.g. "storage") or primary key (e.g. "fileid") must be mentioned in the query. + * Or the query must be explicitly marked as running across all shards. + * + * For queries where it isn't possible to set one of these keys in the query normally, you can set it using `hintShardKey` + * + * 2. Insert statements must always explicitly set the shard key + * 3. A query on a sharded table is not allowed to join on the same table + * 4. Right joins are not allowed on sharded tables + * 5. Updating the shard key where the new shard key maps to a different shard is not allowed + * + * Moving rows to a different shard needs to be implemented manually. `CrossShardMoveHelper` provides + * some tools to help make this easier. + */ +class InvalidShardedQueryException extends \Exception { + +} diff --git a/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php b/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php new file mode 100644 index 00000000000..a5694b06507 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php @@ -0,0 +1,20 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Map string key to an int-range by hashing the key + */ +class RoundRobinShardMapper implements IShardMapper { + public function getShardForKey(int $key, int $count): int { + return $key % $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php new file mode 100644 index 00000000000..74358e3ca96 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php @@ -0,0 +1,52 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\ConnectionAdapter; +use OC\DB\ConnectionFactory; +use OC\SystemConfig; +use OCP\IDBConnection; + +/** + * Keeps track of the db connections to the various shards + */ +class ShardConnectionManager { + /** @var array<string, IDBConnection> */ + private array $connections = []; + + public function __construct( + private SystemConfig $config, + private ConnectionFactory $factory, + ) { + } + + public function getConnection(ShardDefinition $shardDefinition, int $shard): IDBConnection { + $connectionKey = $shardDefinition->table . '_' . $shard; + + if (isset($this->connections[$connectionKey])) { + return $this->connections[$connectionKey]; + } + + if ($shard === ShardDefinition::MIGRATION_SHARD) { + $this->connections[$connectionKey] = \OC::$server->get(IDBConnection::class); + } elseif (isset($shardDefinition->shards[$shard])) { + $this->connections[$connectionKey] = $this->createConnection($shardDefinition->shards[$shard]); + } else { + throw new \InvalidArgumentException("invalid shard key $shard only " . count($shardDefinition->shards) . ' configured'); + } + + return $this->connections[$connectionKey]; + } + + private function createConnection(array $shardConfig): IDBConnection { + $shardConfig['sharding'] = []; + $type = $this->config->getValue('dbtype', 'sqlite'); + return new ConnectionAdapter($this->factory->getConnection($type, $shardConfig)); + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php new file mode 100644 index 00000000000..4f98079d92d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php @@ -0,0 +1,80 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Configuration for a shard setup + */ +class ShardDefinition { + // we reserve the bottom byte of the primary key for the initial shard, so the total shard count is limited to what we can fit there + // additionally, shard id 255 is reserved for migration purposes + public const MAX_SHARDS = 255; + public const MIGRATION_SHARD = 255; + + public const PRIMARY_KEY_MASK = 0x7F_FF_FF_FF_FF_FF_FF_00; + public const PRIMARY_KEY_SHARD_MASK = 0x00_00_00_00_00_00_00_FF; + // since we reserve 1 byte for the shard index, we only have 56 bits of primary key space + public const MAX_PRIMARY_KEY = PHP_INT_MAX >> 8; + + /** + * @param string $table + * @param string $primaryKey + * @param string $shardKey + * @param string[] $companionKeys + * @param IShardMapper $shardMapper + * @param string[] $companionTables + * @param array $shards + */ + public function __construct( + public string $table, + public string $primaryKey, + public array $companionKeys, + public string $shardKey, + public IShardMapper $shardMapper, + public array $companionTables, + public array $shards, + public int $fromFileId, + public int $fromStorageId, + ) { + if (count($this->shards) >= self::MAX_SHARDS) { + throw new \Exception('Only allowed maximum of ' . self::MAX_SHARDS . ' shards allowed'); + } + } + + public function hasTable(string $table): bool { + if ($this->table === $table) { + return true; + } + return in_array($table, $this->companionTables); + } + + public function getShardForKey(int $key): int { + if ($key < $this->fromStorageId) { + return self::MIGRATION_SHARD; + } + return $this->shardMapper->getShardForKey($key, count($this->shards)); + } + + /** + * @return list<int> + */ + public function getAllShards(): array { + if ($this->fromStorageId !== 0) { + return array_merge(array_keys($this->shards), [self::MIGRATION_SHARD]); + } else { + return array_keys($this->shards); + } + } + + public function isKey(string $column): bool { + return $column === $this->primaryKey || in_array($column, $this->companionKeys); + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php new file mode 100644 index 00000000000..25e2a3d5f2d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php @@ -0,0 +1,200 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\ArrayResult; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; + +/** + * Logic for running a query across a number of shards, combining the results + */ +class ShardQueryRunner { + public function __construct( + private ShardConnectionManager $shardConnectionManager, + private ShardDefinition $shardDefinition, + ) { + } + + /** + * Get the shards for a specific query or null if the shards aren't known in advance + * + * @param bool $allShards + * @param int[] $shardKeys + * @return null|int[] + */ + public function getShards(bool $allShards, array $shardKeys): ?array { + if ($allShards) { + return $this->shardDefinition->getAllShards(); + } + $allConfiguredShards = $this->shardDefinition->getAllShards(); + if (count($allConfiguredShards) === 1) { + return $allConfiguredShards; + } + if (empty($shardKeys)) { + return null; + } + $shards = array_map(function ($shardKey) { + return $this->shardDefinition->getShardForKey((int)$shardKey); + }, $shardKeys); + return array_values(array_unique($shards)); + } + + /** + * Try to get the shards that the keys are likely to be in, based on the shard the row was created + * + * @param int[] $primaryKeys + * @return int[] + */ + private function getLikelyShards(array $primaryKeys): array { + $shards = []; + foreach ($primaryKeys as $primaryKey) { + if ($primaryKey < $this->shardDefinition->fromFileId && !in_array(ShardDefinition::MIGRATION_SHARD, $shards)) { + $shards[] = ShardDefinition::MIGRATION_SHARD; + } + $encodedShard = $primaryKey & ShardDefinition::PRIMARY_KEY_SHARD_MASK; + if ($encodedShard < count($this->shardDefinition->shards) && !in_array($encodedShard, $shards)) { + $shards[] = $encodedShard; + } + } + return $shards; + } + + /** + * Execute a SELECT statement across the configured shards + * + * @param IQueryBuilder $query + * @param bool $allShards + * @param int[] $shardKeys + * @param int[] $primaryKeys + * @param array{column: string, order: string}[] $sortList + * @param int|null $limit + * @param int|null $offset + * @return IResult + */ + public function executeQuery( + IQueryBuilder $query, + bool $allShards, + array $shardKeys, + array $primaryKeys, + ?array $sortList = null, + ?int $limit = null, + ?int $offset = null, + ): IResult { + $shards = $this->getShards($allShards, $shardKeys); + $results = []; + if ($shards && count($shards) === 1) { + // trivial case + return $query->executeQuery($this->shardConnectionManager->getConnection($this->shardDefinition, $shards[0])); + } + // we have to emulate limit and offset, so we select offset+limit from all shards to ensure we have enough rows + // and then filter them down after we merged the results + if ($limit !== null && $offset !== null) { + $query->setMaxResults($limit + $offset); + } + + if ($shards) { + // we know exactly what shards we need to query + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $subResult = $query->executeQuery($shardConnection); + $results = array_merge($results, $subResult->fetchAll()); + $subResult->closeCursor(); + } + } else { + // we don't know for sure what shards we need to query, + // we first try the shards that are "likely" to have the rows we want, based on the shard that the row was + // originally created in. If we then still haven't found all rows we try the rest of the shards + $likelyShards = $this->getLikelyShards($primaryKeys); + $unlikelyShards = array_diff($this->shardDefinition->getAllShards(), $likelyShards); + $shards = array_merge($likelyShards, $unlikelyShards); + + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $subResult = $query->executeQuery($shardConnection); + $rows = $subResult->fetchAll(); + $results = array_merge($results, $rows); + $subResult->closeCursor(); + + if (count($rows) >= count($primaryKeys)) { + // we have all the rows we're looking for + break; + } + } + } + + if ($sortList) { + usort($results, function ($a, $b) use ($sortList) { + foreach ($sortList as $sort) { + $valueA = $a[$sort['column']] ?? null; + $valueB = $b[$sort['column']] ?? null; + $cmp = $valueA <=> $valueB; + if ($cmp === 0) { + continue; + } + if ($sort['order'] === 'DESC') { + $cmp = -$cmp; + } + return $cmp; + } + }); + } + + if ($limit !== null && $offset !== null) { + $results = array_slice($results, $offset, $limit); + } elseif ($limit !== null) { + $results = array_slice($results, 0, $limit); + } elseif ($offset !== null) { + $results = array_slice($results, $offset); + } + + return new ArrayResult($results); + } + + /** + * Execute an UPDATE or DELETE statement + * + * @param IQueryBuilder $query + * @param bool $allShards + * @param int[] $shardKeys + * @param int[] $primaryKeys + * @return int + * @throws \OCP\DB\Exception + */ + public function executeStatement(IQueryBuilder $query, bool $allShards, array $shardKeys, array $primaryKeys): int { + if ($query->getType() === \Doctrine\DBAL\Query\QueryBuilder::INSERT) { + throw new \Exception('insert queries need special handling'); + } + + $shards = $this->getShards($allShards, $shardKeys); + $maxCount = count($primaryKeys); + if ($shards && count($shards) === 1) { + return $query->executeStatement($this->shardConnectionManager->getConnection($this->shardDefinition, $shards[0])); + } elseif ($shards) { + $maxCount = PHP_INT_MAX; + } else { + // sort the likely shards before the rest, similar logic to `self::executeQuery` + $likelyShards = $this->getLikelyShards($primaryKeys); + $unlikelyShards = array_diff($this->shardDefinition->getAllShards(), $likelyShards); + $shards = array_merge($likelyShards, $unlikelyShards); + } + + $count = 0; + + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $count += $query->executeStatement($shardConnection); + + if ($count >= $maxCount) { + break; + } + } + return $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php new file mode 100644 index 00000000000..04082f76ae8 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php @@ -0,0 +1,407 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\ExtendedQueryBuilder; +use OC\DB\QueryBuilder\Parameter; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * A special query builder that automatically distributes queries over multiple database shards. + * + * This relies on `PartitionedQueryBuilder` to handle splitting of parts of the query that touch the sharded tables + * from the non-sharded tables. So the query build here should only either touch only sharded table or only non-sharded tables. + * + * Most of the logic in this class is concerned with extracting either the shard key (e.g. "storage") or primary key (e.g. "fileid") + * from the query. The logic for actually running the query across the shards is mostly delegated to `ShardQueryRunner`. + */ +class ShardedQueryBuilder extends ExtendedQueryBuilder { + private array $shardKeys = []; + private array $primaryKeys = []; + private ?ShardDefinition $shardDefinition = null; + /** @var bool Run the query across all shards */ + private bool $allShards = false; + private ?string $insertTable = null; + private mixed $lastInsertId = null; + private ?IDBConnection $lastInsertConnection = null; + private ?int $updateShardKey = null; + private ?int $limit = null; + private ?int $offset = null; + /** @var array{column: string, order: string}[] */ + private array $sortList = []; + private string $mainTable = ''; + + public function __construct( + IQueryBuilder $builder, + protected array $shardDefinitions, + protected ShardConnectionManager $shardConnectionManager, + protected AutoIncrementHandler $autoIncrementHandler, + ) { + parent::__construct($builder); + } + + public function getShardKeys(): array { + return $this->getKeyValues($this->shardKeys); + } + + public function getPrimaryKeys(): array { + return $this->getKeyValues($this->primaryKeys); + } + + private function getKeyValues(array $keys): array { + $values = []; + foreach ($keys as $key) { + $values = array_merge($values, $this->getKeyValue($key)); + } + return array_values(array_unique($values)); + } + + private function getKeyValue($value): array { + if ($value instanceof Parameter) { + $value = (string)$value; + } + if (is_string($value) && str_starts_with($value, ':')) { + $param = $this->getParameter(substr($value, 1)); + if (is_array($param)) { + return $param; + } else { + return [$param]; + } + } elseif ($value !== null) { + return [$value]; + } else { + return []; + } + } + + public function where(...$predicates) { + return $this->andWhere(...$predicates); + } + + public function andWhere(...$where) { + if ($where) { + foreach ($where as $predicate) { + $this->tryLoadShardKey($predicate); + } + parent::andWhere(...$where); + } + return $this; + } + + private function tryLoadShardKey($predicate): void { + if (!$this->shardDefinition) { + return; + } + if ($keys = $this->tryExtractShardKeys($predicate, $this->shardDefinition->shardKey)) { + $this->shardKeys += $keys; + } + if ($keys = $this->tryExtractShardKeys($predicate, $this->shardDefinition->primaryKey)) { + $this->primaryKeys += $keys; + } + foreach ($this->shardDefinition->companionKeys as $companionKey) { + if ($keys = $this->tryExtractShardKeys($predicate, $companionKey)) { + $this->primaryKeys += $keys; + } + } + } + + /** + * @param $predicate + * @param string $column + * @return string[] + */ + private function tryExtractShardKeys($predicate, string $column): array { + if ($predicate instanceof CompositeExpression) { + $values = []; + foreach ($predicate->getParts() as $part) { + $partValues = $this->tryExtractShardKeys($part, $column); + // for OR expressions, we can only rely on the predicate if all parts contain the comparison + if ($predicate->getType() === CompositeExpression::TYPE_OR && !$partValues) { + return []; + } + $values = array_merge($values, $partValues); + } + return $values; + } + $predicate = (string)$predicate; + // expect a condition in the form of 'alias1.column1 = placeholder' or 'alias1.column1 in placeholder' + if (substr_count($predicate, ' ') > 2) { + return []; + } + if (str_contains($predicate, ' = ')) { + $parts = explode(' = ', $predicate); + if ($parts[0] === "`{$column}`" || str_ends_with($parts[0], "`.`{$column}`")) { + return [$parts[1]]; + } else { + return []; + } + } + + if (str_contains($predicate, ' IN ')) { + $parts = explode(' IN ', $predicate); + if ($parts[0] === "`{$column}`" || str_ends_with($parts[0], "`.`{$column}`")) { + return [trim(trim($parts[1], '('), ')')]; + } else { + return []; + } + } + + return []; + } + + public function set($key, $value) { + if ($this->shardDefinition && $key === $this->shardDefinition->shardKey) { + $updateShardKey = $value; + } + return parent::set($key, $value); + } + + public function setValue($column, $value) { + if ($this->shardDefinition) { + if ($this->shardDefinition->isKey($column)) { + $this->primaryKeys[] = $value; + } + if ($column === $this->shardDefinition->shardKey) { + $this->shardKeys[] = $value; + } + } + return parent::setValue($column, $value); + } + + public function values(array $values) { + foreach ($values as $column => $value) { + $this->setValue($column, $value); + } + return $this; + } + + private function actOnTable(string $table): void { + $this->mainTable = $table; + foreach ($this->shardDefinitions as $shardDefinition) { + if ($shardDefinition->hasTable($table)) { + $this->shardDefinition = $shardDefinition; + } + } + } + + public function from($from, $alias = null) { + if (is_string($from) && $from) { + $this->actOnTable($from); + } + return parent::from($from, $alias); + } + + public function update($update = null, $alias = null) { + if (is_string($update) && $update) { + $this->actOnTable($update); + } + return parent::update($update, $alias); + } + + public function insert($insert = null) { + if (is_string($insert) && $insert) { + $this->insertTable = $insert; + $this->actOnTable($insert); + } + return parent::insert($insert); + } + + public function delete($delete = null, $alias = null) { + if (is_string($delete) && $delete) { + $this->actOnTable($delete); + } + return parent::delete($delete, $alias); + } + + private function checkJoin(string $table): void { + if ($this->shardDefinition) { + if ($table === $this->mainTable) { + throw new InvalidShardedQueryException("Sharded query on {$this->mainTable} isn't allowed to join on itself"); + } + if (!$this->shardDefinition->hasTable($table)) { + // this generally shouldn't happen as the partitioning logic should prevent this + // but the check is here just in case + throw new InvalidShardedQueryException("Sharded query on {$this->shardDefinition->table} isn't allowed to join on $table"); + } + } + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkJoin($join); + return parent::innerJoin($fromAlias, $join, $alias, $condition); + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkJoin($join); + return parent::leftJoin($fromAlias, $join, $alias, $condition); + } + + public function rightJoin($fromAlias, $join, $alias, $condition = null) { + if ($this->shardDefinition) { + throw new InvalidShardedQueryException("Sharded query on {$this->shardDefinition->table} isn't allowed to right join"); + } + return parent::rightJoin($fromAlias, $join, $alias, $condition); + } + + public function join($fromAlias, $join, $alias, $condition = null) { + return $this->innerJoin($fromAlias, $join, $alias, $condition); + } + + public function setMaxResults($maxResults) { + if ($maxResults > 0) { + $this->limit = (int)$maxResults; + } + return parent::setMaxResults($maxResults); + } + + public function setFirstResult($firstResult) { + if ($firstResult > 0) { + $this->offset = (int)$firstResult; + } + if ($this->shardDefinition && count($this->shardDefinition->shards) > 1) { + // we have to emulate offset + return $this; + } else { + return parent::setFirstResult($firstResult); + } + } + + public function addOrderBy($sort, $order = null) { + $this->registerOrder((string)$sort, (string)$order ?? 'ASC'); + return parent::addOrderBy($sort, $order); + } + + public function orderBy($sort, $order = null) { + $this->sortList = []; + $this->registerOrder((string)$sort, (string)$order ?? 'ASC'); + return parent::orderBy($sort, $order); + } + + private function registerOrder(string $column, string $order): void { + // handle `mime + 0` and similar by just sorting on the first part of the expression + [$column] = explode(' ', $column); + $column = trim($column, '`'); + $this->sortList[] = [ + 'column' => $column, + 'order' => strtoupper($order), + ]; + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if ($overwrite) { + $this->primaryKeys = []; + $this->shardKeys = []; + } + if ($this->shardDefinition?->isKey($column)) { + $this->primaryKeys[] = $value; + } + if ($column === $this->shardDefinition?->shardKey) { + $this->shardKeys[] = $value; + } + return $this; + } + + public function runAcrossAllShards(): self { + $this->allShards = true; + return $this; + } + + /** + * @throws InvalidShardedQueryException + */ + public function validate(): void { + if ($this->shardDefinition && $this->insertTable) { + if ($this->allShards) { + throw new InvalidShardedQueryException("Can't insert across all shards"); + } + if (empty($this->getShardKeys())) { + throw new InvalidShardedQueryException("Can't insert without shard key"); + } + } + if ($this->shardDefinition && !$this->allShards) { + if (empty($this->getShardKeys()) && empty($this->getPrimaryKeys())) { + throw new InvalidShardedQueryException('No shard key or primary key set for query'); + } + } + if ($this->shardDefinition && $this->updateShardKey) { + $newShardKey = $this->getKeyValue($this->updateShardKey); + $oldShardKeys = $this->getShardKeys(); + if (count($newShardKey) !== 1) { + throw new InvalidShardedQueryException("Can't set shard key to an array"); + } + $newShardKey = current($newShardKey); + if (empty($oldShardKeys)) { + throw new InvalidShardedQueryException("Can't update without shard key"); + } + $oldShards = array_values(array_unique(array_map(function ($shardKey) { + return $this->shardDefinition->getShardForKey((int)$shardKey); + }, $oldShardKeys))); + $newShard = $this->shardDefinition->getShardForKey((int)$newShardKey); + if ($oldShards === [$newShard]) { + throw new InvalidShardedQueryException('Update statement would move rows to a different shard'); + } + } + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + $this->validate(); + if ($this->shardDefinition) { + $runner = new ShardQueryRunner($this->shardConnectionManager, $this->shardDefinition); + return $runner->executeQuery($this->builder, $this->allShards, $this->getShardKeys(), $this->getPrimaryKeys(), $this->sortList, $this->limit, $this->offset); + } + return parent::executeQuery($connection); + } + + public function executeStatement(?IDBConnection $connection = null): int { + $this->validate(); + if ($this->shardDefinition) { + $runner = new ShardQueryRunner($this->shardConnectionManager, $this->shardDefinition); + if ($this->insertTable) { + $shards = $runner->getShards($this->allShards, $this->getShardKeys()); + if (!$shards) { + throw new InvalidShardedQueryException("Can't insert without shard key"); + } + $count = 0; + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + if (!$this->primaryKeys && $this->shardDefinition->table === $this->insertTable) { + $id = $this->autoIncrementHandler->getNextPrimaryKey($this->shardDefinition, $shard); + parent::setValue($this->shardDefinition->primaryKey, $this->createParameter('__generated_primary_key')); + $this->setParameter('__generated_primary_key', $id, self::PARAM_INT); + $this->lastInsertId = $id; + } + $count += parent::executeStatement($shardConnection); + + $this->lastInsertConnection = $shardConnection; + } + return $count; + } else { + return $runner->executeStatement($this->builder, $this->allShards, $this->getShardKeys(), $this->getPrimaryKeys()); + } + } + return parent::executeStatement($connection); + } + + public function getLastInsertId(): int { + if ($this->lastInsertId) { + return $this->lastInsertId; + } + if ($this->lastInsertConnection) { + $table = $this->builder->prefixTableName($this->insertTable); + return $this->lastInsertConnection->lastInsertId($table); + } else { + return parent::getLastInsertId(); + } + } + + +} diff --git a/lib/private/DB/ResultAdapter.php b/lib/private/DB/ResultAdapter.php index 176de5b45ea..95a7620e0ff 100644 --- a/lib/private/DB/ResultAdapter.php +++ b/lib/private/DB/ResultAdapter.php @@ -1,27 +1,11 @@ <?php -/* - * @copyright 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @author 2021 Christoph Wurst <christoph@winzerhof-wurst.at> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ namespace OC\DB; use Doctrine\DBAL\Result; @@ -32,7 +16,6 @@ use PDO; * Adapts DBAL 2.6 API for DBAL 3.x for backwards compatibility of a leaked type */ class ResultAdapter implements IResult { - /** @var Result */ private $inner; @@ -47,14 +30,21 @@ class ResultAdapter implements IResult { } public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { - return $this->inner->fetch($fetchMode); + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->inner->fetchAssociative(), + PDO::FETCH_NUM => $this->inner->fetchNumeric(), + PDO::FETCH_COLUMN => $this->inner->fetchOne(), + default => throw new \Exception('Fetch mode needs to be assoc, num or column.'), + }; } public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { - if ($fetchMode !== PDO::FETCH_ASSOC && $fetchMode !== PDO::FETCH_NUM && $fetchMode !== PDO::FETCH_COLUMN) { - throw new \Exception('Fetch mode needs to be assoc, num or column.'); - } - return $this->inner->fetchAll($fetchMode); + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->inner->fetchAllAssociative(), + PDO::FETCH_NUM => $this->inner->fetchAllNumeric(), + PDO::FETCH_COLUMN => $this->inner->fetchFirstColumn(), + default => throw new \Exception('Fetch mode needs to be assoc, num or column.'), + }; } public function fetchColumn($columnIndex = 0) { diff --git a/lib/private/DB/SQLiteMigrator.php b/lib/private/DB/SQLiteMigrator.php index 24b6c02b31c..6be17625476 100644 --- a/lib/private/DB/SQLiteMigrator.php +++ b/lib/private/DB/SQLiteMigrator.php @@ -1,53 +1,26 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * @author Victor Dubiniuk <dubiniuk@owncloud.com> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\DBAL\Schema\Schema; -use Doctrine\DBAL\Types\BigIntType; -use Doctrine\DBAL\Types\Type; class SQLiteMigrator extends Migrator { - /** * @param Schema $targetSchema * @param \Doctrine\DBAL\Connection $connection * @return \Doctrine\DBAL\Schema\SchemaDiff */ protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) { - $platform = $connection->getDatabasePlatform(); - $platform->registerDoctrineTypeMapping('tinyint unsigned', 'integer'); - $platform->registerDoctrineTypeMapping('smallint unsigned', 'integer'); - $platform->registerDoctrineTypeMapping('varchar ', 'string'); - - // with sqlite autoincrement columns is of type integer foreach ($targetSchema->getTables() as $table) { foreach ($table->getColumns() as $column) { - if ($column->getType() instanceof BigIntType && $column->getAutoincrement()) { - $column->setType(Type::getType('integer')); + // column comments are not supported on SQLite + if ($column->getComment() !== null) { + $column->setComment(null); } } } diff --git a/lib/private/DB/SQLiteSessionInit.php b/lib/private/DB/SQLiteSessionInit.php index 924a3b2758c..5fe0cb3abf6 100644 --- a/lib/private/DB/SQLiteSessionInit.php +++ b/lib/private/DB/SQLiteSessionInit.php @@ -1,29 +1,10 @@ <?php + /** - * @copyright Copyright (c) 2016, ownCloud, Inc. - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Morris Jobke <hey@morrisjobke.de> - * @author Robin Appelman <robin@icewind.nl> - * @author Roeland Jago Douma <roeland@famdouma.nl> - * @author Thomas Müller <thomas.mueller@tmit.eu> - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see <http://www.gnu.org/licenses/> - * + * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-FileCopyrightText: 2016 ownCloud, Inc. + * SPDX-License-Identifier: AGPL-3.0-only */ - namespace OC\DB; use Doctrine\Common\EventSubscriber; diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index 20ae5b6faa6..0d5b2040513 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -1,34 +1,17 @@ <?php + /** - * @copyright Copyright (c) 2017 Joas Schilling <coding@schilljs.com> - * - * @author Christoph Wurst <christoph@winzerhof-wurst.at> - * @author Joas Schilling <coding@schilljs.com> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; +use Doctrine\DBAL\Exception; +use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\Schema; use OCP\DB\ISchemaWrapper; class SchemaWrapper implements ISchemaWrapper { - /** @var Connection */ protected $connection; @@ -38,9 +21,13 @@ class SchemaWrapper implements ISchemaWrapper { /** @var array */ protected $tablesToDelete = []; - public function __construct(Connection $connection) { + public function __construct(Connection $connection, ?Schema $schema = null) { $this->connection = $connection; - $this->schema = $this->connection->createSchema(); + if ($schema) { + $this->schema = $schema; + } else { + $this->schema = $this->connection->createSchema(); + } } public function getWrappedSchema() { @@ -50,6 +37,9 @@ class SchemaWrapper implements ISchemaWrapper { public function performDropTableCalls() { foreach ($this->tablesToDelete as $tableName => $true) { $this->connection->dropTable($tableName); + foreach ($this->connection->getShardConnections() as $shardConnection) { + $shardConnection->dropTable($tableName); + } unset($this->tablesToDelete[$tableName]); } } @@ -62,7 +52,7 @@ class SchemaWrapper implements ISchemaWrapper { public function getTableNamesWithoutPrefix() { $tableNames = $this->schema->getTableNames(); return array_map(function ($tableName) { - if (strpos($tableName, $this->connection->getPrefix()) === 0) { + if (str_starts_with($tableName, $this->connection->getPrefix())) { return substr($tableName, strlen($this->connection->getPrefix())); } @@ -130,4 +120,15 @@ class SchemaWrapper implements ISchemaWrapper { public function getTables() { return $this->schema->getTables(); } + + /** + * Gets the DatabasePlatform for the database. + * + * @return AbstractPlatform + * + * @throws Exception + */ + public function getDatabasePlatform() { + return $this->connection->getDatabasePlatform(); + } } diff --git a/lib/private/DB/SetTransactionIsolationLevel.php b/lib/private/DB/SetTransactionIsolationLevel.php index eb15bf11fbe..cd18c4255e3 100644 --- a/lib/private/DB/SetTransactionIsolationLevel.php +++ b/lib/private/DB/SetTransactionIsolationLevel.php @@ -3,32 +3,16 @@ declare(strict_types=1); /** - * @copyright Copyright (c) 2019 Daniel Kesselberg <mail@danielkesselberg.de> - * - * @author Daniel Kesselberg <mail@danielkesselberg.de> - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * + * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later */ - namespace OC\DB; use Doctrine\Common\EventSubscriber; +use Doctrine\DBAL\Connections\PrimaryReadReplicaConnection; use Doctrine\DBAL\Event\ConnectionEventArgs; use Doctrine\DBAL\Events; +use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\TransactionIsolationLevel; class SetTransactionIsolationLevel implements EventSubscriber { @@ -37,7 +21,13 @@ class SetTransactionIsolationLevel implements EventSubscriber { * @return void */ public function postConnect(ConnectionEventArgs $args) { - $args->getConnection()->setTransactionIsolation(TransactionIsolationLevel::READ_COMMITTED); + $connection = $args->getConnection(); + if ($connection instanceof PrimaryReadReplicaConnection && $connection->isConnectedToPrimary()) { + $connection->setTransactionIsolation(TransactionIsolationLevel::READ_COMMITTED); + if ($connection->getDatabasePlatform() instanceof MySQLPlatform) { + $connection->executeStatement('SET SESSION AUTOCOMMIT=1'); + } + } } public function getSubscribedEvents() { |