diff options
Diffstat (limited to 'lib/private/DB')
48 files changed, 3171 insertions, 318 deletions
diff --git a/lib/private/DB/Adapter.php b/lib/private/DB/Adapter.php index b5be14e5dc6..8f1b8e6d75f 100644 --- a/lib/private/DB/Adapter.php +++ b/lib/private/DB/Adapter.php @@ -32,7 +32,7 @@ class Adapter { * @throws Exception */ public function lastInsertId($table) { - return (int) $this->conn->realLastInsertId($table); + return (int)$this->conn->realLastInsertId($table); } /** @@ -46,13 +46,12 @@ class Adapter { /** * Create an exclusive read+write lock on a table * - * @param string $tableName * @throws Exception * @since 9.1.0 */ - public function lockTable($tableName) { + public function lockTable(string $tableName) { $this->conn->beginTransaction(); - $this->conn->executeUpdate('LOCK TABLE `' .$tableName . '` IN EXCLUSIVE MODE'); + $this->conn->executeUpdate('LOCK TABLE `' . $tableName . '` IN EXCLUSIVE MODE'); } /** @@ -73,19 +72,21 @@ class Adapter { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 */ public function insertIfNotExist($table, $input, ?array $compare = null) { - if (empty($compare)) { - $compare = array_keys($input); - } - $query = 'INSERT INTO `' .$table . '` (`' - . implode('`,`', array_keys($input)) . '`) SELECT ' - . str_repeat('?,', count($input) - 1).'? ' // Is there a prettier alternative? + $compare = $compare ?: array_keys($input); + + // Prepare column names and generate placeholders + $columns = '`' . implode('`,`', array_keys($input)) . '`'; + $placeholders = implode(', ', array_fill(0, count($input), '?')); + + $query = 'INSERT INTO `' . $table . '` (' . $columns . ') ' + . 'SELECT ' . $placeholders . ' ' . 'FROM `' . $table . '` WHERE '; $inserts = array_values($input); @@ -104,10 +105,9 @@ class Adapter { try { return $this->conn->executeUpdate($query, $inserts); } catch (UniqueConstraintViolationException $e) { - // if this is thrown then a concurrent insert happened between the insert and the sub-select in the insert, that should have avoided it - // it's fine to ignore this then - // - // more discussions about this can be found at https://github.com/nextcloud/server/pull/12315 + // This exception indicates a concurrent insert happened between + // the insert and the sub-select in the insert, which is safe to ignore. + // More details: https://github.com/nextcloud/server/pull/12315 return 0; } } diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index 598dbc4de20..63c75607379 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -15,7 +15,7 @@ class AdapterMySQL extends Adapter { * @param string $tableName */ public function lockTable($tableName) { - $this->conn->executeUpdate('LOCK TABLES `' .$tableName . '` WRITE'); + $this->conn->executeUpdate('LOCK TABLES `' . $tableName . '` WRITE'); } public function unlockTable() { diff --git a/lib/private/DB/AdapterSqlite.php b/lib/private/DB/AdapterSqlite.php index 24274cbcda6..aeadf55ecf7 100644 --- a/lib/private/DB/AdapterSqlite.php +++ b/lib/private/DB/AdapterSqlite.php @@ -38,8 +38,8 @@ class AdapterSqlite extends Adapter { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws \Doctrine\DBAL\Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 @@ -50,7 +50,7 @@ class AdapterSqlite extends Adapter { } $fieldList = '`' . implode('`,`', array_keys($input)) . '`'; $query = "INSERT INTO `$table` ($fieldList) SELECT " - . str_repeat('?,', count($input) - 1).'? ' + . str_repeat('?,', count($input) - 1) . '? ' . " WHERE NOT EXISTS (SELECT 1 FROM `$table` WHERE "; $inserts = array_values($input); diff --git a/lib/private/DB/ArrayResult.php b/lib/private/DB/ArrayResult.php new file mode 100644 index 00000000000..b567ad23d57 --- /dev/null +++ b/lib/private/DB/ArrayResult.php @@ -0,0 +1,74 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB; + +use OCP\DB\IResult; +use PDO; + +/** + * Wrap an array or rows into a result interface + */ +class ArrayResult implements IResult { + protected int $count; + + public function __construct( + protected array $rows, + ) { + $this->count = count($this->rows); + } + + public function closeCursor(): bool { + // noop + return true; + } + + public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { + $row = array_shift($this->rows); + if (!$row) { + return false; + } + return match ($fetchMode) { + PDO::FETCH_ASSOC => $row, + PDO::FETCH_NUM => array_values($row), + PDO::FETCH_COLUMN => current($row), + default => throw new \InvalidArgumentException('Fetch mode not supported for array result'), + }; + + } + + public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->rows, + PDO::FETCH_NUM => array_map(function ($row) { + return array_values($row); + }, $this->rows), + PDO::FETCH_COLUMN => array_map(function ($row) { + return current($row); + }, $this->rows), + default => throw new \InvalidArgumentException('Fetch mode not supported for array result'), + }; + } + + public function fetchColumn() { + return $this->fetchOne(); + } + + public function fetchOne() { + $row = $this->fetch(); + if ($row) { + return current($row); + } else { + return false; + } + } + + public function rowCount(): int { + return $this->count; + } +} diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index c886cb20235..f86cbc341a4 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -13,18 +13,32 @@ use Doctrine\DBAL\Cache\QueryCacheProfile; use Doctrine\DBAL\Configuration; use Doctrine\DBAL\Connections\PrimaryReadReplicaConnection; use Doctrine\DBAL\Driver; +use Doctrine\DBAL\Driver\ServerInfoAwareConnection; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Exception\ConnectionLost; +use Doctrine\DBAL\Platforms\MariaDBPlatform; use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\Platforms\OraclePlatform; +use Doctrine\DBAL\Platforms\PostgreSQLPlatform; use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Result; use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Statement; +use OC\DB\QueryBuilder\Partitioned\PartitionedQueryBuilder; +use OC\DB\QueryBuilder\Partitioned\PartitionSplit; use OC\DB\QueryBuilder\QueryBuilder; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\CrossShardMoveHelper; +use OC\DB\QueryBuilder\Sharded\RoundRobinShardMapper; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\Sharded\IShardMapper; use OCP\Diagnostics\IEventLogger; +use OCP\ICacheFactory; +use OCP\IDBConnection; +use OCP\ILogger; use OCP\IRequestId; use OCP\PreConditionNotMetException; use OCP\Profiler\IProfiler; @@ -66,9 +80,35 @@ class Connection extends PrimaryReadReplicaConnection { /** @var array<string, int> */ protected $tableDirtyWrites = []; + protected bool $logDbException = false; + private ?array $transactionBacktrace = null; + protected bool $logRequestId; protected string $requestId; + /** @var array<string, list<string>> */ + protected array $partitions; + /** @var ShardDefinition[] */ + protected array $shards = []; + protected ShardConnectionManager $shardConnectionManager; + protected AutoIncrementHandler $autoIncrementHandler; + protected bool $isShardingEnabled; + + public const SHARD_PRESETS = [ + 'filecache' => [ + 'companion_keys' => [ + 'file_id', + ], + 'companion_tables' => [ + 'filecache_extended', + 'files_metadata', + ], + 'primary_key' => 'fileid', + 'shard_key' => 'storage', + 'table' => 'filecache', + ], + ]; + /** * Initializes a new instance of the Connection class. * @@ -78,7 +118,7 @@ class Connection extends PrimaryReadReplicaConnection { private array $params, Driver $driver, ?Configuration $config = null, - ?EventManager $eventManager = null + ?EventManager $eventManager = null, ) { if (!isset($params['adapter'])) { throw new \Exception('adapter not set'); @@ -92,12 +132,23 @@ class Connection extends PrimaryReadReplicaConnection { parent::__construct($params, $driver, $config, $eventManager); $this->adapter = new $params['adapter']($this); $this->tablePrefix = $params['tablePrefix']; - + $this->isShardingEnabled = isset($this->params['sharding']) && !empty($this->params['sharding']); + + if ($this->isShardingEnabled) { + /** @psalm-suppress InvalidArrayOffset */ + $this->shardConnectionManager = $this->params['shard_connection_manager'] ?? Server::get(ShardConnectionManager::class); + /** @psalm-suppress InvalidArrayOffset */ + $this->autoIncrementHandler = $this->params['auto_increment_handler'] ?? new AutoIncrementHandler( + Server::get(ICacheFactory::class), + $this->shardConnectionManager, + ); + } $this->systemConfig = \OC::$server->getSystemConfig(); $this->clock = Server::get(ClockInterface::class); $this->logger = Server::get(LoggerInterface::class); $this->logRequestId = $this->systemConfig->getValue('db.log_request_id', false); + $this->logDbException = $this->systemConfig->getValue('db.log_exceptions', false); $this->requestId = Server::get(IRequestId::class)->getId(); /** @var \OCP\Profiler\IProfiler */ @@ -110,10 +161,58 @@ class Connection extends PrimaryReadReplicaConnection { $this->_config->setSQLLogger($debugStack); } + /** @var array<string, array{shards: array[], mapper: ?string, from_primary_key: ?int, from_shard_key: ?int}> $shardConfig */ + $shardConfig = $this->params['sharding'] ?? []; + $shardNames = array_keys($shardConfig); + $this->shards = array_map(function (array $config, string $name) { + if (!isset(self::SHARD_PRESETS[$name])) { + throw new \Exception("Shard preset $name not found"); + } + + $shardMapperClass = $config['mapper'] ?? RoundRobinShardMapper::class; + $shardMapper = Server::get($shardMapperClass); + if (!$shardMapper instanceof IShardMapper) { + throw new \Exception("Invalid shard mapper: $shardMapperClass"); + } + return new ShardDefinition( + self::SHARD_PRESETS[$name]['table'], + self::SHARD_PRESETS[$name]['primary_key'], + self::SHARD_PRESETS[$name]['companion_keys'], + self::SHARD_PRESETS[$name]['shard_key'], + $shardMapper, + self::SHARD_PRESETS[$name]['companion_tables'], + $config['shards'], + $config['from_primary_key'] ?? 0, + $config['from_shard_key'] ?? 0, + ); + }, $shardConfig, $shardNames); + $this->shards = array_combine($shardNames, $this->shards); + $this->partitions = array_map(function (ShardDefinition $shard) { + return array_merge([$shard->table], $shard->companionTables); + }, $this->shards); + $this->setNestTransactionsWithSavepoints(true); } /** + * @return IDBConnection[] + */ + public function getShardConnections(): array { + $connections = []; + if ($this->isShardingEnabled) { + foreach ($this->shards as $shardDefinition) { + foreach ($shardDefinition->getAllShards() as $shard) { + if ($shard !== ShardDefinition::MIGRATION_SHARD) { + /** @var ConnectionAdapter $connection */ + $connections[] = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + } + } + } + } + return $connections; + } + + /** * @throws Exception */ public function connect($connectionName = null) { @@ -124,8 +223,6 @@ class Connection extends PrimaryReadReplicaConnection { return parent::connect(); } - $this->lastConnectionCheck[$this->getConnectionName()] = time(); - // Only trigger the event logger for the initial connect call $eventLogger = Server::get(IEventLogger::class); $eventLogger->start('connect:db', 'db connection opened'); @@ -133,6 +230,8 @@ class Connection extends PrimaryReadReplicaConnection { $status = parent::connect(); $eventLogger->end('connect:db'); + $this->lastConnectionCheck[$this->getConnectionName()] = time(); + return $status; } catch (Exception $e) { // throw a new exception to prevent leaking info from the stacktrace @@ -161,18 +260,34 @@ class Connection extends PrimaryReadReplicaConnection { */ public function getQueryBuilder(): IQueryBuilder { $this->queriesBuilt++; - return new QueryBuilder( + + $builder = new QueryBuilder( new ConnectionAdapter($this), $this->systemConfig, $this->logger ); + if ($this->isShardingEnabled && count($this->partitions) > 0) { + $builder = new PartitionedQueryBuilder( + $builder, + $this->shards, + $this->shardConnectionManager, + $this->autoIncrementHandler, + ); + foreach ($this->partitions as $name => $tables) { + $partition = new PartitionSplit($name, $tables); + $builder->addPartition($partition); + } + return $builder; + } else { + return $builder; + } } /** * Gets the QueryBuilder for the connection. * * @return \Doctrine\DBAL\Query\QueryBuilder - * @deprecated please use $this->getQueryBuilder() instead + * @deprecated 8.0.0 please use $this->getQueryBuilder() instead */ public function createQueryBuilder() { $backtrace = $this->getCallerBacktrace(); @@ -185,7 +300,7 @@ class Connection extends PrimaryReadReplicaConnection { * Gets the ExpressionBuilder for the connection. * * @return \Doctrine\DBAL\Query\Expression\ExpressionBuilder - * @deprecated please use $this->getQueryBuilder()->expr() instead + * @deprecated 8.0.0 please use $this->getQueryBuilder()->expr() instead */ public function getExpressionBuilder() { $backtrace = $this->getCallerBacktrace(); @@ -232,10 +347,10 @@ class Connection extends PrimaryReadReplicaConnection { if ($limit === -1 || $limit === null) { $limit = null; } else { - $limit = (int) $limit; + $limit = (int)$limit; } if ($offset !== null) { - $offset = (int) $offset; + $offset = (int)$offset; } if (!is_null($limit)) { $platform = $this->getDatabasePlatform(); @@ -252,10 +367,10 @@ class Connection extends PrimaryReadReplicaConnection { * If the query is parametrized, a prepared statement is used. * If an SQLLogger is configured, the execution is logged. * - * @param string $sql The SQL query to execute. - * @param array $params The parameters to bind to the query, if any. - * @param array $types The types the previous parameters are in. - * @param \Doctrine\DBAL\Cache\QueryCacheProfile|null $qcp The query cache profile, optional. + * @param string $sql The SQL query to execute. + * @param array $params The parameters to bind to the query, if any. + * @param array $types The types the previous parameters are in. + * @param \Doctrine\DBAL\Cache\QueryCacheProfile|null $qcp The query cache profile, optional. * * @return Result The executed statement. * @@ -284,7 +399,7 @@ class Connection extends PrimaryReadReplicaConnection { // Read to a table that has been written to previously // While this might not necessarily mean that we did a read after write it is an indication for a code path to check $this->logger->log( - (int) ($this->systemConfig->getValue('loglevel_dirty_database_queries', null) ?? 0), + (int)($this->systemConfig->getValue('loglevel_dirty_database_queries', null) ?? 0), 'dirty table reads: ' . $sql, [ 'tables' => array_keys($this->tableDirtyWrites), @@ -300,8 +415,13 @@ class Connection extends PrimaryReadReplicaConnection { $sql = $this->finishQuery($sql); $this->queriesExecuted++; - $this->logQueryToFile($sql); - return parent::executeQuery($sql, $params, $types, $qcp); + $this->logQueryToFile($sql, $params); + try { + return parent::executeQuery($sql, $params, $types, $qcp); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } /** @@ -318,10 +438,7 @@ class Connection extends PrimaryReadReplicaConnection { * @throws Exception */ public function executeUpdate(string $sql, array $params = [], array $types = []): int { - $sql = $this->finishQuery($sql); - $this->queriesExecuted++; - $this->logQueryToFile($sql); - return parent::executeUpdate($sql, $params, $types); + return $this->executeStatement($sql, $params, $types); } /** @@ -330,9 +447,9 @@ class Connection extends PrimaryReadReplicaConnection { * * This method supports PDO binding types as well as DBAL mapping types. * - * @param string $sql The SQL query. - * @param array $params The query parameters. - * @param array $types The parameter types. + * @param string $sql The SQL query. + * @param array $params The query parameters. + * @param array $types The parameter types. * * @return int The number of affected rows. * @@ -345,18 +462,28 @@ class Connection extends PrimaryReadReplicaConnection { } $sql = $this->finishQuery($sql); $this->queriesExecuted++; - $this->logQueryToFile($sql); - return (int)parent::executeStatement($sql, $params, $types); + $this->logQueryToFile($sql, $params); + try { + return (int)parent::executeStatement($sql, $params, $types); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } - protected function logQueryToFile(string $sql): void { + protected function logQueryToFile(string $sql, array $params): void { $logFile = $this->systemConfig->getValue('query_log_file'); if ($logFile !== '' && is_writable(dirname($logFile)) && (!file_exists($logFile) || is_writable($logFile))) { $prefix = ''; if ($this->systemConfig->getValue('query_log_file_requestid') === 'yes') { $prefix .= Server::get(IRequestId::class)->getId() . "\t"; } + $postfix = ''; + if ($this->systemConfig->getValue('query_log_file_parameters') === 'yes') { + $postfix .= '; ' . json_encode($params); + } + if ($this->systemConfig->getValue('query_log_file_backtrace') === 'yes') { $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); array_pop($trace); @@ -412,18 +539,28 @@ class Connection extends PrimaryReadReplicaConnection { * @param string $table The table name (will replace *PREFIX* with the actual prefix) * @param array $input data that should be inserted into the table (column name => value) * @param array|null $compare List of values that should be checked for "if not exists" - * If this is null or an empty array, all keys of $input will be compared - * Please note: text fields (clob) must not be used in the compare array + * If this is null or an empty array, all keys of $input will be compared + * Please note: text fields (clob) must not be used in the compare array * @return int number of inserted rows * @throws \Doctrine\DBAL\Exception * @deprecated 15.0.0 - use unique index and "try { $db->insert() } catch (UniqueConstraintViolationException $e) {}" instead, because it is more reliable and does not have the risk for deadlocks - see https://github.com/nextcloud/server/pull/12371 */ public function insertIfNotExist($table, $input, ?array $compare = null) { - return $this->adapter->insertIfNotExist($table, $input, $compare); + try { + return $this->adapter->insertIfNotExist($table, $input, $compare); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } public function insertIgnoreConflict(string $table, array $values) : int { - return $this->adapter->insertIgnoreConflict($table, $values); + try { + return $this->adapter->insertIgnoreConflict($table, $values); + } catch (\Exception $e) { + $this->logDatabaseException($e); + throw $e; + } } private function getType($value) { @@ -472,22 +609,22 @@ class Connection extends PrimaryReadReplicaConnection { foreach ($values as $name => $value) { $updateQb->set($name, $updateQb->createNamedParameter($value, $this->getType($value))); } - $where = $updateQb->expr()->andX(); + $where = []; $whereValues = array_merge($keys, $updatePreconditionValues); foreach ($whereValues as $name => $value) { if ($value === '') { - $where->add($updateQb->expr()->emptyString( + $where[] = $updateQb->expr()->emptyString( $name - )); + ); } else { - $where->add($updateQb->expr()->eq( + $where[] = $updateQb->expr()->eq( $name, $updateQb->createNamedParameter($value, $this->getType($value)), $this->getType($value) - )); + ); } } - $updateQb->where($where); + $updateQb->where($updateQb->expr()->andX(...$where)); $affected = $updateQb->executeStatement(); if ($affected === 0 && !empty($updatePreconditionValues)) { @@ -537,9 +674,9 @@ class Connection extends PrimaryReadReplicaConnection { $msg = $this->errorCode() . ': '; $errorInfo = $this->errorInfo(); if (!empty($errorInfo)) { - $msg .= 'SQLSTATE = '.$errorInfo[0] . ', '; - $msg .= 'Driver Code = '.$errorInfo[1] . ', '; - $msg .= 'Driver Message = '.$errorInfo[2]; + $msg .= 'SQLSTATE = ' . $errorInfo[0] . ', '; + $msg .= 'Driver Code = ' . $errorInfo[1] . ', '; + $msg .= 'Driver Message = ' . $errorInfo[2]; } return $msg; } @@ -561,13 +698,26 @@ class Connection extends PrimaryReadReplicaConnection { */ public function dropTable($table) { $table = $this->tablePrefix . trim($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); if ($schema->tablesExist([$table])) { $schema->dropTable($table); } } /** + * Truncate a table data if it exists + * + * @param string $table table name without the prefix + * @param bool $cascade whether to truncate cascading + * + * @throws Exception + */ + public function truncateTable(string $table, bool $cascade) { + $this->executeStatement($this->getDatabasePlatform() + ->getTruncateTableSQL($this->tablePrefix . trim($table), $cascade)); + } + + /** * Check if a table exists * * @param string $table table name without the prefix @@ -577,7 +727,7 @@ class Connection extends PrimaryReadReplicaConnection { */ public function tableExists($table) { $table = $this->tablePrefix . trim($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); return $schema->tablesExist([$table]); } @@ -585,7 +735,7 @@ class Connection extends PrimaryReadReplicaConnection { $statement = $this->replaceTablePrefix($statement); $statement = $this->adapter->fixupStatement($statement); if ($this->logRequestId) { - return $statement . " /* reqid: " . $this->requestId . " */"; + return $statement . ' /* reqid: ' . $this->requestId . ' */'; } else { return $statement; } @@ -662,6 +812,9 @@ class Connection extends PrimaryReadReplicaConnection { return $migrator->generateChangeScript($toSchema); } else { $migrator->migrate($toSchema); + foreach ($this->getShardConnections() as $shardConnection) { + $shardConnection->migrateToSchema($toSchema); + } } } @@ -682,6 +835,7 @@ class Connection extends PrimaryReadReplicaConnection { public function beginTransaction() { if (!$this->inTransaction()) { + $this->transactionBacktrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); $this->transactionActiveSince = microtime(true); } return parent::beginTransaction(); @@ -691,9 +845,23 @@ class Connection extends PrimaryReadReplicaConnection { $result = parent::commit(); if ($this->getTransactionNestingLevel() === 0) { $timeTook = microtime(true) - $this->transactionActiveSince; + $this->transactionBacktrace = null; $this->transactionActiveSince = null; if ($timeTook > 1) { - $this->logger->debug('Transaction took ' . $timeTook . 's', ['exception' => new \Exception('Transaction took ' . $timeTook . 's')]); + $logLevel = match (true) { + $timeTook > 20 * 60 => ILogger::ERROR, + $timeTook > 5 * 60 => ILogger::WARN, + $timeTook > 10 => ILogger::INFO, + default => ILogger::DEBUG, + }; + $this->logger->log( + $logLevel, + 'Transaction took ' . $timeTook . 's', + [ + 'exception' => new \Exception('Transaction took ' . $timeTook . 's'), + 'timeSpent' => $timeTook, + ] + ); } } return $result; @@ -703,9 +871,23 @@ class Connection extends PrimaryReadReplicaConnection { $result = parent::rollBack(); if ($this->getTransactionNestingLevel() === 0) { $timeTook = microtime(true) - $this->transactionActiveSince; + $this->transactionBacktrace = null; $this->transactionActiveSince = null; if ($timeTook > 1) { - $this->logger->debug('Transaction rollback took longer than 1s: ' . $timeTook, ['exception' => new \Exception('Long running transaction rollback')]); + $logLevel = match (true) { + $timeTook > 20 * 60 => ILogger::ERROR, + $timeTook > 5 * 60 => ILogger::WARN, + $timeTook > 10 => ILogger::INFO, + default => ILogger::DEBUG, + }; + $this->logger->log( + $logLevel, + 'Transaction rollback took longer than 1s: ' . $timeTook, + [ + 'exception' => new \Exception('Long running transaction rollback'), + 'timeSpent' => $timeTook, + ] + ); } } return $result; @@ -713,9 +895,9 @@ class Connection extends PrimaryReadReplicaConnection { private function reconnectIfNeeded(): void { if ( - !isset($this->lastConnectionCheck[$this->getConnectionName()]) || - time() <= $this->lastConnectionCheck[$this->getConnectionName()] + 30 || - $this->isTransactionActive() + !isset($this->lastConnectionCheck[$this->getConnectionName()]) + || time() <= $this->lastConnectionCheck[$this->getConnectionName()] + 30 + || $this->isTransactionActive() ) { return; } @@ -732,4 +914,57 @@ class Connection extends PrimaryReadReplicaConnection { private function getConnectionName(): string { return $this->isConnectedToPrimary() ? 'primary' : 'replica'; } + + /** + * @return IDBConnection::PLATFORM_MYSQL|IDBConnection::PLATFORM_ORACLE|IDBConnection::PLATFORM_POSTGRES|IDBConnection::PLATFORM_SQLITE|IDBConnection::PLATFORM_MARIADB + */ + public function getDatabaseProvider(bool $strict = false): string { + $platform = $this->getDatabasePlatform(); + if ($strict && $platform instanceof MariaDBPlatform) { + return IDBConnection::PLATFORM_MARIADB; + } elseif ($platform instanceof MySQLPlatform) { + return IDBConnection::PLATFORM_MYSQL; + } elseif ($platform instanceof OraclePlatform) { + return IDBConnection::PLATFORM_ORACLE; + } elseif ($platform instanceof PostgreSQLPlatform) { + return IDBConnection::PLATFORM_POSTGRES; + } elseif ($platform instanceof SqlitePlatform) { + return IDBConnection::PLATFORM_SQLITE; + } else { + throw new \Exception('Database ' . $platform::class . ' not supported'); + } + } + + /** + * @internal Should only be used inside the QueryBuilder, ExpressionBuilder and FunctionBuilder + * All apps and API code should not need this and instead use provided functionality from the above. + */ + public function getServerVersion(): string { + /** @var ServerInfoAwareConnection $this->_conn */ + return $this->_conn->getServerVersion(); + } + + /** + * Log a database exception if enabled + * + * @param \Exception $exception + * @return void + */ + public function logDatabaseException(\Exception $exception): void { + if ($this->logDbException) { + if ($exception instanceof Exception\UniqueConstraintViolationException) { + $this->logger->info($exception->getMessage(), ['exception' => $exception, 'transaction' => $this->transactionBacktrace]); + } else { + $this->logger->error($exception->getMessage(), ['exception' => $exception, 'transaction' => $this->transactionBacktrace]); + } + } + } + + public function getShardDefinition(string $name): ?ShardDefinition { + return $this->shards[$name] ?? null; + } + + public function getCrossShardMoveHelper(): CrossShardMoveHelper { + return new CrossShardMoveHelper($this->shardConnectionManager); + } } diff --git a/lib/private/DB/ConnectionAdapter.php b/lib/private/DB/ConnectionAdapter.php index 86a901a7de3..d9ccb3c54f2 100644 --- a/lib/private/DB/ConnectionAdapter.php +++ b/lib/private/DB/ConnectionAdapter.php @@ -10,12 +10,10 @@ namespace OC\DB; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Platforms\AbstractPlatform; -use Doctrine\DBAL\Platforms\MySQLPlatform; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQLPlatform; -use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Schema\Schema; use OC\DB\Exceptions\DbalException; +use OC\DB\QueryBuilder\Sharded\CrossShardMoveHelper; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; use OCP\DB\IPreparedStatement; use OCP\DB\IResult; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -52,7 +50,7 @@ class ConnectionAdapter implements IDBConnection { $this->inner->executeQuery($sql, $params, $types) ); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -60,7 +58,7 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeUpdate($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -68,7 +66,7 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeStatement($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -191,6 +189,14 @@ class ConnectionAdapter implements IDBConnection { } } + public function truncateTable(string $table, bool $cascade): void { + try { + $this->inner->truncateTable($table, $cascade); + } catch (Exception $e) { + throw DbalException::wrap($e); + } + } + public function tableExists(string $table): bool { try { return $this->inner->tableExists($table); @@ -230,18 +236,30 @@ class ConnectionAdapter implements IDBConnection { return $this->inner; } - public function getDatabaseProvider(): string { - $platform = $this->inner->getDatabasePlatform(); - if ($platform instanceof MySQLPlatform) { - return IDBConnection::PLATFORM_MYSQL; - } elseif ($platform instanceof OraclePlatform) { - return IDBConnection::PLATFORM_ORACLE; - } elseif ($platform instanceof PostgreSQLPlatform) { - return IDBConnection::PLATFORM_POSTGRES; - } elseif ($platform instanceof SqlitePlatform) { - return IDBConnection::PLATFORM_SQLITE; - } else { - throw new \Exception('Database ' . $platform::class . ' not supported'); - } + /** + * @return self::PLATFORM_MYSQL|self::PLATFORM_ORACLE|self::PLATFORM_POSTGRES|self::PLATFORM_SQLITE|self::PLATFORM_MARIADB + */ + public function getDatabaseProvider(bool $strict = false): string { + return $this->inner->getDatabaseProvider($strict); + } + + /** + * @internal Should only be used inside the QueryBuilder, ExpressionBuilder and FunctionBuilder + * All apps and API code should not need this and instead use provided functionality from the above. + */ + public function getServerVersion(): string { + return $this->inner->getServerVersion(); + } + + public function logDatabaseException(\Exception $exception) { + $this->inner->logDatabaseException($exception); + } + + public function getShardDefinition(string $name): ?ShardDefinition { + return $this->inner->getShardDefinition($name); + } + + public function getCrossShardMoveHelper(): CrossShardMoveHelper { + return $this->inner->getCrossShardMoveHelper(); } } diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index 5e50d01538a..d9b80b81992 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -11,7 +11,11 @@ use Doctrine\Common\EventManager; use Doctrine\DBAL\Configuration; use Doctrine\DBAL\DriverManager; use Doctrine\DBAL\Event\Listeners\OracleSessionInit; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; use OC\SystemConfig; +use OCP\ICacheFactory; +use OCP\Server; /** * Takes care of creating and configuring Doctrine connections. @@ -54,16 +58,13 @@ class ConnectionFactory { ], ]; - /** @var SystemConfig */ - private $config; + private ShardConnectionManager $shardConnectionManager; + private ICacheFactory $cacheFactory; - /** - * ConnectionFactory constructor. - * - * @param SystemConfig $systemConfig - */ - public function __construct(SystemConfig $systemConfig) { - $this->config = $systemConfig; + public function __construct( + private SystemConfig $config, + ?ICacheFactory $cacheFactory = null, + ) { if ($this->config->getValue('mysql.utf8mb4', false)) { $this->defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; } @@ -71,6 +72,8 @@ class ConnectionFactory { if ($collationOverride) { $this->defaultConnectionParams['mysql']['collation'] = $collationOverride; } + $this->shardConnectionManager = new ShardConnectionManager($this->config, $this); + $this->cacheFactory = $cacheFactory ?? Server::get(ICacheFactory::class); } /** @@ -101,49 +104,37 @@ class ConnectionFactory { * @param array $additionalConnectionParams Additional connection parameters * @return \OC\DB\Connection */ - public function getConnection($type, $additionalConnectionParams) { + public function getConnection(string $type, array $additionalConnectionParams): Connection { $normalizedType = $this->normalizeType($type); $eventManager = new EventManager(); $eventManager->addEventSubscriber(new SetTransactionIsolationLevel()); - $additionalConnectionParams = array_merge($this->createConnectionParams(), $additionalConnectionParams); + $connectionParams = $this->createConnectionParams('', $additionalConnectionParams, $type); switch ($normalizedType) { case 'pgsql': // pg_connect used by Doctrine DBAL does not support URI notation (enclosed in brackets) $matches = []; - if (preg_match('/^\[([^\]]+)\]$/', $additionalConnectionParams['host'], $matches)) { + if (preg_match('/^\[([^\]]+)\]$/', $connectionParams['host'], $matches)) { // Host variable carries a port or socket. - $additionalConnectionParams['host'] = $matches[1]; + $connectionParams['host'] = $matches[1]; } break; case 'oci': $eventManager->addEventSubscriber(new OracleSessionInit); - // the driverOptions are unused in dbal and need to be mapped to the parameters - if (isset($additionalConnectionParams['driverOptions'])) { - $additionalConnectionParams = array_merge($additionalConnectionParams, $additionalConnectionParams['driverOptions']); - } - $host = $additionalConnectionParams['host']; - $port = $additionalConnectionParams['port'] ?? null; - $dbName = $additionalConnectionParams['dbname']; - - // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string - if ($host === '') { - $additionalConnectionParams['dbname'] = $dbName; // use dbname as easy connect name - } else { - $additionalConnectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : "") . '/' . $dbName; - } - unset($additionalConnectionParams['host']); + $connectionParams = $this->forceConnectionStringOracle($connectionParams); + $connectionParams['primary'] = $this->forceConnectionStringOracle($connectionParams['primary']); + $connectionParams['replica'] = array_map([$this, 'forceConnectionStringOracle'], $connectionParams['replica']); break; case 'sqlite3': - $journalMode = $additionalConnectionParams['sqlite.journal_mode']; - $additionalConnectionParams['platform'] = new OCSqlitePlatform(); + $journalMode = $connectionParams['sqlite.journal_mode']; + $connectionParams['platform'] = new OCSqlitePlatform(); $eventManager->addEventSubscriber(new SQLiteSessionInit(true, $journalMode)); break; } /** @var Connection $connection */ $connection = DriverManager::getConnection( - $additionalConnectionParams, + $connectionParams, new Configuration(), $eventManager ); @@ -172,12 +163,10 @@ class ConnectionFactory { /** * Create the connection parameters for the config - * - * @param string $configPrefix - * @return array */ - public function createConnectionParams(string $configPrefix = '') { - $type = $this->config->getValue('dbtype', 'sqlite'); + public function createConnectionParams(string $configPrefix = '', array $additionalConnectionParams = [], ?string $type = null) { + // use provided type or if null use type from config + $type = $type ?? $this->config->getValue('dbtype', 'sqlite'); $connectionParams = array_merge($this->getDefaultConnectionParams($type), [ 'user' => $this->config->getValue($configPrefix . 'dbuser', $this->config->getValue('dbuser', '')), @@ -186,7 +175,7 @@ class ConnectionFactory { $name = $this->config->getValue($configPrefix . 'dbname', $this->config->getValue('dbname', self::DEFAULT_DBNAME)); if ($this->normalizeType($type) === 'sqlite3') { - $dataDir = $this->config->getValue("datadirectory", \OC::$SERVERROOT . '/data'); + $dataDir = $this->config->getValue('datadirectory', \OC::$SERVERROOT . '/data'); $connectionParams['path'] = $dataDir . '/' . $name . '.db'; } else { $host = $this->config->getValue($configPrefix . 'dbhost', $this->config->getValue('dbhost', '')); @@ -209,7 +198,7 @@ class ConnectionFactory { 'tablePrefix' => $connectionParams['tablePrefix'] ]; - if ($this->config->getValue('mysql.utf8mb4', false)) { + if ($type === 'mysql' && $this->config->getValue('mysql.utf8mb4', false)) { $connectionParams['defaultTableOptions'] = [ 'collate' => 'utf8mb4_bin', 'charset' => 'utf8mb4', @@ -221,6 +210,20 @@ class ConnectionFactory { $connectionParams['persistent'] = true; } + $connectionParams['sharding'] = $this->config->getValue('dbsharding', []); + if (!empty($connectionParams['sharding'])) { + $connectionParams['shard_connection_manager'] = $this->shardConnectionManager; + $connectionParams['auto_increment_handler'] = new AutoIncrementHandler( + $this->cacheFactory, + $this->shardConnectionManager, + ); + } else { + // just in case only the presence could lead to funny behaviour + unset($connectionParams['sharding']); + } + + $connectionParams = array_merge($connectionParams, $additionalConnectionParams); + $replica = $this->config->getValue($configPrefix . 'dbreplica', $this->config->getValue('dbreplica', [])) ?: [$connectionParams]; return array_merge($connectionParams, [ 'primary' => $connectionParams, @@ -242,7 +245,7 @@ class ConnectionFactory { // Host variable carries a port or socket. $params['host'] = $matches[1]; if (is_numeric($matches[2])) { - $params['port'] = (int) $matches[2]; + $params['port'] = (int)$matches[2]; } else { $params['unix_socket'] = $matches[2]; } @@ -250,4 +253,24 @@ class ConnectionFactory { return $params; } + + protected function forceConnectionStringOracle(array $connectionParams): array { + // the driverOptions are unused in dbal and need to be mapped to the parameters + if (isset($connectionParams['driverOptions'])) { + $connectionParams = array_merge($connectionParams, $connectionParams['driverOptions']); + } + $host = $connectionParams['host']; + $port = $connectionParams['port'] ?? null; + $dbName = $connectionParams['dbname']; + + // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string + if ($host === '') { + $connectionParams['dbname'] = $dbName; // use dbname as easy connect name + } else { + $connectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : '') . '/' . $dbName; + } + unset($connectionParams['host']); + + return $connectionParams; + } } diff --git a/lib/private/DB/DbDataCollector.php b/lib/private/DB/DbDataCollector.php index fcaa74daeab..e3c7cd35855 100644 --- a/lib/private/DB/DbDataCollector.php +++ b/lib/private/DB/DbDataCollector.php @@ -111,7 +111,7 @@ class DbDataCollector extends \OCP\DataCollector\AbstractDataCollector { } if ($error) { - return ['âš '.$error->getMessage(), false, false]; + return ['âš ' . $error->getMessage(), false, false]; } if (\is_array($var)) { diff --git a/lib/private/DB/Exceptions/DbalException.php b/lib/private/DB/Exceptions/DbalException.php index 05ea9e22a5d..2ce6ddf80a6 100644 --- a/lib/private/DB/Exceptions/DbalException.php +++ b/lib/private/DB/Exceptions/DbalException.php @@ -35,26 +35,29 @@ use OCP\DB\Exception; class DbalException extends Exception { /** @var \Doctrine\DBAL\Exception */ private $original; + public readonly ?string $query; /** * @param \Doctrine\DBAL\Exception $original * @param int $code * @param string $message */ - private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message) { + private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message, ?string $query = null) { parent::__construct( $message, $code, $original ); $this->original = $original; + $this->query = $query; } - public static function wrap(\Doctrine\DBAL\Exception $original, string $message = ''): self { + public static function wrap(\Doctrine\DBAL\Exception $original, string $message = '', ?string $query = null): self { return new self( $original, is_int($original->getCode()) ? $original->getCode() : 0, - empty($message) ? $original->getMessage() : $message + empty($message) ? $original->getMessage() : $message, + $query, ); } diff --git a/lib/private/DB/MigrationService.php b/lib/private/DB/MigrationService.php index 19d1b240736..40579c7a898 100644 --- a/lib/private/DB/MigrationService.php +++ b/lib/private/DB/MigrationService.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-FileCopyrightText: 2017 ownCloud GmbH @@ -6,20 +7,19 @@ */ namespace OC\DB; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Schema\SchemaException; use Doctrine\DBAL\Schema\Sequence; use Doctrine\DBAL\Schema\Table; -use Doctrine\DBAL\Types\Types; use OC\App\InfoParser; use OC\IntegrityCheck\Helpers\AppLocator; use OC\Migration\SimpleOutput; use OCP\AppFramework\App; use OCP\AppFramework\QueryException; use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\IDBConnection; use OCP\Migration\IMigrationStep; use OCP\Migration\IOutput; use OCP\Server; @@ -159,7 +159,7 @@ class MigrationService { /** * Returns all versions which have already been applied * - * @return string[] + * @return list<string> * @codeCoverageIgnore - no need to test this */ public function getMigratedVersions() { @@ -175,6 +175,8 @@ class MigrationService { $rows = $result->fetchAll(\PDO::FETCH_COLUMN); $result->closeCursor(); + usort($rows, $this->sortMigrations(...)); + return $rows; } @@ -184,7 +186,23 @@ class MigrationService { */ public function getAvailableVersions(): array { $this->ensureMigrationsAreLoaded(); - return array_map('strval', array_keys($this->migrations)); + $versions = array_map('strval', array_keys($this->migrations)); + usort($versions, $this->sortMigrations(...)); + return $versions; + } + + protected function sortMigrations(string $a, string $b): int { + preg_match('/(\d+)Date(\d+)/', basename($a), $matchA); + preg_match('/(\d+)Date(\d+)/', basename($b), $matchB); + if (!empty($matchA) && !empty($matchB)) { + $versionA = (int)$matchA[1]; + $versionB = (int)$matchB[1]; + if ($versionA !== $versionB) { + return ($versionA < $versionB) ? -1 : 1; + } + return strnatcmp($matchA[2], $matchB[2]); + } + return strnatcmp(basename($a), basename($b)); } /** @@ -205,23 +223,13 @@ class MigrationService { \RegexIterator::GET_MATCH); $files = array_keys(iterator_to_array($iterator)); - uasort($files, function ($a, $b) { - preg_match('/^Version(\d+)Date(\d+)\\.php$/', basename($a), $matchA); - preg_match('/^Version(\d+)Date(\d+)\\.php$/', basename($b), $matchB); - if (!empty($matchA) && !empty($matchB)) { - if ($matchA[1] !== $matchB[1]) { - return ($matchA[1] < $matchB[1]) ? -1 : 1; - } - return ($matchA[2] < $matchB[2]) ? -1 : 1; - } - return (basename($a) < basename($b)) ? -1 : 1; - }); + usort($files, $this->sortMigrations(...)); $migrations = []; foreach ($files as $file) { $className = basename($file, '.php'); - $version = (string) substr($className, 7); + $version = (string)substr($className, 7); if ($version === '0') { throw new \InvalidArgumentException( "Cannot load a migrations with the name '$version' because it is a reserved number" @@ -243,7 +251,7 @@ class MigrationService { $toBeExecuted = []; foreach ($availableMigrations as $v) { - if ($to !== 'latest' && $v > $to) { + if ($to !== 'latest' && ($this->sortMigrations($v, $to) > 0)) { continue; } if ($this->shallBeExecuted($v, $knownMigrations)) { @@ -389,7 +397,7 @@ class MigrationService { } catch (\Exception $e) { // The exception itself does not contain the name of the migration, // so we wrap it here, to make debugging easier. - throw new \Exception('Database error when running migration ' . $version . ' for app ' . $this->getApp() . PHP_EOL. $e->getMessage(), 0, $e); + throw new \Exception('Database error when running migration ' . $version . ' for app ' . $this->getApp() . PHP_EOL . $e->getMessage(), 0, $e); } } } @@ -599,7 +607,7 @@ class MigrationService { $indexName = strtolower($primaryKey->getName()); $isUsingDefaultName = $indexName === 'primary'; - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { + if ($this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_POSTGRES) { $defaultName = $table->getName() . '_pkey'; $isUsingDefaultName = strtolower($defaultName) === $indexName; @@ -609,7 +617,7 @@ class MigrationService { return $sequence->getName() !== $sequenceName; }); } - } elseif ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { + } elseif ($this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_ORACLE) { $defaultName = $table->getName() . '_seq'; $isUsingDefaultName = strtolower($defaultName) === $indexName; } diff --git a/lib/private/DB/Migrator.php b/lib/private/DB/Migrator.php index 4fd457b4bc6..40f8ad9676a 100644 --- a/lib/private/DB/Migrator.php +++ b/lib/private/DB/Migrator.php @@ -139,7 +139,7 @@ class Migrator { $step = 0; foreach ($sqls as $sql) { $this->emit($sql, $step++, count($sqls)); - $connection->executeQuery($sql); + $connection->executeStatement($sql); } if (!$connection->getDatabasePlatform() instanceof MySQLPlatform) { $connection->commit(); diff --git a/lib/private/DB/MigratorExecuteSqlEvent.php b/lib/private/DB/MigratorExecuteSqlEvent.php index cfcfe7fa512..340cd636300 100644 --- a/lib/private/DB/MigratorExecuteSqlEvent.php +++ b/lib/private/DB/MigratorExecuteSqlEvent.php @@ -18,7 +18,7 @@ class MigratorExecuteSqlEvent extends Event { public function __construct( string $sql, int $current, - int $max + int $max, ) { $this->sql = $sql; $this->current = $current; diff --git a/lib/private/DB/MySqlTools.php b/lib/private/DB/MySqlTools.php index cd6b812be61..3413be43417 100644 --- a/lib/private/DB/MySqlTools.php +++ b/lib/private/DB/MySqlTools.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 ownCloud GmbH * SPDX-License-Identifier: AGPL-3.0-only @@ -45,7 +46,7 @@ class MySqlTools { return false; } - return str_contains($row, 'maria') && version_compare($row, '10.3', '>=') || - !str_contains($row, 'maria') && version_compare($row, '8.0', '>='); + return str_contains($row, 'maria') && version_compare($row, '10.3', '>=') + || !str_contains($row, 'maria') && version_compare($row, '8.0', '>='); } } diff --git a/lib/private/DB/OracleConnection.php b/lib/private/DB/OracleConnection.php index abfb69f129b..5ffb65d801d 100644 --- a/lib/private/DB/OracleConnection.php +++ b/lib/private/DB/OracleConnection.php @@ -68,7 +68,7 @@ class OracleConnection extends Connection { public function dropTable($table) { $table = $this->tablePrefix . trim($table); $table = $this->quoteIdentifier($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); if ($schema->tablesExist([$table])) { $schema->dropTable($table); } @@ -83,7 +83,7 @@ class OracleConnection extends Connection { public function tableExists($table) { $table = $this->tablePrefix . trim($table); $table = $this->quoteIdentifier($table); - $schema = $this->getSchemaManager(); + $schema = $this->createSchemaManager(); return $schema->tablesExist([$table]); } } diff --git a/lib/private/DB/PgSqlTools.php b/lib/private/DB/PgSqlTools.php index 35e8016191c..d529cb26b09 100644 --- a/lib/private/DB/PgSqlTools.php +++ b/lib/private/DB/PgSqlTools.php @@ -43,7 +43,7 @@ class PgSqlTools { return preg_match($filterExpression, $asset) !== false; }); - foreach ($conn->getSchemaManager()->listSequences() as $sequence) { + foreach ($conn->createSchemaManager()->listSequences() as $sequence) { $sequenceName = $sequence->getName(); $sqlInfo = 'SELECT table_schema, table_name, column_name FROM information_schema.columns diff --git a/lib/private/DB/PreparedStatement.php b/lib/private/DB/PreparedStatement.php index 5fdfa2b03e8..54561ed96cd 100644 --- a/lib/private/DB/PreparedStatement.php +++ b/lib/private/DB/PreparedStatement.php @@ -78,6 +78,6 @@ class PreparedStatement implements IPreparedStatement { return $this->result; } - throw new Exception("You have to execute the prepared statement before accessing the results"); + throw new Exception('You have to execute the prepared statement before accessing the results'); } } diff --git a/lib/private/DB/QueryBuilder/CompositeExpression.php b/lib/private/DB/QueryBuilder/CompositeExpression.php index 493d804d54d..122998036ae 100644 --- a/lib/private/DB/QueryBuilder/CompositeExpression.php +++ b/lib/private/DB/QueryBuilder/CompositeExpression.php @@ -5,21 +5,19 @@ * SPDX-FileCopyrightText: 2016 ownCloud, Inc. * SPDX-License-Identifier: AGPL-3.0-only */ + namespace OC\DB\QueryBuilder; use OCP\DB\QueryBuilder\ICompositeExpression; class CompositeExpression implements ICompositeExpression, \Countable { - /** @var \Doctrine\DBAL\Query\Expression\CompositeExpression */ - protected $compositeExpression; + public const TYPE_AND = 'AND'; + public const TYPE_OR = 'OR'; - /** - * Constructor. - * - * @param \Doctrine\DBAL\Query\Expression\CompositeExpression $compositeExpression - */ - public function __construct(\Doctrine\DBAL\Query\Expression\CompositeExpression $compositeExpression) { - $this->compositeExpression = $compositeExpression; + public function __construct( + private string $type, + private array $parts = [], + ) { } /** @@ -30,7 +28,9 @@ class CompositeExpression implements ICompositeExpression, \Countable { * @return \OCP\DB\QueryBuilder\ICompositeExpression */ public function addMultiple(array $parts = []): ICompositeExpression { - $this->compositeExpression->addMultiple($parts); + foreach ($parts as $part) { + $this->add($part); + } return $this; } @@ -43,7 +43,15 @@ class CompositeExpression implements ICompositeExpression, \Countable { * @return \OCP\DB\QueryBuilder\ICompositeExpression */ public function add($part): ICompositeExpression { - $this->compositeExpression->add($part); + if ($part === null) { + return $this; + } + + if ($part instanceof self && count($part) === 0) { + return $this; + } + + $this->parts[] = $part; return $this; } @@ -54,7 +62,7 @@ class CompositeExpression implements ICompositeExpression, \Countable { * @return integer */ public function count(): int { - return $this->compositeExpression->count(); + return count($this->parts); } /** @@ -63,7 +71,7 @@ class CompositeExpression implements ICompositeExpression, \Countable { * @return string */ public function getType(): string { - return $this->compositeExpression->getType(); + return $this->type; } /** @@ -72,6 +80,13 @@ class CompositeExpression implements ICompositeExpression, \Countable { * @return string */ public function __toString(): string { - return (string) $this->compositeExpression; + if ($this->count() === 1) { + return (string)$this->parts[0]; + } + return '(' . implode(') ' . $this->type . ' (', $this->parts) . ')'; + } + + public function getParts(): array { + return $this->parts; } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php index a7af00b9f97..b922c861630 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/ExpressionBuilder.php @@ -21,6 +21,7 @@ use OCP\DB\QueryBuilder\IParameter; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; use OCP\IDBConnection; +use Psr\Log\LoggerInterface; class ExpressionBuilder implements IExpressionBuilder { /** @var \Doctrine\DBAL\Query\Expression\ExpressionBuilder */ @@ -32,17 +33,15 @@ class ExpressionBuilder implements IExpressionBuilder { /** @var IDBConnection */ protected $connection; + /** @var LoggerInterface */ + protected $logger; + /** @var FunctionBuilder */ protected $functionBuilder; - /** - * Initializes a new <tt>ExpressionBuilder</tt>. - * - * @param ConnectionAdapter $connection - * @param IQueryBuilder $queryBuilder - */ - public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder) { + public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder, LoggerInterface $logger) { $this->connection = $connection; + $this->logger = $logger; $this->helper = new QuoteHelper(); $this->expressionBuilder = new DoctrineExpressionBuilder($connection->getInner()); $this->functionBuilder = $queryBuilder->func(); @@ -58,13 +57,15 @@ class ExpressionBuilder implements IExpressionBuilder { * $expr->andX('u.type = ?', 'u.role = ?')); * * @param mixed ...$x Optional clause. Defaults = null, but requires - * at least one defined when converting to string. + * at least one defined when converting to string. * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ public function andX(...$x): ICompositeExpression { - $compositeExpression = call_user_func_array([$this->expressionBuilder, 'andX'], $x); - return new CompositeExpression($compositeExpression); + if (empty($x)) { + $this->logger->debug('Calling ' . IQueryBuilder::class . '::' . __FUNCTION__ . ' without parameters is deprecated and will throw soon.', ['exception' => new \Exception('No parameters in call to ' . __METHOD__)]); + } + return new CompositeExpression(CompositeExpression::TYPE_AND, $x); } /** @@ -77,13 +78,15 @@ class ExpressionBuilder implements IExpressionBuilder { * $qb->where($qb->expr()->orX('u.type = ?', 'u.role = ?')); * * @param mixed ...$x Optional clause. Defaults = null, but requires - * at least one defined when converting to string. + * at least one defined when converting to string. * * @return \OCP\DB\QueryBuilder\ICompositeExpression */ public function orX(...$x): ICompositeExpression { - $compositeExpression = call_user_func_array([$this->expressionBuilder, 'orX'], $x); - return new CompositeExpression($compositeExpression); + if (empty($x)) { + $this->logger->debug('Calling ' . IQueryBuilder::class . '::' . __FUNCTION__ . ' without parameters is deprecated and will throw soon.', ['exception' => new \Exception('No parameters in call to ' . __METHOD__)]); + } + return new CompositeExpression(CompositeExpression::TYPE_OR, $x); } /** @@ -93,7 +96,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param string $operator One of the IExpressionBuilder::* constants. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -116,7 +119,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -138,7 +141,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -160,7 +163,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -182,7 +185,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -204,7 +207,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -226,7 +229,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param mixed $x The left expression. * @param mixed $y The right expression. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -266,7 +269,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x Field in string format to be inspected by LIKE() comparison. * @param mixed $y Argument to be used in LIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -282,7 +285,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param string $x Field in string format to be inspected by ILIKE() comparison. * @param mixed $y Argument to be used in ILIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string * @since 9.0.0 @@ -297,7 +300,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x Field in string format to be inspected by NOT LIKE() comparison. * @param mixed $y Argument to be used in NOT LIKE() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -313,7 +316,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x The field in string format to be inspected by IN() comparison. * @param ILiteral|IParameter|IQueryFunction|string|array $y The placeholder or the array of values to be used by IN() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ @@ -329,7 +332,7 @@ class ExpressionBuilder implements IExpressionBuilder { * @param ILiteral|IParameter|IQueryFunction|string $x The field in string format to be inspected by NOT IN() comparison. * @param ILiteral|IParameter|IQueryFunction|string|array $y The placeholder or the array of values to be used by NOT IN() comparison. * @param mixed|null $type one of the IQueryBuilder::PARAM_* constants - * required when comparing text fields for oci compatibility + * required when comparing text fields for oci compatibility * * @return string */ diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php index cd2fb5a5cc3..7216fd8807b 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php @@ -11,17 +11,13 @@ use OC\DB\ConnectionAdapter; use OC\DB\QueryBuilder\QueryFunction; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; +use Psr\Log\LoggerInterface; class MySqlExpressionBuilder extends ExpressionBuilder { - /** @var string */ - protected $collation; + protected string $collation; - /** - * @param ConnectionAdapter $connection - * @param IQueryBuilder $queryBuilder - */ - public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder) { - parent::__construct($connection, $queryBuilder); + public function __construct(ConnectionAdapter $connection, IQueryBuilder $queryBuilder, LoggerInterface $logger) { + parent::__construct($connection, $queryBuilder, $logger); $params = $connection->getInner()->getParams(); $this->collation = $params['collation'] ?? (($params['charset'] ?? 'utf8') . '_general_ci'); diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php index 1a162f04a49..53a566a7eb6 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php @@ -23,7 +23,7 @@ class PgSqlExpressionBuilder extends ExpressionBuilder { public function castColumn($column, $type): IQueryFunction { switch ($type) { case IQueryBuilder::PARAM_INT: - return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS INT)'); + return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS BIGINT)'); case IQueryBuilder::PARAM_STR: return new QueryFunction('CAST(' . $this->helper->quoteColumnName($column) . ' AS TEXT)'); default: diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php index ac4698cdc76..52f82db2232 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -29,7 +30,11 @@ class SqliteExpressionBuilder extends ExpressionBuilder { * @return array|IQueryFunction|string */ protected function prepareColumn($column, $type) { - if ($type === IQueryBuilder::PARAM_DATE && !is_array($column) && !($column instanceof IParameter) && !($column instanceof ILiteral)) { + if ($type !== null + && !is_array($column) + && !($column instanceof IParameter) + && !($column instanceof ILiteral) + && (str_starts_with($type, 'date') || str_starts_with($type, 'time'))) { return $this->castColumn($column, $type); } @@ -44,9 +49,21 @@ class SqliteExpressionBuilder extends ExpressionBuilder { * @return IQueryFunction */ public function castColumn($column, $type): IQueryFunction { - if ($type === IQueryBuilder::PARAM_DATE) { - $column = $this->helper->quoteColumnName($column); - return new QueryFunction('DATETIME(' . $column . ')'); + switch ($type) { + case IQueryBuilder::PARAM_DATE_MUTABLE: + case IQueryBuilder::PARAM_DATE_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('DATE(' . $column . ')'); + case IQueryBuilder::PARAM_DATETIME_MUTABLE: + case IQueryBuilder::PARAM_DATETIME_IMMUTABLE: + case IQueryBuilder::PARAM_DATETIME_TZ_MUTABLE: + case IQueryBuilder::PARAM_DATETIME_TZ_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('DATETIME(' . $column . ')'); + case IQueryBuilder::PARAM_TIME_MUTABLE: + case IQueryBuilder::PARAM_TIME_IMMUTABLE: + $column = $this->helper->quoteColumnName($column); + return new QueryFunction('TIME(' . $column . ')'); } return parent::castColumn($column, $type); diff --git a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php new file mode 100644 index 00000000000..f928132a123 --- /dev/null +++ b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php @@ -0,0 +1,309 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder; + +use OC\DB\Exceptions\DbalException; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Base class for creating classes that extend the builtin query builder + */ +abstract class ExtendedQueryBuilder implements IQueryBuilder { + public function __construct( + protected IQueryBuilder $builder, + ) { + } + + public function automaticTablePrefix($enabled) { + $this->builder->automaticTablePrefix($enabled); + return $this; + } + + public function expr() { + return $this->builder->expr(); + } + + public function func() { + return $this->builder->func(); + } + + public function getType() { + return $this->builder->getType(); + } + + public function getConnection() { + return $this->builder->getConnection(); + } + + public function getState() { + return $this->builder->getState(); + } + + public function execute(?IDBConnection $connection = null) { + try { + if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + return $this->executeQuery($connection); + } else { + return $this->executeStatement($connection); + } + } catch (DBALException $e) { + // `IQueryBuilder->execute` never wrapped the exception, but `executeQuery` and `executeStatement` do + /** @var \Doctrine\DBAL\Exception $previous */ + $previous = $e->getPrevious(); + throw $previous; + } + } + + public function getSQL() { + return $this->builder->getSQL(); + } + + public function setParameter($key, $value, $type = null) { + $this->builder->setParameter($key, $value, $type); + return $this; + } + + public function setParameters(array $params, array $types = []) { + $this->builder->setParameters($params, $types); + return $this; + } + + public function getParameters() { + return $this->builder->getParameters(); + } + + public function getParameter($key) { + return $this->builder->getParameter($key); + } + + public function getParameterTypes() { + return $this->builder->getParameterTypes(); + } + + public function getParameterType($key) { + return $this->builder->getParameterType($key); + } + + public function setFirstResult($firstResult) { + $this->builder->setFirstResult($firstResult); + return $this; + } + + public function getFirstResult() { + return $this->builder->getFirstResult(); + } + + public function setMaxResults($maxResults) { + $this->builder->setMaxResults($maxResults); + return $this; + } + + public function getMaxResults() { + return $this->builder->getMaxResults(); + } + + public function select(...$selects) { + $this->builder->select(...$selects); + return $this; + } + + public function selectAlias($select, $alias) { + $this->builder->selectAlias($select, $alias); + return $this; + } + + public function selectDistinct($select) { + $this->builder->selectDistinct($select); + return $this; + } + + public function addSelect(...$select) { + $this->builder->addSelect(...$select); + return $this; + } + + public function delete($delete = null, $alias = null) { + $this->builder->delete($delete, $alias); + return $this; + } + + public function update($update = null, $alias = null) { + $this->builder->update($update, $alias); + return $this; + } + + public function insert($insert = null) { + $this->builder->insert($insert); + return $this; + } + + public function from($from, $alias = null) { + $this->builder->from($from, $alias); + return $this; + } + + public function join($fromAlias, $join, $alias, $condition = null) { + $this->builder->join($fromAlias, $join, $alias, $condition); + return $this; + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->innerJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->leftJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function rightJoin($fromAlias, $join, $alias, $condition = null) { + $this->builder->rightJoin($fromAlias, $join, $alias, $condition); + return $this; + } + + public function set($key, $value) { + $this->builder->set($key, $value); + return $this; + } + + public function where(...$predicates) { + $this->builder->where(...$predicates); + return $this; + } + + public function andWhere(...$where) { + $this->builder->andWhere(...$where); + return $this; + } + + public function orWhere(...$where) { + $this->builder->orWhere(...$where); + return $this; + } + + public function groupBy(...$groupBys) { + $this->builder->groupBy(...$groupBys); + return $this; + } + + public function addGroupBy(...$groupBy) { + $this->builder->addGroupBy(...$groupBy); + return $this; + } + + public function setValue($column, $value) { + $this->builder->setValue($column, $value); + return $this; + } + + public function values(array $values) { + $this->builder->values($values); + return $this; + } + + public function having(...$having) { + $this->builder->having(...$having); + return $this; + } + + public function andHaving(...$having) { + $this->builder->andHaving(...$having); + return $this; + } + + public function orHaving(...$having) { + $this->builder->orHaving(...$having); + return $this; + } + + public function orderBy($sort, $order = null) { + $this->builder->orderBy($sort, $order); + return $this; + } + + public function addOrderBy($sort, $order = null) { + $this->builder->addOrderBy($sort, $order); + return $this; + } + + public function getQueryPart($queryPartName) { + return $this->builder->getQueryPart($queryPartName); + } + + public function getQueryParts() { + return $this->builder->getQueryParts(); + } + + public function resetQueryParts($queryPartNames = null) { + $this->builder->resetQueryParts($queryPartNames); + return $this; + } + + public function resetQueryPart($queryPartName) { + $this->builder->resetQueryPart($queryPartName); + return $this; + } + + public function createNamedParameter($value, $type = self::PARAM_STR, $placeHolder = null) { + return $this->builder->createNamedParameter($value, $type, $placeHolder); + } + + public function createPositionalParameter($value, $type = self::PARAM_STR) { + return $this->builder->createPositionalParameter($value, $type); + } + + public function createParameter($name) { + return $this->builder->createParameter($name); + } + + public function createFunction($call) { + return $this->builder->createFunction($call); + } + + public function getLastInsertId(): int { + return $this->builder->getLastInsertId(); + } + + public function getTableName($table) { + return $this->builder->getTableName($table); + } + + public function getColumnName($column, $tableAlias = '') { + return $this->builder->getColumnName($column, $tableAlias); + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + return $this->builder->executeQuery($connection); + } + + public function executeStatement(?IDBConnection $connection = null): int { + return $this->builder->executeStatement($connection); + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + $this->builder->hintShardKey($column, $value, $overwrite); + return $this; + } + + public function runAcrossAllShards(): self { + $this->builder->runAcrossAllShards(); + return $this; + } + + public function getOutputColumns(): array { + return $this->builder->getOutputColumns(); + } + + public function prefixTableName(string $table): string { + return $this->builder->prefixTableName($table); + } +} diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php index b168d2c1a84..48dc1da6330 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php @@ -1,10 +1,12 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later */ namespace OC\DB\QueryBuilder\FunctionBuilder; +use OC\DB\Connection; use OC\DB\QueryBuilder\QueryFunction; use OC\DB\QueryBuilder\QuoteHelper; use OCP\DB\QueryBuilder\IFunctionBuilder; @@ -13,7 +15,7 @@ use OCP\DB\QueryBuilder\IQueryFunction; use OCP\IDBConnection; class FunctionBuilder implements IFunctionBuilder { - /** @var IDBConnection */ + /** @var IDBConnection|Connection */ protected $connection; /** @var IQueryBuilder */ diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php index d0258eafea8..47a8eaa6fd0 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -12,7 +13,10 @@ use OCP\DB\QueryBuilder\IQueryFunction; class OCIFunctionBuilder extends FunctionBuilder { public function md5($input): IQueryFunction { - return new QueryFunction('LOWER(DBMS_OBFUSCATION_TOOLKIT.md5 (input => UTL_RAW.cast_to_raw(' . $this->helper->quoteColumnName($input) .')))'); + if (version_compare($this->connection->getServerVersion(), '20', '>=')) { + return new QueryFunction('LOWER(STANDARD_HASH(' . $this->helper->quoteColumnName($input) . ", 'MD5'))"); + } + return new QueryFunction('LOWER(DBMS_OBFUSCATION_TOOLKIT.md5 (input => UTL_RAW.cast_to_raw(' . $this->helper->quoteColumnName($input) . ')))'); } /** @@ -77,12 +81,12 @@ class OCIFunctionBuilder extends FunctionBuilder { public function octetLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTHB(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias); } public function charLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias); } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php index ee430a6bd71..354a2b126d7 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php index 956b2123f2c..53aa530054b 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/Literal.php b/lib/private/DB/QueryBuilder/Literal.php index f0accce1d93..3fb897328e5 100644 --- a/lib/private/DB/QueryBuilder/Literal.php +++ b/lib/private/DB/QueryBuilder/Literal.php @@ -18,6 +18,6 @@ class Literal implements ILiteral { } public function __toString(): string { - return (string) $this->literal; + return (string)$this->literal; } } diff --git a/lib/private/DB/QueryBuilder/Parameter.php b/lib/private/DB/QueryBuilder/Parameter.php index dbd723639fc..a272c744d62 100644 --- a/lib/private/DB/QueryBuilder/Parameter.php +++ b/lib/private/DB/QueryBuilder/Parameter.php @@ -18,6 +18,6 @@ class Parameter implements IParameter { } public function __toString(): string { - return (string) $this->name; + return (string)$this->name; } } diff --git a/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php b/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php new file mode 100644 index 00000000000..3a5aa2f3e0e --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/InvalidPartitionedQueryException.php @@ -0,0 +1,79 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +/** + * Partitioned queries impose limitations that queries have to follow: + * + * 1. Any reference to columns not in the "main table" (the table referenced by "FROM"), needs to explicitly include the + * table or alias the column belongs to. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("root_id", "fileid")); + * ``` + * will not work, as the query builder doesn't know that the `mimetype` column belongs to the "filecache partition". + * Instead, you need to do + * ``` + * $query->select("mount_point", "f.mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid")); + * ``` + * + * 2. The "ON" condition for the join can only perform a comparison between both sides of the join once. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->andX($query->expr()->eq("m.root_id", "f.fileid"), $query->expr()->eq("m.storage_id", "f.storage"))); + * ``` + * will not work. + * + * 3. An "OR" expression in the "WHERE" cannot mention both sides of the join, this does not apply to "AND" expressions. + * + * For example: + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid"))) + * ->where($query->expr()->orX( + * $query->expr()-eq("m.user_id", $query->createNamedParameter("test"))), + * $query->expr()-eq("f.name", $query->createNamedParameter("test"))), + * )); + * ``` + * will not work, but. + * ``` + * $query->select("mount_point", "mimetype") + * ->from("mounts", "m") + * ->innerJoin("m", "filecache", "f", $query->expr()->eq("m.root_id", "f.fileid"))) + * ->where($query->expr()->andX( + * $query->expr()-eq("m.user_id", $query->createNamedParameter("test"))), + * $query->expr()-eq("f.name", $query->createNamedParameter("test"))), + * )); + * ``` + * will. + * + * 4. Queries that join cross-partition cannot use position parameters, only named parameters are allowed + * 5. The "ON" condition of a join cannot contain and "OR" expression. + * 6. Right-joins are not allowed. + * 7. Update, delete and insert statements aren't allowed to contain cross-partition joins. + * 8. Queries that "GROUP BY" a column from the joined partition are not allowed. + * 9. Any `join` call needs to be made before any `where` call. + * 10. Queries that join cross-partition with an "INNER JOIN" or "LEFT JOIN" with a condition on the left side + * cannot use "LIMIT" or "OFFSET" in queries. + * + * The part of the query running on the sharded table has some additional limitations, + * see the `InvalidShardedQueryException` documentation for more information. + */ +class InvalidPartitionedQueryException extends \Exception { + +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php b/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php new file mode 100644 index 00000000000..a08858d1d6b --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/JoinCondition.php @@ -0,0 +1,173 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\QueryFunction; +use OCP\DB\QueryBuilder\IQueryFunction; + +/** + * Utility class for working with join conditions + */ +class JoinCondition { + public function __construct( + public string|IQueryFunction $fromColumn, + public ?string $fromAlias, + public string|IQueryFunction $toColumn, + public ?string $toAlias, + public array $fromConditions, + public array $toConditions, + ) { + if (is_string($this->fromColumn) && str_starts_with($this->fromColumn, '(')) { + $this->fromColumn = new QueryFunction($this->fromColumn); + } + if (is_string($this->toColumn) && str_starts_with($this->toColumn, '(')) { + $this->toColumn = new QueryFunction($this->toColumn); + } + } + + /** + * @param JoinCondition[] $conditions + * @return JoinCondition + */ + public static function merge(array $conditions): JoinCondition { + $fromColumn = ''; + $toColumn = ''; + $fromAlias = null; + $toAlias = null; + $fromConditions = []; + $toConditions = []; + foreach ($conditions as $condition) { + if (($condition->fromColumn && $fromColumn) || ($condition->toColumn && $toColumn)) { + throw new InvalidPartitionedQueryException("Can't join from {$condition->fromColumn} to {$condition->toColumn} as it already join froms {$fromColumn} to {$toColumn}"); + } + if ($condition->fromColumn) { + $fromColumn = $condition->fromColumn; + } + if ($condition->toColumn) { + $toColumn = $condition->toColumn; + } + if ($condition->fromAlias) { + $fromAlias = $condition->fromAlias; + } + if ($condition->toAlias) { + $toAlias = $condition->toAlias; + } + $fromConditions = array_merge($fromConditions, $condition->fromConditions); + $toConditions = array_merge($toConditions, $condition->toConditions); + } + return new JoinCondition($fromColumn, $fromAlias, $toColumn, $toAlias, $fromConditions, $toConditions); + } + + /** + * @param null|string|CompositeExpression $condition + * @param string $join + * @param string $alias + * @param string $fromAlias + * @return JoinCondition + * @throws InvalidPartitionedQueryException + */ + public static function parse($condition, string $join, string $alias, string $fromAlias): JoinCondition { + if ($condition === null) { + throw new InvalidPartitionedQueryException("Can't join on $join without a condition"); + } + + $result = self::parseSubCondition($condition, $join, $alias, $fromAlias); + if (!$result->fromColumn || !$result->toColumn) { + throw new InvalidPartitionedQueryException("No join condition found from $fromAlias to $alias"); + } + return $result; + } + + private static function parseSubCondition($condition, string $join, string $alias, string $fromAlias): JoinCondition { + if ($condition instanceof CompositeExpression) { + if ($condition->getType() === CompositeExpression::TYPE_OR) { + throw new InvalidPartitionedQueryException("Cannot join on $join with an OR expression"); + } + return self::merge(array_map(function ($subCondition) use ($join, $alias, $fromAlias) { + return self::parseSubCondition($subCondition, $join, $alias, $fromAlias); + }, $condition->getParts())); + } + + $condition = (string)$condition; + $isSubCondition = self::isExtraCondition($condition); + if ($isSubCondition) { + if (self::mentionsAlias($condition, $fromAlias)) { + return new JoinCondition('', null, '', null, [$condition], []); + } else { + return new JoinCondition('', null, '', null, [], [$condition]); + } + } + + $condition = str_replace('`', '', $condition); + + // expect a condition in the form of 'alias1.column1 = alias2.column2' + if (!str_contains($condition, ' = ')) { + throw new InvalidPartitionedQueryException("Can only join on $join with an `eq` condition"); + } + $parts = explode(' = ', $condition, 2); + $parts = array_map(function (string $part) { + return self::clearConditionPart($part); + }, $parts); + + if (!self::isSingleCondition($parts[0]) || !self::isSingleCondition($parts[1])) { + throw new InvalidPartitionedQueryException("Can only join on $join with a single condition"); + } + + + if (self::mentionsAlias($parts[0], $fromAlias)) { + return new JoinCondition($parts[0], self::getAliasForPart($parts[0]), $parts[1], self::getAliasForPart($parts[1]), [], []); + } elseif (self::mentionsAlias($parts[1], $fromAlias)) { + return new JoinCondition($parts[1], self::getAliasForPart($parts[1]), $parts[0], self::getAliasForPart($parts[0]), [], []); + } else { + throw new InvalidPartitionedQueryException("join condition for $join needs to explicitly refer to the table by alias"); + } + } + + private static function isSingleCondition(string $condition): bool { + return !(str_contains($condition, ' OR ') || str_contains($condition, ' AND ')); + } + + private static function getAliasForPart(string $part): ?string { + if (str_contains($part, ' ')) { + return uniqid('join_alias_'); + } else { + return null; + } + } + + private static function clearConditionPart(string $part): string { + if (str_starts_with($part, 'CAST(')) { + // pgsql/mysql cast + $part = substr($part, strlen('CAST(')); + [$part] = explode(' AS ', $part); + } elseif (str_starts_with($part, 'to_number(to_char(')) { + // oracle cast to int + $part = substr($part, strlen('to_number(to_char('), -2); + } elseif (str_starts_with($part, 'to_number(to_char(')) { + // oracle cast to string + $part = substr($part, strlen('to_char('), -1); + } + return $part; + } + + /** + * Check that a condition is an extra limit on the from/to part, and not the join condition + * + * This is done by checking that only one of the halves of the condition references a column + */ + private static function isExtraCondition(string $condition): bool { + $parts = explode(' ', $condition, 2); + return str_contains($parts[0], '`') xor str_contains($parts[1], '`'); + } + + private static function mentionsAlias(string $condition, string $alias): bool { + return str_contains($condition, "$alias."); + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php new file mode 100644 index 00000000000..a5024b478d3 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionQuery.php @@ -0,0 +1,75 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OCP\DB\QueryBuilder\IQueryBuilder; + +/** + * A sub-query from a partitioned join + */ +class PartitionQuery { + public const JOIN_MODE_INNER = 'inner'; + public const JOIN_MODE_LEFT = 'left'; + // left-join where the left side IS NULL + public const JOIN_MODE_LEFT_NULL = 'left_null'; + + public const JOIN_MODE_RIGHT = 'right'; + + public function __construct( + public IQueryBuilder $query, + public string $joinFromColumn, + public string $joinToColumn, + public string $joinMode, + ) { + if ($joinMode !== self::JOIN_MODE_LEFT && $joinMode !== self::JOIN_MODE_INNER) { + throw new InvalidPartitionedQueryException("$joinMode joins aren't allowed in partitioned queries"); + } + } + + public function mergeWith(array $rows): array { + if (empty($rows)) { + return []; + } + // strip table/alias from column names + $joinFromColumn = preg_replace('/\w+\./', '', $this->joinFromColumn); + $joinToColumn = preg_replace('/\w+\./', '', $this->joinToColumn); + + $joinFromValues = array_map(function (array $row) use ($joinFromColumn) { + return $row[$joinFromColumn]; + }, $rows); + $joinFromValues = array_filter($joinFromValues, function ($value) { + return $value !== null; + }); + $this->query->andWhere($this->query->expr()->in($this->joinToColumn, $this->query->createNamedParameter($joinFromValues, IQueryBuilder::PARAM_STR_ARRAY, ':' . uniqid()))); + + $s = $this->query->getSQL(); + $partitionedRows = $this->query->executeQuery()->fetchAll(); + + $columns = $this->query->getOutputColumns(); + $nullResult = array_combine($columns, array_fill(0, count($columns), null)); + + $partitionedRowsByKey = []; + foreach ($partitionedRows as $partitionedRow) { + $partitionedRowsByKey[$partitionedRow[$joinToColumn]][] = $partitionedRow; + } + $result = []; + foreach ($rows as $row) { + if (isset($partitionedRowsByKey[$row[$joinFromColumn]])) { + if ($this->joinMode !== self::JOIN_MODE_LEFT_NULL) { + foreach ($partitionedRowsByKey[$row[$joinFromColumn]] as $partitionedRow) { + $result[] = array_merge($row, $partitionedRow); + } + } + } elseif ($this->joinMode === self::JOIN_MODE_LEFT || $this->joinMode === self::JOIN_MODE_LEFT_NULL) { + $result[] = array_merge($nullResult, $row); + } + } + return $result; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php new file mode 100644 index 00000000000..ad4c0fab055 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionSplit.php @@ -0,0 +1,74 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +/** + * Information about a database partition, containing the tables in the partition and any active alias + */ +class PartitionSplit { + /** @var array<string, string> */ + public array $aliases = []; + + /** + * @param string[] $tables + */ + public function __construct( + public string $name, + public array $tables, + ) { + } + + public function addAlias(string $table, string $alias): void { + if ($this->containsTable($table)) { + $this->aliases[$alias] = $table; + } + } + + public function addTable(string $table): void { + if (!$this->containsTable($table)) { + $this->tables[] = $table; + } + } + + public function containsTable(string $table): bool { + return in_array($table, $this->tables); + } + + public function containsAlias(string $alias): bool { + return array_key_exists($alias, $this->aliases); + } + + private function getTablesAndAliases(): array { + return array_keys($this->aliases) + $this->tables; + } + + /** + * Check if a query predicate mentions a table or alias from this partition + * + * @param string $predicate + * @return bool + */ + public function checkPredicateForTable(string $predicate): bool { + foreach ($this->getTablesAndAliases() as $name) { + if (str_contains($predicate, "`$name`.`")) { + return true; + } + } + return false; + } + + public function isColumnInPartition(string $column): bool { + foreach ($this->getTablesAndAliases() as $name) { + if (str_starts_with($column, "$name.")) { + return true; + } + } + return false; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php new file mode 100644 index 00000000000..d748c791321 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php @@ -0,0 +1,462 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\QuoteHelper; +use OC\DB\QueryBuilder\Sharded\AutoIncrementHandler; +use OC\DB\QueryBuilder\Sharded\ShardConnectionManager; +use OC\DB\QueryBuilder\Sharded\ShardedQueryBuilder; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\DB\QueryBuilder\IQueryFunction; +use OCP\IDBConnection; + +/** + * A special query builder that automatically splits queries that span across multiple database partitions[1]. + * + * This is done by inspecting the query as it's being built, and when a cross-partition join is detected, + * the part of the query that touches the partition is split of into a different sub-query. + * Then, when the query is executed, the results from the sub-queries are automatically merged. + * + * This whole process is intended to be transparent to any code using the query builder, however it does impose some extra + * limitation for queries that work cross-partition. See the documentation from `InvalidPartitionedQueryException` for more details. + * + * When a join is created in the query, this builder checks if it belongs to the same partition as the table from the + * original FROM/UPDATE/DELETE/INSERT and if not, creates a new "sub query" for the partition. + * Then for every part that is added the query, the part is analyzed to determine which partition the query part is referencing + * and the query part is added to the sub query for that partition. + * + * [1]: A set of tables which can't be queried together with the rest of the tables, such as when sharding is used. + */ +class PartitionedQueryBuilder extends ShardedQueryBuilder { + /** @var array<string, PartitionQuery> $splitQueries */ + private array $splitQueries = []; + /** @var list<PartitionSplit> */ + private array $partitions = []; + + /** @var array{'select': string|array, 'alias': ?string}[] */ + private array $selects = []; + private ?PartitionSplit $mainPartition = null; + private bool $hasPositionalParameter = false; + private QuoteHelper $quoteHelper; + private ?int $limit = null; + private ?int $offset = null; + + public function __construct( + IQueryBuilder $builder, + array $shardDefinitions, + ShardConnectionManager $shardConnectionManager, + AutoIncrementHandler $autoIncrementHandler, + ) { + parent::__construct($builder, $shardDefinitions, $shardConnectionManager, $autoIncrementHandler); + $this->quoteHelper = new QuoteHelper(); + } + + private function newQuery(): IQueryBuilder { + // get a fresh, non-partitioning query builder + $builder = $this->builder->getConnection()->getQueryBuilder(); + if ($builder instanceof PartitionedQueryBuilder) { + $builder = $builder->builder; + } + + return new ShardedQueryBuilder( + $builder, + $this->shardDefinitions, + $this->shardConnectionManager, + $this->autoIncrementHandler, + ); + } + + // we need to save selects until we know all the table aliases + public function select(...$selects) { + if (count($selects) === 1 && is_array($selects[0])) { + $selects = $selects[0]; + } + $this->selects = []; + $this->addSelect(...$selects); + return $this; + } + + public function addSelect(...$select) { + $select = array_map(function ($select) { + return ['select' => $select, 'alias' => null]; + }, $select); + $this->selects = array_merge($this->selects, $select); + return $this; + } + + public function selectAlias($select, $alias) { + $this->selects[] = ['select' => $select, 'alias' => $alias]; + return $this; + } + + /** + * Ensure that a column is being selected by the query + * + * This is mainly used to ensure that the returned rows from both sides of a partition contains the columns of the join predicate + * + * @param string|IQueryFunction $column + * @return void + */ + private function ensureSelect(string|IQueryFunction $column, ?string $alias = null): void { + $checkColumn = $alias ?: $column; + if (str_contains($checkColumn, '.')) { + [$table, $checkColumn] = explode('.', $checkColumn); + $partition = $this->getPartition($table); + } else { + $partition = null; + } + foreach ($this->selects as $select) { + $select = $select['select']; + if (!is_string($select)) { + continue; + } + + if (str_contains($select, '.')) { + [$table, $select] = explode('.', $select); + $selectPartition = $this->getPartition($table); + } else { + $selectPartition = null; + } + if ( + ($select === $checkColumn || $select === '*') + && $selectPartition === $partition + ) { + return; + } + } + if ($alias) { + $this->selectAlias($column, $alias); + } else { + $this->addSelect($column); + } + } + + /** + * Distribute the select statements to the correct partition + * + * This is done at the end instead of when the `select` call is made, because the `select` calls are generally done + * before we know what tables are involved in the query + * + * @return void + */ + private function applySelects(): void { + foreach ($this->selects as $select) { + foreach ($this->partitions as $partition) { + if (is_string($select['select']) && ( + $select['select'] === '*' + || $partition->isColumnInPartition($select['select'])) + ) { + if (isset($this->splitQueries[$partition->name])) { + if ($select['alias']) { + $this->splitQueries[$partition->name]->query->selectAlias($select['select'], $select['alias']); + } else { + $this->splitQueries[$partition->name]->query->addSelect($select['select']); + } + if ($select['select'] !== '*') { + continue 2; + } + } + } + } + + if ($select['alias']) { + parent::selectAlias($select['select'], $select['alias']); + } else { + parent::addSelect($select['select']); + } + } + $this->selects = []; + } + + + public function addPartition(PartitionSplit $partition): void { + $this->partitions[] = $partition; + } + + private function getPartition(string $table): ?PartitionSplit { + foreach ($this->partitions as $partition) { + if ($partition->containsTable($table) || $partition->containsAlias($table)) { + return $partition; + } + } + return null; + } + + public function from($from, $alias = null) { + if (is_string($from) && $partition = $this->getPartition($from)) { + $this->mainPartition = $partition; + if ($alias) { + $this->mainPartition->addAlias($from, $alias); + } + } + return parent::from($from, $alias); + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null): self { + return $this->join($fromAlias, $join, $alias, $condition); + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null): self { + return $this->join($fromAlias, (string)$join, $alias, $condition, PartitionQuery::JOIN_MODE_LEFT); + } + + public function join($fromAlias, $join, $alias, $condition = null, $joinMode = PartitionQuery::JOIN_MODE_INNER): self { + $partition = $this->getPartition($join); + $fromPartition = $this->getPartition($fromAlias); + if ($partition && $partition !== $this->mainPartition) { + // join from the main db to a partition + + $joinCondition = JoinCondition::parse($condition, $join, $alias, $fromAlias); + $partition->addAlias($join, $alias); + + if (!isset($this->splitQueries[$partition->name])) { + $this->splitQueries[$partition->name] = new PartitionQuery( + $this->newQuery(), + $joinCondition->fromAlias ?? $joinCondition->fromColumn, $joinCondition->toAlias ?? $joinCondition->toColumn, + $joinMode + ); + $this->splitQueries[$partition->name]->query->from($join, $alias); + $this->ensureSelect($joinCondition->fromColumn, $joinCondition->fromAlias); + $this->ensureSelect($joinCondition->toColumn, $joinCondition->toAlias); + } else { + $query = $this->splitQueries[$partition->name]->query; + if ($partition->containsAlias($fromAlias)) { + $query->innerJoin($fromAlias, $join, $alias, $condition); + } else { + throw new InvalidPartitionedQueryException("Can't join across partition boundaries more than once"); + } + } + $this->splitQueries[$partition->name]->query->andWhere(...$joinCondition->toConditions); + parent::andWhere(...$joinCondition->fromConditions); + return $this; + } elseif ($fromPartition && $fromPartition !== $partition) { + // join from partition, to the main db + + $joinCondition = JoinCondition::parse($condition, $join, $alias, $fromAlias); + if (str_starts_with($fromPartition->name, 'from_')) { + $partitionName = $fromPartition->name; + } else { + $partitionName = 'from_' . $fromPartition->name; + } + + if (!isset($this->splitQueries[$partitionName])) { + $newPartition = new PartitionSplit($partitionName, [$join]); + $newPartition->addAlias($join, $alias); + $this->partitions[] = $newPartition; + + $this->splitQueries[$partitionName] = new PartitionQuery( + $this->newQuery(), + $joinCondition->fromAlias ?? $joinCondition->fromColumn, $joinCondition->toAlias ?? $joinCondition->toColumn, + $joinMode + ); + $this->ensureSelect($joinCondition->fromColumn, $joinCondition->fromAlias); + $this->ensureSelect($joinCondition->toColumn, $joinCondition->toAlias); + $this->splitQueries[$partitionName]->query->from($join, $alias); + $this->splitQueries[$partitionName]->query->andWhere(...$joinCondition->toConditions); + parent::andWhere(...$joinCondition->fromConditions); + } else { + $fromPartition->addTable($join); + $fromPartition->addAlias($join, $alias); + + $query = $this->splitQueries[$partitionName]->query; + $query->innerJoin($fromAlias, $join, $alias, $condition); + } + return $this; + } else { + // join within the main db or a partition + if ($joinMode === PartitionQuery::JOIN_MODE_INNER) { + return parent::innerJoin($fromAlias, $join, $alias, $condition); + } elseif ($joinMode === PartitionQuery::JOIN_MODE_LEFT) { + return parent::leftJoin($fromAlias, $join, $alias, $condition); + } elseif ($joinMode === PartitionQuery::JOIN_MODE_RIGHT) { + return parent::rightJoin($fromAlias, $join, $alias, $condition); + } else { + throw new \InvalidArgumentException("Invalid join mode: $joinMode"); + } + } + } + + /** + * Flatten a list of predicates by merging the parts of any "AND" expression into the list of predicates + * + * @param array $predicates + * @return array + */ + private function flattenPredicates(array $predicates): array { + $result = []; + foreach ($predicates as $predicate) { + if ($predicate instanceof CompositeExpression && $predicate->getType() === CompositeExpression::TYPE_AND) { + $result = array_merge($result, $this->flattenPredicates($predicate->getParts())); + } else { + $result[] = $predicate; + } + } + return $result; + } + + /** + * Split an array of predicates (WHERE query parts) by the partition they reference + * + * @param array $predicates + * @return array<string, array> + */ + private function splitPredicatesByParts(array $predicates): array { + $predicates = $this->flattenPredicates($predicates); + + $partitionPredicates = []; + foreach ($predicates as $predicate) { + $partition = $this->getPartitionForPredicate((string)$predicate); + if ($this->mainPartition === $partition) { + $partitionPredicates[''][] = $predicate; + } elseif ($partition) { + $partitionPredicates[$partition->name][] = $predicate; + } else { + $partitionPredicates[''][] = $predicate; + } + } + return $partitionPredicates; + } + + public function where(...$predicates) { + return $this->andWhere(...$predicates); + } + + public function andWhere(...$where) { + if ($where) { + foreach ($this->splitPredicatesByParts($where) as $alias => $predicates) { + if (isset($this->splitQueries[$alias])) { + // when there is a condition on a table being left-joined it starts to behave as if it's an inner join + // since any joined column that doesn't have the left part will not match the condition + // when there the condition is `$joinToColumn IS NULL` we instead mark the query as excluding the left half + if ($this->splitQueries[$alias]->joinMode === PartitionQuery::JOIN_MODE_LEFT) { + $this->splitQueries[$alias]->joinMode = PartitionQuery::JOIN_MODE_INNER; + + $column = $this->quoteHelper->quoteColumnName($this->splitQueries[$alias]->joinToColumn); + foreach ($predicates as $predicate) { + if ((string)$predicate === "$column IS NULL") { + $this->splitQueries[$alias]->joinMode = PartitionQuery::JOIN_MODE_LEFT_NULL; + } else { + $this->splitQueries[$alias]->query->andWhere($predicate); + } + } + } else { + $this->splitQueries[$alias]->query->andWhere(...$predicates); + } + } else { + parent::andWhere(...$predicates); + } + } + } + return $this; + } + + + private function getPartitionForPredicate(string $predicate): ?PartitionSplit { + foreach ($this->partitions as $partition) { + + if (str_contains($predicate, '?')) { + $this->hasPositionalParameter = true; + } + if ($partition->checkPredicateForTable($predicate)) { + return $partition; + } + } + return null; + } + + public function update($update = null, $alias = null) { + return parent::update($update, $alias); + } + + public function insert($insert = null) { + return parent::insert($insert); + } + + public function delete($delete = null, $alias = null) { + return parent::delete($delete, $alias); + } + + public function setMaxResults($maxResults) { + if ($maxResults > 0) { + $this->limit = (int)$maxResults; + } + return parent::setMaxResults($maxResults); + } + + public function setFirstResult($firstResult) { + if ($firstResult > 0) { + $this->offset = (int)$firstResult; + } + return parent::setFirstResult($firstResult); + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + $this->applySelects(); + if ($this->splitQueries && $this->hasPositionalParameter) { + throw new InvalidPartitionedQueryException("Partitioned queries aren't allowed to to positional arguments"); + } + foreach ($this->splitQueries as $split) { + $split->query->setParameters($this->getParameters(), $this->getParameterTypes()); + } + if (count($this->splitQueries) > 0) { + $hasNonLeftJoins = array_reduce($this->splitQueries, function (bool $hasNonLeftJoins, PartitionQuery $query) { + return $hasNonLeftJoins || $query->joinMode !== PartitionQuery::JOIN_MODE_LEFT; + }, false); + if ($hasNonLeftJoins) { + if (is_int($this->limit)) { + throw new InvalidPartitionedQueryException('Limit is not allowed in partitioned queries'); + } + if (is_int($this->offset)) { + throw new InvalidPartitionedQueryException('Offset is not allowed in partitioned queries'); + } + } + } + + $s = $this->getSQL(); + $result = parent::executeQuery($connection); + if (count($this->splitQueries) > 0) { + return new PartitionedResult($this->splitQueries, $result); + } else { + return $result; + } + } + + public function executeStatement(?IDBConnection $connection = null): int { + if (count($this->splitQueries)) { + throw new InvalidPartitionedQueryException("Partitioning write queries isn't supported"); + } + return parent::executeStatement($connection); + } + + public function getSQL() { + $this->applySelects(); + return parent::getSQL(); + } + + public function getPartitionCount(): int { + return count($this->splitQueries) + 1; + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if (str_contains($column, '.')) { + [$alias, $column] = explode('.', $column); + $partition = $this->getPartition($alias); + if ($partition) { + $this->splitQueries[$partition->name]->query->hintShardKey($column, $value, $overwrite); + } else { + parent::hintShardKey($column, $value, $overwrite); + } + } else { + parent::hintShardKey($column, $value, $overwrite); + } + return $this; + } +} diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php new file mode 100644 index 00000000000..b3b59e26298 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedResult.php @@ -0,0 +1,61 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Partitioned; + +use OC\DB\ArrayResult; +use OCP\DB\IResult; +use PDO; + +/** + * Combine the results of multiple join parts into a single result + */ +class PartitionedResult extends ArrayResult { + private bool $fetched = false; + + /** + * @param PartitionQuery[] $splitOfParts + * @param IResult $result + */ + public function __construct( + private array $splitOfParts, + private IResult $result, + ) { + parent::__construct([]); + } + + public function closeCursor(): bool { + return $this->result->closeCursor(); + } + + public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { + $this->fetchRows(); + return parent::fetch($fetchMode); + } + + public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { + $this->fetchRows(); + return parent::fetchAll($fetchMode); + } + + public function rowCount(): int { + $this->fetchRows(); + return parent::rowCount(); + } + + private function fetchRows(): void { + if (!$this->fetched) { + $this->fetched = true; + $this->rows = $this->result->fetchAll(); + foreach ($this->splitOfParts as $part) { + $this->rows = $part->mergeWith($this->rows); + } + $this->count = count($this->rows); + } + } +} diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 0e7d8d2ff3e..1d44c049793 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -7,13 +7,9 @@ */ namespace OC\DB\QueryBuilder; -use Doctrine\DBAL\Platforms\MySQLPlatform; -use Doctrine\DBAL\Platforms\OraclePlatform; -use Doctrine\DBAL\Platforms\PostgreSQL94Platform; -use Doctrine\DBAL\Platforms\SqlitePlatform; use Doctrine\DBAL\Query\QueryException; use OC\DB\ConnectionAdapter; -use OC\DB\QueryBuilder\ExpressionBuilder\ExpressionBuilder; +use OC\DB\Exceptions\DbalException; use OC\DB\QueryBuilder\ExpressionBuilder\MySqlExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\OCIExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\PgSqlExpressionBuilder; @@ -22,7 +18,6 @@ use OC\DB\QueryBuilder\FunctionBuilder\FunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\OCIFunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\PgSqlFunctionBuilder; use OC\DB\QueryBuilder\FunctionBuilder\SqliteFunctionBuilder; -use OC\DB\ResultAdapter; use OC\SystemConfig; use OCP\DB\IResult; use OCP\DB\QueryBuilder\ICompositeExpression; @@ -30,6 +25,7 @@ use OCP\DB\QueryBuilder\ILiteral; use OCP\DB\QueryBuilder\IParameter; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; +use OCP\IDBConnection; use Psr\Log\LoggerInterface; class QueryBuilder implements IQueryBuilder { @@ -49,9 +45,11 @@ class QueryBuilder implements IQueryBuilder { /** @var bool */ private $automaticTablePrefix = true; + private bool $nonEmptyWhere = false; /** @var string */ protected $lastInsertedTable; + private array $selectedColumns = []; /** * Initializes a new QueryBuilder. @@ -71,11 +69,11 @@ class QueryBuilder implements IQueryBuilder { * Enable/disable automatic prefixing of table names with the oc_ prefix * * @param bool $enabled If set to true table names will be prefixed with the - * owncloud database prefix automatically. + * owncloud database prefix automatically. * @since 8.2.0 */ public function automaticTablePrefix($enabled) { - $this->automaticTablePrefix = (bool) $enabled; + $this->automaticTablePrefix = (bool)$enabled; } /** @@ -95,20 +93,13 @@ class QueryBuilder implements IQueryBuilder { * @return \OCP\DB\QueryBuilder\IExpressionBuilder */ public function expr() { - if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { - return new OCIExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { - return new PgSqlExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof MySQLPlatform) { - return new MySqlExpressionBuilder($this->connection, $this); - } - if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { - return new SqliteExpressionBuilder($this->connection, $this); - } - - return new ExpressionBuilder($this->connection, $this); + return match($this->connection->getDatabaseProvider()) { + IDBConnection::PLATFORM_ORACLE => new OCIExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_POSTGRES => new PgSqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_MARIADB, + IDBConnection::PLATFORM_MYSQL => new MySqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_SQLITE => new SqliteExpressionBuilder($this->connection, $this, $this->logger), + }; } /** @@ -128,17 +119,13 @@ class QueryBuilder implements IQueryBuilder { * @return \OCP\DB\QueryBuilder\IFunctionBuilder */ public function func() { - if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { - return new OCIFunctionBuilder($this->connection, $this, $this->helper); - } - if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { - return new SqliteFunctionBuilder($this->connection, $this, $this->helper); - } - if ($this->connection->getDatabasePlatform() instanceof PostgreSQL94Platform) { - return new PgSqlFunctionBuilder($this->connection, $this, $this->helper); - } - - return new FunctionBuilder($this->connection, $this, $this->helper); + return match($this->connection->getDatabaseProvider()) { + IDBConnection::PLATFORM_ORACLE => new OCIFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_POSTGRES => new PgSqlFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_MARIADB, + IDBConnection::PLATFORM_MYSQL => new FunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_SQLITE => new SqliteFunctionBuilder($this->connection, $this, $this->helper), + }; } /** @@ -162,26 +149,21 @@ class QueryBuilder implements IQueryBuilder { /** * Gets the state of this query builder instance. * - * @return integer Either QueryBuilder::STATE_DIRTY or QueryBuilder::STATE_CLEAN. + * @return int Always returns 0 which is former `QueryBuilder::STATE_DIRTY` + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. */ public function getState() { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getState(); } - /** - * Executes this query using the bound parameters and their types. - * - * Uses {@see Connection::executeQuery} for select statements and {@see Connection::executeUpdate} - * for insert, update and delete statements. - * - * @return IResult|int - */ - public function execute() { + private function prepareForExecute() { if ($this->systemConfig->getValue('log_query', false)) { try { $params = []; foreach ($this->getParameters() as $placeholder => $value) { - if ($value instanceof \DateTime) { + if ($value instanceof \DateTimeInterface) { $params[] = $placeholder . ' => DateTime:\'' . $value->format('c') . '\''; } elseif (is_array($value)) { $params[] = $placeholder . ' => (\'' . implode('\', \'', $value) . '\')'; @@ -207,25 +189,42 @@ class QueryBuilder implements IQueryBuilder { } } - if (!empty($this->getQueryPart('select'))) { - $select = $this->getQueryPart('select'); - $hasSelectAll = array_filter($select, static function ($s) { - return $s === '*'; - }); - $hasSelectSpecific = array_filter($select, static function ($s) { - return $s !== '*'; - }); - - if (empty($hasSelectAll) === empty($hasSelectSpecific)) { - $exception = new QueryException('Query is selecting * and specific values in the same query. This is not supported in Oracle.'); - $this->logger->error($exception->getMessage(), [ - 'query' => $this->getSQL(), - 'app' => 'core', - 'exception' => $exception, - ]); + // if (!empty($this->getQueryPart('select'))) { + // $select = $this->getQueryPart('select'); + // $hasSelectAll = array_filter($select, static function ($s) { + // return $s === '*'; + // }); + // $hasSelectSpecific = array_filter($select, static function ($s) { + // return $s !== '*'; + // }); + + // if (empty($hasSelectAll) === empty($hasSelectSpecific)) { + // $exception = new QueryException('Query is selecting * and specific values in the same query. This is not supported in Oracle.'); + // $this->logger->error($exception->getMessage(), [ + // 'query' => $this->getSQL(), + // 'app' => 'core', + // 'exception' => $exception, + // ]); + // } + // } + + $tooLongOutputColumns = []; + foreach ($this->getOutputColumns() as $column) { + if (strlen($column) > 30) { + $tooLongOutputColumns[] = $column; } } + if (!empty($tooLongOutputColumns)) { + $exception = new QueryException('More than 30 characters for an output column name are not allowed on Oracle.'); + $this->logger->error($exception->getMessage(), [ + 'query' => $this->getSQL(), + 'columns' => $tooLongOutputColumns, + 'app' => 'core', + 'exception' => $exception, + ]); + } + $numberOfParameters = 0; $hasTooLargeArrayParameter = false; foreach ($this->getParameters() as $parameter) { @@ -253,48 +252,64 @@ class QueryBuilder implements IQueryBuilder { 'exception' => $exception, ]); } + } + + /** + * Executes this query using the bound parameters and their types. + * + * Uses {@see Connection::executeQuery} for select statements and {@see Connection::executeUpdate} + * for insert, update and delete statements. + * + * @return IResult|int + */ + public function execute(?IDBConnection $connection = null) { + try { + if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { + return $this->executeQuery($connection); + } else { + return $this->executeStatement($connection); + } + } catch (DBALException $e) { + // `IQueryBuilder->execute` never wrapped the exception, but `executeQuery` and `executeStatement` do + /** @var \Doctrine\DBAL\Exception $previous */ + $previous = $e->getPrevious(); - $result = $this->queryBuilder->execute(); - if (is_int($result)) { - return $result; + throw $previous; } - return new ResultAdapter($result); } - public function executeQuery(): IResult { + public function executeQuery(?IDBConnection $connection = null): IResult { if ($this->getType() !== \Doctrine\DBAL\Query\QueryBuilder::SELECT) { throw new \RuntimeException('Invalid query type, expected SELECT query'); } - try { - $result = $this->execute(); - } catch (\Doctrine\DBAL\Exception $e) { - throw \OC\DB\Exceptions\DbalException::wrap($e); - } - - if ($result instanceof IResult) { - return $result; + $this->prepareForExecute(); + if (!$connection) { + $connection = $this->connection; } - throw new \RuntimeException('Invalid return type for query'); + return $connection->executeQuery( + $this->getSQL(), + $this->getParameters(), + $this->getParameterTypes(), + ); } - public function executeStatement(): int { + public function executeStatement(?IDBConnection $connection = null): int { if ($this->getType() === \Doctrine\DBAL\Query\QueryBuilder::SELECT) { throw new \RuntimeException('Invalid query type, expected INSERT, DELETE or UPDATE statement'); } - try { - $result = $this->execute(); - } catch (\Doctrine\DBAL\Exception $e) { - throw \OC\DB\Exceptions\DbalException::wrap($e); - } - - if (!is_int($result)) { - throw new \RuntimeException('Invalid return type for statement'); + $this->prepareForExecute(); + if (!$connection) { + $connection = $this->connection; } - return $result; + return $connection->executeStatement( + $this->getSQL(), + $this->getParameters(), + $this->getParameterTypes(), + ); } @@ -410,7 +425,7 @@ class QueryBuilder implements IQueryBuilder { * @return $this This QueryBuilder instance. */ public function setFirstResult($firstResult) { - $this->queryBuilder->setFirstResult((int) $firstResult); + $this->queryBuilder->setFirstResult((int)$firstResult); return $this; } @@ -440,7 +455,7 @@ class QueryBuilder implements IQueryBuilder { if ($maxResults === null) { $this->queryBuilder->setMaxResults($maxResults); } else { - $this->queryBuilder->setMaxResults((int) $maxResults); + $this->queryBuilder->setMaxResults((int)$maxResults); } return $this; @@ -475,6 +490,7 @@ class QueryBuilder implements IQueryBuilder { if (count($selects) === 1 && is_array($selects[0])) { $selects = $selects[0]; } + $this->addOutputColumns($selects); $this->queryBuilder->select( $this->helper->quoteColumnNames($selects) @@ -502,6 +518,7 @@ class QueryBuilder implements IQueryBuilder { $this->queryBuilder->addSelect( $this->helper->quoteColumnName($select) . ' AS ' . $this->helper->quoteColumnName($alias) ); + $this->addOutputColumns([$alias]); return $this; } @@ -523,6 +540,7 @@ class QueryBuilder implements IQueryBuilder { if (!is_array($select)) { $select = [$select]; } + $this->addOutputColumns($select); $quotedSelect = $this->helper->quoteColumnNames($select); @@ -552,6 +570,7 @@ class QueryBuilder implements IQueryBuilder { if (count($selects) === 1 && is_array($selects[0])) { $selects = $selects[0]; } + $this->addOutputColumns($selects); $this->queryBuilder->addSelect( $this->helper->quoteColumnNames($selects) @@ -560,6 +579,26 @@ class QueryBuilder implements IQueryBuilder { return $this; } + private function addOutputColumns(array $columns): void { + foreach ($columns as $column) { + if (is_array($column)) { + $this->addOutputColumns($column); + } elseif (is_string($column) && !str_contains($column, '*')) { + if (str_contains(strtolower($column), ' as ')) { + [, $column] = preg_split('/ as /i', $column); + } + if (str_contains($column, '.')) { + [, $column] = explode('.', $column); + } + $this->selectedColumns[] = $column; + } + } + } + + public function getOutputColumns(): array { + return array_unique($this->selectedColumns); + } + /** * Turns the query being built into a bulk delete query that ranges over * a certain table. @@ -575,8 +614,13 @@ class QueryBuilder implements IQueryBuilder { * @param string $alias The table alias used in the constructed query. * * @return $this This QueryBuilder instance. + * @since 30.0.0 Alias is deprecated and will no longer be used with the next Doctrine/DBAL update */ public function delete($delete = null, $alias = null) { + if ($alias !== null) { + $this->logger->debug('DELETE queries with alias are no longer supported and the provided alias is ignored', ['exception' => new \InvalidArgumentException('Table alias provided for DELETE query')]); + } + $this->queryBuilder->delete( $this->getTableName($delete), $alias @@ -600,8 +644,13 @@ class QueryBuilder implements IQueryBuilder { * @param string $alias The table alias used in the constructed query. * * @return $this This QueryBuilder instance. + * @since 30.0.0 Alias is deprecated and will no longer be used with the next Doctrine/DBAL update */ public function update($update = null, $alias = null) { + if ($alias !== null) { + $this->logger->debug('UPDATE queries with alias are no longer supported and the provided alias is ignored', ['exception' => new \InvalidArgumentException('Table alias provided for UPDATE query')]); + } + $this->queryBuilder->update( $this->getTableName($update), $alias @@ -730,7 +779,7 @@ class QueryBuilder implements IQueryBuilder { * </code> * * @param string $fromAlias The alias that points to a from clause. - * @param string $join The table name to join. + * @param string|IQueryFunction $join The table name or sub-query to join. * @param string $alias The alias of the join table. * @param string|ICompositeExpression|null $condition The condition for the join. * @@ -812,9 +861,10 @@ class QueryBuilder implements IQueryBuilder { * // You can optionally programmatically build and/or expressions * $qb = $conn->getQueryBuilder(); * - * $or = $qb->expr()->orx(); - * $or->add($qb->expr()->eq('u.id', 1)); - * $or->add($qb->expr()->eq('u.id', 2)); + * $or = $qb->expr()->orx( + * $qb->expr()->eq('u.id', 1), + * $qb->expr()->eq('u.id', 2), + * ); * * $qb->update('users', 'u') * ->set('u.password', md5('password')) @@ -826,12 +876,14 @@ class QueryBuilder implements IQueryBuilder { * @return $this This QueryBuilder instance. */ public function where(...$predicates) { - if ($this->getQueryPart('where') !== null && $this->systemConfig->getValue('debug', false)) { + if ($this->nonEmptyWhere && $this->systemConfig->getValue('debug', false)) { // Only logging a warning, not throwing for now. $e = new QueryException('Using where() on non-empty WHERE part, please verify it is intentional to not call andWhere() or orWhere() instead. Otherwise consider creating a new query builder object or call resetQueryPart(\'where\') first.'); $this->logger->warning($e->getMessage(), ['exception' => $e]); } + $this->nonEmptyWhere = true; + call_user_func_array( [$this->queryBuilder, 'where'], $predicates @@ -859,6 +911,7 @@ class QueryBuilder implements IQueryBuilder { * @see where() */ public function andWhere(...$where) { + $this->nonEmptyWhere = true; call_user_func_array( [$this->queryBuilder, 'andWhere'], $where @@ -886,6 +939,7 @@ class QueryBuilder implements IQueryBuilder { * @see where() */ public function orWhere(...$where) { + $this->nonEmptyWhere = true; call_user_func_array( [$this->queryBuilder, 'orWhere'], $where @@ -968,7 +1022,7 @@ class QueryBuilder implements IQueryBuilder { public function setValue($column, $value) { $this->queryBuilder->setValue( $this->helper->quoteColumnName($column), - (string) $value + (string)$value ); return $this; @@ -1096,8 +1150,11 @@ class QueryBuilder implements IQueryBuilder { * @param string $queryPartName * * @return mixed + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please track the details you need, outside the object. */ public function getQueryPart($queryPartName) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getQueryPart($queryPartName); } @@ -1105,8 +1162,11 @@ class QueryBuilder implements IQueryBuilder { * Gets all query parts. * * @return array + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please track the details you need, outside the object. */ public function getQueryParts() { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); return $this->queryBuilder->getQueryParts(); } @@ -1116,8 +1176,11 @@ class QueryBuilder implements IQueryBuilder { * @param array|null $queryPartNames * * @return $this This QueryBuilder instance. + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please create a new IQueryBuilder instead. */ public function resetQueryParts($queryPartNames = null) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); $this->queryBuilder->resetQueryParts($queryPartNames); return $this; @@ -1129,8 +1192,11 @@ class QueryBuilder implements IQueryBuilder { * @param string $queryPartName * * @return $this This QueryBuilder instance. + * @deprecated 30.0.0 This function is going to be removed with the next Doctrine/DBAL update + * and we can not fix this in our wrapper. Please create a new IQueryBuilder instead. */ public function resetQueryPart($queryPartName) { + $this->logger->debug(IQueryBuilder::class . '::' . __FUNCTION__ . ' is deprecated and will be removed soon.', ['exception' => new \Exception('Deprecated call to ' . __METHOD__)]); $this->queryBuilder->resetQueryPart($queryPartName); return $this; @@ -1265,7 +1331,7 @@ class QueryBuilder implements IQueryBuilder { */ public function getTableName($table) { if ($table instanceof IQueryFunction) { - return (string) $table; + return (string)$table; } $table = $this->prefixTableName($table); @@ -1275,10 +1341,12 @@ class QueryBuilder implements IQueryBuilder { /** * Returns the table name with database prefix as needed by the implementation * + * Was protected until version 30. + * * @param string $table * @return string */ - protected function prefixTableName($table) { + public function prefixTableName(string $table): string { if ($this->automaticTablePrefix === false || str_starts_with($table, '*PREFIX*')) { return $table; } @@ -1314,4 +1382,18 @@ class QueryBuilder implements IQueryBuilder { return $this->helper->quoteColumnName($alias); } + + public function escapeLikeParameter(string $parameter): string { + return $this->connection->escapeLikeParameter($parameter); + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + return $this; + } + + public function runAcrossAllShards(): self { + // noop + return $this; + } + } diff --git a/lib/private/DB/QueryBuilder/QueryFunction.php b/lib/private/DB/QueryBuilder/QueryFunction.php index 9cdd6c31c7b..7f2ab584a73 100644 --- a/lib/private/DB/QueryBuilder/QueryFunction.php +++ b/lib/private/DB/QueryBuilder/QueryFunction.php @@ -18,6 +18,6 @@ class QueryFunction implements IQueryFunction { } public function __toString(): string { - return (string) $this->function; + return (string)$this->function; } } diff --git a/lib/private/DB/QueryBuilder/QuoteHelper.php b/lib/private/DB/QueryBuilder/QuoteHelper.php index a60a9731aa2..7ce4b359638 100644 --- a/lib/private/DB/QueryBuilder/QuoteHelper.php +++ b/lib/private/DB/QueryBuilder/QuoteHelper.php @@ -35,7 +35,7 @@ class QuoteHelper { */ public function quoteColumnName($string) { if ($string instanceof IParameter || $string instanceof ILiteral || $string instanceof IQueryFunction) { - return (string) $string; + return (string)$string; } if ($string === null || $string === 'null' || $string === '*') { diff --git a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php new file mode 100644 index 00000000000..3a230ea544d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php @@ -0,0 +1,155 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\ICacheFactory; +use OCP\IMemcache; +use OCP\IMemcacheTTL; + +/** + * A helper to atomically determine the next auto increment value for a sharded table + * + * Since we can't use the database's auto-increment (since each db doesn't know about the keys in the other shards) + * we need external logic for doing the auto increment + */ +class AutoIncrementHandler { + public const MIN_VALID_KEY = 1000; + public const TTL = 365 * 24 * 60 * 60; + + private ?IMemcache $cache = null; + + public function __construct( + private ICacheFactory $cacheFactory, + private ShardConnectionManager $shardConnectionManager, + ) { + if (PHP_INT_SIZE < 8) { + throw new \Exception('sharding is only supported with 64bit php'); + } + } + + private function getCache(): IMemcache { + if (is_null($this->cache)) { + $cache = $this->cacheFactory->createDistributed('shared_autoincrement'); + if ($cache instanceof IMemcache) { + $this->cache = $cache; + } else { + throw new \Exception('Distributed cache ' . get_class($cache) . ' is not suitable'); + } + } + return $this->cache; + } + + /** + * Get the next value for the given shard definition + * + * The returned key is unique and incrementing, but not sequential. + * The shard id is encoded in the first byte of the returned value + * + * @param ShardDefinition $shardDefinition + * @return int + * @throws \Exception + */ + public function getNextPrimaryKey(ShardDefinition $shardDefinition, int $shard): int { + $retries = 0; + while ($retries < 5) { + $next = $this->getNextInner($shardDefinition); + if ($next !== null) { + if ($next > ShardDefinition::MAX_PRIMARY_KEY) { + throw new \Exception('Max primary key of ' . ShardDefinition::MAX_PRIMARY_KEY . ' exceeded'); + } + // we encode the shard the primary key was originally inserted into to allow guessing the shard by primary key later on + return ($next << 8) | $shard; + } else { + $retries++; + } + } + throw new \Exception('Failed to get next primary key'); + } + + /** + * auto increment logic without retry + * + * @param ShardDefinition $shardDefinition + * @return int|null either the next primary key or null if the call needs to be retried + */ + private function getNextInner(ShardDefinition $shardDefinition): ?int { + $cache = $this->getCache(); + // because this function will likely be called concurrently from different requests + // the implementation needs to ensure that the cached value can be cleared, invalidated or re-calculated at any point between our cache calls + // care must be taken that the logic remains fully resilient against race conditions + + // in the ideal case, the last primary key is stored in the cache and we can just do an `inc` + // if that is not the case we find the highest used id in the database increment it, and save it in the cache + + // prevent inc from returning `1` if the key doesn't exist by setting it to a non-numeric value + $cache->add($shardDefinition->table, 'empty-placeholder', self::TTL); + $next = $cache->inc($shardDefinition->table); + + if ($cache instanceof IMemcacheTTL) { + $cache->setTTL($shardDefinition->table, self::TTL); + } + + // the "add + inc" trick above isn't strictly atomic, so as a safety we reject any result that to small + // to handle the edge case of the stored value disappearing between the add and inc + if (is_int($next) && $next >= self::MIN_VALID_KEY) { + return $next; + } elseif (is_int($next)) { + // we hit the edge case, so invalidate the cached value + if (!$cache->cas($shardDefinition->table, $next, 'empty-placeholder')) { + // someone else is changing the value concurrently, give up and retry + return null; + } + } + + // discard the encoded initial shard + $current = $this->getMaxFromDb($shardDefinition); + $next = max($current, self::MIN_VALID_KEY) + 1; + if ($cache->cas($shardDefinition->table, 'empty-placeholder', $next)) { + return $next; + } + + // another request set the cached value before us, so we should just be able to inc + $next = $cache->inc($shardDefinition->table); + if (is_int($next) && $next >= self::MIN_VALID_KEY) { + return $next; + } elseif (is_int($next)) { + // key got cleared, invalidate and retry + $cache->cas($shardDefinition->table, $next, 'empty-placeholder'); + return null; + } else { + // cleanup any non-numeric value other than the placeholder if that got stored somehow + $cache->ncad($shardDefinition->table, 'empty-placeholder'); + // retry + return null; + } + } + + /** + * Get the maximum primary key value from the shards, note that this has already stripped any embedded shard id + */ + private function getMaxFromDb(ShardDefinition $shardDefinition): int { + $max = $shardDefinition->fromFileId; + $query = $this->shardConnectionManager->getConnection($shardDefinition, 0)->getQueryBuilder(); + $query->select($shardDefinition->primaryKey) + ->from($shardDefinition->table) + ->orderBy($shardDefinition->primaryKey, 'DESC') + ->setMaxResults(1); + foreach ($shardDefinition->getAllShards() as $shard) { + $connection = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + $result = $query->executeQuery($connection)->fetchOne(); + if ($result) { + if ($result > $shardDefinition->fromFileId) { + $result = $result >> 8; + } + $max = max($max, $result); + } + } + return $max; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php b/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php new file mode 100644 index 00000000000..81530b56725 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/CrossShardMoveHelper.php @@ -0,0 +1,162 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Utility methods for implementing logic that moves data across shards + */ +class CrossShardMoveHelper { + public function __construct( + private ShardConnectionManager $connectionManager, + ) { + } + + public function getConnection(ShardDefinition $shardDefinition, int $shardKey): IDBConnection { + return $this->connectionManager->getConnection($shardDefinition, $shardDefinition->getShardForKey($shardKey)); + } + + /** + * Update the shard key of a set of rows, moving them to a different shard if needed + * + * @param ShardDefinition $shardDefinition + * @param string $table + * @param string $shardColumn + * @param int $sourceShardKey + * @param int $targetShardKey + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function moveCrossShards(ShardDefinition $shardDefinition, string $table, string $shardColumn, int $sourceShardKey, int $targetShardKey, string $primaryColumn, array $primaryKeys): void { + $sourceShard = $shardDefinition->getShardForKey($sourceShardKey); + $targetShard = $shardDefinition->getShardForKey($targetShardKey); + $sourceConnection = $this->connectionManager->getConnection($shardDefinition, $sourceShard); + if ($sourceShard === $targetShard) { + $this->updateItems($sourceConnection, $table, $shardColumn, $targetShardKey, $primaryColumn, $primaryKeys); + + return; + } + $targetConnection = $this->connectionManager->getConnection($shardDefinition, $targetShard); + + $sourceItems = $this->loadItems($sourceConnection, $table, $primaryColumn, $primaryKeys); + foreach ($sourceItems as &$sourceItem) { + $sourceItem[$shardColumn] = $targetShardKey; + } + if (!$sourceItems) { + return; + } + + $sourceConnection->beginTransaction(); + $targetConnection->beginTransaction(); + try { + $this->saveItems($targetConnection, $table, $sourceItems); + $this->deleteItems($sourceConnection, $table, $primaryColumn, $primaryKeys); + + $targetConnection->commit(); + $sourceConnection->commit(); + } catch (\Exception $e) { + $sourceConnection->rollback(); + $targetConnection->rollback(); + throw $e; + } + } + + /** + * Load rows from a table to move + * + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return array[] + */ + public function loadItems(IDBConnection $connection, string $table, string $primaryColumn, array $primaryKeys): array { + $query = $connection->getQueryBuilder(); + $query->select('*') + ->from($table) + ->where($query->expr()->in($primaryColumn, $query->createParameter('keys'))); + + $chunks = array_chunk($primaryKeys, 1000); + + $results = []; + foreach ($chunks as $chunk) { + $query->setParameter('keys', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $results = array_merge($results, $query->execute()->fetchAll()); + } + + return $results; + } + + /** + * Save modified rows + * + * @param IDBConnection $connection + * @param string $table + * @param array[] $items + * @return void + */ + public function saveItems(IDBConnection $connection, string $table, array $items): void { + if (count($items) === 0) { + return; + } + $query = $connection->getQueryBuilder(); + $query->insert($table); + foreach ($items[0] as $column => $value) { + $query->setValue($column, $query->createParameter($column)); + } + + foreach ($items as $item) { + foreach ($item as $column => $value) { + if (is_int($column)) { + $query->setParameter($column, $value, IQueryBuilder::PARAM_INT); + } else { + $query->setParameter($column, $value); + } + } + $query->executeStatement(); + } + } + + /** + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function updateItems(IDBConnection $connection, string $table, string $shardColumn, int $targetShardKey, string $primaryColumn, array $primaryKeys): void { + $query = $connection->getQueryBuilder(); + $query->update($table) + ->set($shardColumn, $query->createNamedParameter($targetShardKey, IQueryBuilder::PARAM_INT)) + ->where($query->expr()->in($primaryColumn, $query->createNamedParameter($primaryKeys, IQueryBuilder::PARAM_INT_ARRAY))); + $query->executeQuery()->fetchAll(); + } + + /** + * @param IDBConnection $connection + * @param string $table + * @param string $primaryColumn + * @param int[] $primaryKeys + * @return void + */ + public function deleteItems(IDBConnection $connection, string $table, string $primaryColumn, array $primaryKeys): void { + $query = $connection->getQueryBuilder(); + $query->delete($table) + ->where($query->expr()->in($primaryColumn, $query->createParameter('keys'))); + $chunks = array_chunk($primaryKeys, 1000); + + foreach ($chunks as $chunk) { + $query->setParameter('keys', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $query->executeStatement(); + } + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php b/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php new file mode 100644 index 00000000000..af778489a2d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/HashShardMapper.php @@ -0,0 +1,21 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Map string key to an int-range by hashing the key + */ +class HashShardMapper implements IShardMapper { + public function getShardForKey(int $key, int $count): int { + $int = unpack('L', substr(md5((string)$key, true), 0, 4))[1]; + return $int % $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php b/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php new file mode 100644 index 00000000000..733a6acaf9d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/InvalidShardedQueryException.php @@ -0,0 +1,29 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +namespace OC\DB\QueryBuilder\Sharded; + +/** + * Queries on sharded table has the following limitations: + * + * 1. Either the shard key (e.g. "storage") or primary key (e.g. "fileid") must be mentioned in the query. + * Or the query must be explicitly marked as running across all shards. + * + * For queries where it isn't possible to set one of these keys in the query normally, you can set it using `hintShardKey` + * + * 2. Insert statements must always explicitly set the shard key + * 3. A query on a sharded table is not allowed to join on the same table + * 4. Right joins are not allowed on sharded tables + * 5. Updating the shard key where the new shard key maps to a different shard is not allowed + * + * Moving rows to a different shard needs to be implemented manually. `CrossShardMoveHelper` provides + * some tools to help make this easier. + */ +class InvalidShardedQueryException extends \Exception { + +} diff --git a/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php b/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php new file mode 100644 index 00000000000..a5694b06507 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/RoundRobinShardMapper.php @@ -0,0 +1,20 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Map string key to an int-range by hashing the key + */ +class RoundRobinShardMapper implements IShardMapper { + public function getShardForKey(int $key, int $count): int { + return $key % $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php new file mode 100644 index 00000000000..74358e3ca96 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php @@ -0,0 +1,52 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\ConnectionAdapter; +use OC\DB\ConnectionFactory; +use OC\SystemConfig; +use OCP\IDBConnection; + +/** + * Keeps track of the db connections to the various shards + */ +class ShardConnectionManager { + /** @var array<string, IDBConnection> */ + private array $connections = []; + + public function __construct( + private SystemConfig $config, + private ConnectionFactory $factory, + ) { + } + + public function getConnection(ShardDefinition $shardDefinition, int $shard): IDBConnection { + $connectionKey = $shardDefinition->table . '_' . $shard; + + if (isset($this->connections[$connectionKey])) { + return $this->connections[$connectionKey]; + } + + if ($shard === ShardDefinition::MIGRATION_SHARD) { + $this->connections[$connectionKey] = \OC::$server->get(IDBConnection::class); + } elseif (isset($shardDefinition->shards[$shard])) { + $this->connections[$connectionKey] = $this->createConnection($shardDefinition->shards[$shard]); + } else { + throw new \InvalidArgumentException("invalid shard key $shard only " . count($shardDefinition->shards) . ' configured'); + } + + return $this->connections[$connectionKey]; + } + + private function createConnection(array $shardConfig): IDBConnection { + $shardConfig['sharding'] = []; + $type = $this->config->getValue('dbtype', 'sqlite'); + return new ConnectionAdapter($this->factory->getConnection($type, $shardConfig)); + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php new file mode 100644 index 00000000000..4f98079d92d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php @@ -0,0 +1,80 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OCP\DB\QueryBuilder\Sharded\IShardMapper; + +/** + * Configuration for a shard setup + */ +class ShardDefinition { + // we reserve the bottom byte of the primary key for the initial shard, so the total shard count is limited to what we can fit there + // additionally, shard id 255 is reserved for migration purposes + public const MAX_SHARDS = 255; + public const MIGRATION_SHARD = 255; + + public const PRIMARY_KEY_MASK = 0x7F_FF_FF_FF_FF_FF_FF_00; + public const PRIMARY_KEY_SHARD_MASK = 0x00_00_00_00_00_00_00_FF; + // since we reserve 1 byte for the shard index, we only have 56 bits of primary key space + public const MAX_PRIMARY_KEY = PHP_INT_MAX >> 8; + + /** + * @param string $table + * @param string $primaryKey + * @param string $shardKey + * @param string[] $companionKeys + * @param IShardMapper $shardMapper + * @param string[] $companionTables + * @param array $shards + */ + public function __construct( + public string $table, + public string $primaryKey, + public array $companionKeys, + public string $shardKey, + public IShardMapper $shardMapper, + public array $companionTables, + public array $shards, + public int $fromFileId, + public int $fromStorageId, + ) { + if (count($this->shards) >= self::MAX_SHARDS) { + throw new \Exception('Only allowed maximum of ' . self::MAX_SHARDS . ' shards allowed'); + } + } + + public function hasTable(string $table): bool { + if ($this->table === $table) { + return true; + } + return in_array($table, $this->companionTables); + } + + public function getShardForKey(int $key): int { + if ($key < $this->fromStorageId) { + return self::MIGRATION_SHARD; + } + return $this->shardMapper->getShardForKey($key, count($this->shards)); + } + + /** + * @return list<int> + */ + public function getAllShards(): array { + if ($this->fromStorageId !== 0) { + return array_merge(array_keys($this->shards), [self::MIGRATION_SHARD]); + } else { + return array_keys($this->shards); + } + } + + public function isKey(string $column): bool { + return $column === $this->primaryKey || in_array($column, $this->companionKeys); + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php new file mode 100644 index 00000000000..25e2a3d5f2d --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php @@ -0,0 +1,200 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\ArrayResult; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; + +/** + * Logic for running a query across a number of shards, combining the results + */ +class ShardQueryRunner { + public function __construct( + private ShardConnectionManager $shardConnectionManager, + private ShardDefinition $shardDefinition, + ) { + } + + /** + * Get the shards for a specific query or null if the shards aren't known in advance + * + * @param bool $allShards + * @param int[] $shardKeys + * @return null|int[] + */ + public function getShards(bool $allShards, array $shardKeys): ?array { + if ($allShards) { + return $this->shardDefinition->getAllShards(); + } + $allConfiguredShards = $this->shardDefinition->getAllShards(); + if (count($allConfiguredShards) === 1) { + return $allConfiguredShards; + } + if (empty($shardKeys)) { + return null; + } + $shards = array_map(function ($shardKey) { + return $this->shardDefinition->getShardForKey((int)$shardKey); + }, $shardKeys); + return array_values(array_unique($shards)); + } + + /** + * Try to get the shards that the keys are likely to be in, based on the shard the row was created + * + * @param int[] $primaryKeys + * @return int[] + */ + private function getLikelyShards(array $primaryKeys): array { + $shards = []; + foreach ($primaryKeys as $primaryKey) { + if ($primaryKey < $this->shardDefinition->fromFileId && !in_array(ShardDefinition::MIGRATION_SHARD, $shards)) { + $shards[] = ShardDefinition::MIGRATION_SHARD; + } + $encodedShard = $primaryKey & ShardDefinition::PRIMARY_KEY_SHARD_MASK; + if ($encodedShard < count($this->shardDefinition->shards) && !in_array($encodedShard, $shards)) { + $shards[] = $encodedShard; + } + } + return $shards; + } + + /** + * Execute a SELECT statement across the configured shards + * + * @param IQueryBuilder $query + * @param bool $allShards + * @param int[] $shardKeys + * @param int[] $primaryKeys + * @param array{column: string, order: string}[] $sortList + * @param int|null $limit + * @param int|null $offset + * @return IResult + */ + public function executeQuery( + IQueryBuilder $query, + bool $allShards, + array $shardKeys, + array $primaryKeys, + ?array $sortList = null, + ?int $limit = null, + ?int $offset = null, + ): IResult { + $shards = $this->getShards($allShards, $shardKeys); + $results = []; + if ($shards && count($shards) === 1) { + // trivial case + return $query->executeQuery($this->shardConnectionManager->getConnection($this->shardDefinition, $shards[0])); + } + // we have to emulate limit and offset, so we select offset+limit from all shards to ensure we have enough rows + // and then filter them down after we merged the results + if ($limit !== null && $offset !== null) { + $query->setMaxResults($limit + $offset); + } + + if ($shards) { + // we know exactly what shards we need to query + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $subResult = $query->executeQuery($shardConnection); + $results = array_merge($results, $subResult->fetchAll()); + $subResult->closeCursor(); + } + } else { + // we don't know for sure what shards we need to query, + // we first try the shards that are "likely" to have the rows we want, based on the shard that the row was + // originally created in. If we then still haven't found all rows we try the rest of the shards + $likelyShards = $this->getLikelyShards($primaryKeys); + $unlikelyShards = array_diff($this->shardDefinition->getAllShards(), $likelyShards); + $shards = array_merge($likelyShards, $unlikelyShards); + + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $subResult = $query->executeQuery($shardConnection); + $rows = $subResult->fetchAll(); + $results = array_merge($results, $rows); + $subResult->closeCursor(); + + if (count($rows) >= count($primaryKeys)) { + // we have all the rows we're looking for + break; + } + } + } + + if ($sortList) { + usort($results, function ($a, $b) use ($sortList) { + foreach ($sortList as $sort) { + $valueA = $a[$sort['column']] ?? null; + $valueB = $b[$sort['column']] ?? null; + $cmp = $valueA <=> $valueB; + if ($cmp === 0) { + continue; + } + if ($sort['order'] === 'DESC') { + $cmp = -$cmp; + } + return $cmp; + } + }); + } + + if ($limit !== null && $offset !== null) { + $results = array_slice($results, $offset, $limit); + } elseif ($limit !== null) { + $results = array_slice($results, 0, $limit); + } elseif ($offset !== null) { + $results = array_slice($results, $offset); + } + + return new ArrayResult($results); + } + + /** + * Execute an UPDATE or DELETE statement + * + * @param IQueryBuilder $query + * @param bool $allShards + * @param int[] $shardKeys + * @param int[] $primaryKeys + * @return int + * @throws \OCP\DB\Exception + */ + public function executeStatement(IQueryBuilder $query, bool $allShards, array $shardKeys, array $primaryKeys): int { + if ($query->getType() === \Doctrine\DBAL\Query\QueryBuilder::INSERT) { + throw new \Exception('insert queries need special handling'); + } + + $shards = $this->getShards($allShards, $shardKeys); + $maxCount = count($primaryKeys); + if ($shards && count($shards) === 1) { + return $query->executeStatement($this->shardConnectionManager->getConnection($this->shardDefinition, $shards[0])); + } elseif ($shards) { + $maxCount = PHP_INT_MAX; + } else { + // sort the likely shards before the rest, similar logic to `self::executeQuery` + $likelyShards = $this->getLikelyShards($primaryKeys); + $unlikelyShards = array_diff($this->shardDefinition->getAllShards(), $likelyShards); + $shards = array_merge($likelyShards, $unlikelyShards); + } + + $count = 0; + + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + $count += $query->executeStatement($shardConnection); + + if ($count >= $maxCount) { + break; + } + } + return $count; + } +} diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php new file mode 100644 index 00000000000..04082f76ae8 --- /dev/null +++ b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php @@ -0,0 +1,407 @@ +<?php + +declare(strict_types=1); +/** + * SPDX-FileCopyrightText: 2024 Robin Appelman <robin@icewind.nl> + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB\QueryBuilder\Sharded; + +use OC\DB\QueryBuilder\CompositeExpression; +use OC\DB\QueryBuilder\ExtendedQueryBuilder; +use OC\DB\QueryBuilder\Parameter; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * A special query builder that automatically distributes queries over multiple database shards. + * + * This relies on `PartitionedQueryBuilder` to handle splitting of parts of the query that touch the sharded tables + * from the non-sharded tables. So the query build here should only either touch only sharded table or only non-sharded tables. + * + * Most of the logic in this class is concerned with extracting either the shard key (e.g. "storage") or primary key (e.g. "fileid") + * from the query. The logic for actually running the query across the shards is mostly delegated to `ShardQueryRunner`. + */ +class ShardedQueryBuilder extends ExtendedQueryBuilder { + private array $shardKeys = []; + private array $primaryKeys = []; + private ?ShardDefinition $shardDefinition = null; + /** @var bool Run the query across all shards */ + private bool $allShards = false; + private ?string $insertTable = null; + private mixed $lastInsertId = null; + private ?IDBConnection $lastInsertConnection = null; + private ?int $updateShardKey = null; + private ?int $limit = null; + private ?int $offset = null; + /** @var array{column: string, order: string}[] */ + private array $sortList = []; + private string $mainTable = ''; + + public function __construct( + IQueryBuilder $builder, + protected array $shardDefinitions, + protected ShardConnectionManager $shardConnectionManager, + protected AutoIncrementHandler $autoIncrementHandler, + ) { + parent::__construct($builder); + } + + public function getShardKeys(): array { + return $this->getKeyValues($this->shardKeys); + } + + public function getPrimaryKeys(): array { + return $this->getKeyValues($this->primaryKeys); + } + + private function getKeyValues(array $keys): array { + $values = []; + foreach ($keys as $key) { + $values = array_merge($values, $this->getKeyValue($key)); + } + return array_values(array_unique($values)); + } + + private function getKeyValue($value): array { + if ($value instanceof Parameter) { + $value = (string)$value; + } + if (is_string($value) && str_starts_with($value, ':')) { + $param = $this->getParameter(substr($value, 1)); + if (is_array($param)) { + return $param; + } else { + return [$param]; + } + } elseif ($value !== null) { + return [$value]; + } else { + return []; + } + } + + public function where(...$predicates) { + return $this->andWhere(...$predicates); + } + + public function andWhere(...$where) { + if ($where) { + foreach ($where as $predicate) { + $this->tryLoadShardKey($predicate); + } + parent::andWhere(...$where); + } + return $this; + } + + private function tryLoadShardKey($predicate): void { + if (!$this->shardDefinition) { + return; + } + if ($keys = $this->tryExtractShardKeys($predicate, $this->shardDefinition->shardKey)) { + $this->shardKeys += $keys; + } + if ($keys = $this->tryExtractShardKeys($predicate, $this->shardDefinition->primaryKey)) { + $this->primaryKeys += $keys; + } + foreach ($this->shardDefinition->companionKeys as $companionKey) { + if ($keys = $this->tryExtractShardKeys($predicate, $companionKey)) { + $this->primaryKeys += $keys; + } + } + } + + /** + * @param $predicate + * @param string $column + * @return string[] + */ + private function tryExtractShardKeys($predicate, string $column): array { + if ($predicate instanceof CompositeExpression) { + $values = []; + foreach ($predicate->getParts() as $part) { + $partValues = $this->tryExtractShardKeys($part, $column); + // for OR expressions, we can only rely on the predicate if all parts contain the comparison + if ($predicate->getType() === CompositeExpression::TYPE_OR && !$partValues) { + return []; + } + $values = array_merge($values, $partValues); + } + return $values; + } + $predicate = (string)$predicate; + // expect a condition in the form of 'alias1.column1 = placeholder' or 'alias1.column1 in placeholder' + if (substr_count($predicate, ' ') > 2) { + return []; + } + if (str_contains($predicate, ' = ')) { + $parts = explode(' = ', $predicate); + if ($parts[0] === "`{$column}`" || str_ends_with($parts[0], "`.`{$column}`")) { + return [$parts[1]]; + } else { + return []; + } + } + + if (str_contains($predicate, ' IN ')) { + $parts = explode(' IN ', $predicate); + if ($parts[0] === "`{$column}`" || str_ends_with($parts[0], "`.`{$column}`")) { + return [trim(trim($parts[1], '('), ')')]; + } else { + return []; + } + } + + return []; + } + + public function set($key, $value) { + if ($this->shardDefinition && $key === $this->shardDefinition->shardKey) { + $updateShardKey = $value; + } + return parent::set($key, $value); + } + + public function setValue($column, $value) { + if ($this->shardDefinition) { + if ($this->shardDefinition->isKey($column)) { + $this->primaryKeys[] = $value; + } + if ($column === $this->shardDefinition->shardKey) { + $this->shardKeys[] = $value; + } + } + return parent::setValue($column, $value); + } + + public function values(array $values) { + foreach ($values as $column => $value) { + $this->setValue($column, $value); + } + return $this; + } + + private function actOnTable(string $table): void { + $this->mainTable = $table; + foreach ($this->shardDefinitions as $shardDefinition) { + if ($shardDefinition->hasTable($table)) { + $this->shardDefinition = $shardDefinition; + } + } + } + + public function from($from, $alias = null) { + if (is_string($from) && $from) { + $this->actOnTable($from); + } + return parent::from($from, $alias); + } + + public function update($update = null, $alias = null) { + if (is_string($update) && $update) { + $this->actOnTable($update); + } + return parent::update($update, $alias); + } + + public function insert($insert = null) { + if (is_string($insert) && $insert) { + $this->insertTable = $insert; + $this->actOnTable($insert); + } + return parent::insert($insert); + } + + public function delete($delete = null, $alias = null) { + if (is_string($delete) && $delete) { + $this->actOnTable($delete); + } + return parent::delete($delete, $alias); + } + + private function checkJoin(string $table): void { + if ($this->shardDefinition) { + if ($table === $this->mainTable) { + throw new InvalidShardedQueryException("Sharded query on {$this->mainTable} isn't allowed to join on itself"); + } + if (!$this->shardDefinition->hasTable($table)) { + // this generally shouldn't happen as the partitioning logic should prevent this + // but the check is here just in case + throw new InvalidShardedQueryException("Sharded query on {$this->shardDefinition->table} isn't allowed to join on $table"); + } + } + } + + public function innerJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkJoin($join); + return parent::innerJoin($fromAlias, $join, $alias, $condition); + } + + public function leftJoin($fromAlias, $join, $alias, $condition = null) { + $this->checkJoin($join); + return parent::leftJoin($fromAlias, $join, $alias, $condition); + } + + public function rightJoin($fromAlias, $join, $alias, $condition = null) { + if ($this->shardDefinition) { + throw new InvalidShardedQueryException("Sharded query on {$this->shardDefinition->table} isn't allowed to right join"); + } + return parent::rightJoin($fromAlias, $join, $alias, $condition); + } + + public function join($fromAlias, $join, $alias, $condition = null) { + return $this->innerJoin($fromAlias, $join, $alias, $condition); + } + + public function setMaxResults($maxResults) { + if ($maxResults > 0) { + $this->limit = (int)$maxResults; + } + return parent::setMaxResults($maxResults); + } + + public function setFirstResult($firstResult) { + if ($firstResult > 0) { + $this->offset = (int)$firstResult; + } + if ($this->shardDefinition && count($this->shardDefinition->shards) > 1) { + // we have to emulate offset + return $this; + } else { + return parent::setFirstResult($firstResult); + } + } + + public function addOrderBy($sort, $order = null) { + $this->registerOrder((string)$sort, (string)$order ?? 'ASC'); + return parent::addOrderBy($sort, $order); + } + + public function orderBy($sort, $order = null) { + $this->sortList = []; + $this->registerOrder((string)$sort, (string)$order ?? 'ASC'); + return parent::orderBy($sort, $order); + } + + private function registerOrder(string $column, string $order): void { + // handle `mime + 0` and similar by just sorting on the first part of the expression + [$column] = explode(' ', $column); + $column = trim($column, '`'); + $this->sortList[] = [ + 'column' => $column, + 'order' => strtoupper($order), + ]; + } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if ($overwrite) { + $this->primaryKeys = []; + $this->shardKeys = []; + } + if ($this->shardDefinition?->isKey($column)) { + $this->primaryKeys[] = $value; + } + if ($column === $this->shardDefinition?->shardKey) { + $this->shardKeys[] = $value; + } + return $this; + } + + public function runAcrossAllShards(): self { + $this->allShards = true; + return $this; + } + + /** + * @throws InvalidShardedQueryException + */ + public function validate(): void { + if ($this->shardDefinition && $this->insertTable) { + if ($this->allShards) { + throw new InvalidShardedQueryException("Can't insert across all shards"); + } + if (empty($this->getShardKeys())) { + throw new InvalidShardedQueryException("Can't insert without shard key"); + } + } + if ($this->shardDefinition && !$this->allShards) { + if (empty($this->getShardKeys()) && empty($this->getPrimaryKeys())) { + throw new InvalidShardedQueryException('No shard key or primary key set for query'); + } + } + if ($this->shardDefinition && $this->updateShardKey) { + $newShardKey = $this->getKeyValue($this->updateShardKey); + $oldShardKeys = $this->getShardKeys(); + if (count($newShardKey) !== 1) { + throw new InvalidShardedQueryException("Can't set shard key to an array"); + } + $newShardKey = current($newShardKey); + if (empty($oldShardKeys)) { + throw new InvalidShardedQueryException("Can't update without shard key"); + } + $oldShards = array_values(array_unique(array_map(function ($shardKey) { + return $this->shardDefinition->getShardForKey((int)$shardKey); + }, $oldShardKeys))); + $newShard = $this->shardDefinition->getShardForKey((int)$newShardKey); + if ($oldShards === [$newShard]) { + throw new InvalidShardedQueryException('Update statement would move rows to a different shard'); + } + } + } + + public function executeQuery(?IDBConnection $connection = null): IResult { + $this->validate(); + if ($this->shardDefinition) { + $runner = new ShardQueryRunner($this->shardConnectionManager, $this->shardDefinition); + return $runner->executeQuery($this->builder, $this->allShards, $this->getShardKeys(), $this->getPrimaryKeys(), $this->sortList, $this->limit, $this->offset); + } + return parent::executeQuery($connection); + } + + public function executeStatement(?IDBConnection $connection = null): int { + $this->validate(); + if ($this->shardDefinition) { + $runner = new ShardQueryRunner($this->shardConnectionManager, $this->shardDefinition); + if ($this->insertTable) { + $shards = $runner->getShards($this->allShards, $this->getShardKeys()); + if (!$shards) { + throw new InvalidShardedQueryException("Can't insert without shard key"); + } + $count = 0; + foreach ($shards as $shard) { + $shardConnection = $this->shardConnectionManager->getConnection($this->shardDefinition, $shard); + if (!$this->primaryKeys && $this->shardDefinition->table === $this->insertTable) { + $id = $this->autoIncrementHandler->getNextPrimaryKey($this->shardDefinition, $shard); + parent::setValue($this->shardDefinition->primaryKey, $this->createParameter('__generated_primary_key')); + $this->setParameter('__generated_primary_key', $id, self::PARAM_INT); + $this->lastInsertId = $id; + } + $count += parent::executeStatement($shardConnection); + + $this->lastInsertConnection = $shardConnection; + } + return $count; + } else { + return $runner->executeStatement($this->builder, $this->allShards, $this->getShardKeys(), $this->getPrimaryKeys()); + } + } + return parent::executeStatement($connection); + } + + public function getLastInsertId(): int { + if ($this->lastInsertId) { + return $this->lastInsertId; + } + if ($this->lastInsertConnection) { + $table = $this->builder->prefixTableName($this->insertTable); + return $this->lastInsertConnection->lastInsertId($table); + } else { + return parent::getLastInsertId(); + } + } + + +} diff --git a/lib/private/DB/ResultAdapter.php b/lib/private/DB/ResultAdapter.php index 8b004d471ec..95a7620e0ff 100644 --- a/lib/private/DB/ResultAdapter.php +++ b/lib/private/DB/ResultAdapter.php @@ -30,14 +30,21 @@ class ResultAdapter implements IResult { } public function fetch(int $fetchMode = PDO::FETCH_ASSOC) { - return $this->inner->fetch($fetchMode); + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->inner->fetchAssociative(), + PDO::FETCH_NUM => $this->inner->fetchNumeric(), + PDO::FETCH_COLUMN => $this->inner->fetchOne(), + default => throw new \Exception('Fetch mode needs to be assoc, num or column.'), + }; } public function fetchAll(int $fetchMode = PDO::FETCH_ASSOC): array { - if ($fetchMode !== PDO::FETCH_ASSOC && $fetchMode !== PDO::FETCH_NUM && $fetchMode !== PDO::FETCH_COLUMN) { - throw new \Exception('Fetch mode needs to be assoc, num or column.'); - } - return $this->inner->fetchAll($fetchMode); + return match ($fetchMode) { + PDO::FETCH_ASSOC => $this->inner->fetchAllAssociative(), + PDO::FETCH_NUM => $this->inner->fetchAllNumeric(), + PDO::FETCH_COLUMN => $this->inner->fetchFirstColumn(), + default => throw new \Exception('Fetch mode needs to be assoc, num or column.'), + }; } public function fetchColumn($columnIndex = 0) { diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index 5720e10fbdb..0d5b2040513 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -36,6 +37,9 @@ class SchemaWrapper implements ISchemaWrapper { public function performDropTableCalls() { foreach ($this->tablesToDelete as $tableName => $true) { $this->connection->dropTable($tableName); + foreach ($this->connection->getShardConnections() as $shardConnection) { + $shardConnection->dropTable($tableName); + } unset($this->tablesToDelete[$tableName]); } } |