diff options
Diffstat (limited to 'lib/private/DB')
18 files changed, 170 insertions, 72 deletions
diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 1b61cc83319..f86cbc341a4 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -16,6 +16,7 @@ use Doctrine\DBAL\Driver; use Doctrine\DBAL\Driver\ServerInfoAwareConnection; use Doctrine\DBAL\Exception; use Doctrine\DBAL\Exception\ConnectionLost; +use Doctrine\DBAL\Platforms\MariaDBPlatform; use Doctrine\DBAL\Platforms\MySQLPlatform; use Doctrine\DBAL\Platforms\OraclePlatform; use Doctrine\DBAL\Platforms\PostgreSQLPlatform; @@ -160,7 +161,7 @@ class Connection extends PrimaryReadReplicaConnection { $this->_config->setSQLLogger($debugStack); } - /** @var array<string, array{shards: array[], mapper: ?string}> $shardConfig */ + /** @var array<string, array{shards: array[], mapper: ?string, from_primary_key: ?int, from_shard_key: ?int}> $shardConfig */ $shardConfig = $this->params['sharding'] ?? []; $shardNames = array_keys($shardConfig); $this->shards = array_map(function (array $config, string $name) { @@ -180,7 +181,9 @@ class Connection extends PrimaryReadReplicaConnection { self::SHARD_PRESETS[$name]['shard_key'], $shardMapper, self::SHARD_PRESETS[$name]['companion_tables'], - $config['shards'] + $config['shards'], + $config['from_primary_key'] ?? 0, + $config['from_shard_key'] ?? 0, ); }, $shardConfig, $shardNames); $this->shards = array_combine($shardNames, $this->shards); @@ -199,8 +202,10 @@ class Connection extends PrimaryReadReplicaConnection { if ($this->isShardingEnabled) { foreach ($this->shards as $shardDefinition) { foreach ($shardDefinition->getAllShards() as $shard) { - /** @var ConnectionAdapter $connection */ - $connections[] = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + if ($shard !== ShardDefinition::MIGRATION_SHARD) { + /** @var ConnectionAdapter $connection */ + $connections[] = $this->shardConnectionManager->getConnection($shardDefinition, $shard); + } } } } @@ -218,8 +223,6 @@ class Connection extends PrimaryReadReplicaConnection { return parent::connect(); } - $this->lastConnectionCheck[$this->getConnectionName()] = time(); - // Only trigger the event logger for the initial connect call $eventLogger = Server::get(IEventLogger::class); $eventLogger->start('connect:db', 'db connection opened'); @@ -227,6 +230,8 @@ class Connection extends PrimaryReadReplicaConnection { $status = parent::connect(); $eventLogger->end('connect:db'); + $this->lastConnectionCheck[$this->getConnectionName()] = time(); + return $status; } catch (Exception $e) { // throw a new exception to prevent leaking info from the stacktrace @@ -410,7 +415,7 @@ class Connection extends PrimaryReadReplicaConnection { $sql = $this->finishQuery($sql); $this->queriesExecuted++; - $this->logQueryToFile($sql); + $this->logQueryToFile($sql, $params); try { return parent::executeQuery($sql, $params, $types, $qcp); } catch (\Exception $e) { @@ -457,7 +462,7 @@ class Connection extends PrimaryReadReplicaConnection { } $sql = $this->finishQuery($sql); $this->queriesExecuted++; - $this->logQueryToFile($sql); + $this->logQueryToFile($sql, $params); try { return (int)parent::executeStatement($sql, $params, $types); } catch (\Exception $e) { @@ -466,14 +471,19 @@ class Connection extends PrimaryReadReplicaConnection { } } - protected function logQueryToFile(string $sql): void { + protected function logQueryToFile(string $sql, array $params): void { $logFile = $this->systemConfig->getValue('query_log_file'); if ($logFile !== '' && is_writable(dirname($logFile)) && (!file_exists($logFile) || is_writable($logFile))) { $prefix = ''; if ($this->systemConfig->getValue('query_log_file_requestid') === 'yes') { $prefix .= Server::get(IRequestId::class)->getId() . "\t"; } + $postfix = ''; + if ($this->systemConfig->getValue('query_log_file_parameters') === 'yes') { + $postfix .= '; ' . json_encode($params); + } + if ($this->systemConfig->getValue('query_log_file_backtrace') === 'yes') { $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); array_pop($trace); @@ -695,6 +705,19 @@ class Connection extends PrimaryReadReplicaConnection { } /** + * Truncate a table data if it exists + * + * @param string $table table name without the prefix + * @param bool $cascade whether to truncate cascading + * + * @throws Exception + */ + public function truncateTable(string $table, bool $cascade) { + $this->executeStatement($this->getDatabasePlatform() + ->getTruncateTableSQL($this->tablePrefix . trim($table), $cascade)); + } + + /** * Check if a table exists * * @param string $table table name without the prefix @@ -872,9 +895,9 @@ class Connection extends PrimaryReadReplicaConnection { private function reconnectIfNeeded(): void { if ( - !isset($this->lastConnectionCheck[$this->getConnectionName()]) || - time() <= $this->lastConnectionCheck[$this->getConnectionName()] + 30 || - $this->isTransactionActive() + !isset($this->lastConnectionCheck[$this->getConnectionName()]) + || time() <= $this->lastConnectionCheck[$this->getConnectionName()] + 30 + || $this->isTransactionActive() ) { return; } @@ -893,11 +916,13 @@ class Connection extends PrimaryReadReplicaConnection { } /** - * @return IDBConnection::PLATFORM_MYSQL|IDBConnection::PLATFORM_ORACLE|IDBConnection::PLATFORM_POSTGRES|IDBConnection::PLATFORM_SQLITE + * @return IDBConnection::PLATFORM_MYSQL|IDBConnection::PLATFORM_ORACLE|IDBConnection::PLATFORM_POSTGRES|IDBConnection::PLATFORM_SQLITE|IDBConnection::PLATFORM_MARIADB */ - public function getDatabaseProvider(): string { + public function getDatabaseProvider(bool $strict = false): string { $platform = $this->getDatabasePlatform(); - if ($platform instanceof MySQLPlatform) { + if ($strict && $platform instanceof MariaDBPlatform) { + return IDBConnection::PLATFORM_MARIADB; + } elseif ($platform instanceof MySQLPlatform) { return IDBConnection::PLATFORM_MYSQL; } elseif ($platform instanceof OraclePlatform) { return IDBConnection::PLATFORM_ORACLE; diff --git a/lib/private/DB/ConnectionAdapter.php b/lib/private/DB/ConnectionAdapter.php index 2baeda9cfb7..d9ccb3c54f2 100644 --- a/lib/private/DB/ConnectionAdapter.php +++ b/lib/private/DB/ConnectionAdapter.php @@ -50,7 +50,7 @@ class ConnectionAdapter implements IDBConnection { $this->inner->executeQuery($sql, $params, $types) ); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -58,7 +58,7 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeUpdate($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -66,7 +66,7 @@ class ConnectionAdapter implements IDBConnection { try { return $this->inner->executeStatement($sql, $params, $types); } catch (Exception $e) { - throw DbalException::wrap($e); + throw DbalException::wrap($e, '', $sql); } } @@ -189,6 +189,14 @@ class ConnectionAdapter implements IDBConnection { } } + public function truncateTable(string $table, bool $cascade): void { + try { + $this->inner->truncateTable($table, $cascade); + } catch (Exception $e) { + throw DbalException::wrap($e); + } + } + public function tableExists(string $table): bool { try { return $this->inner->tableExists($table); @@ -229,10 +237,10 @@ class ConnectionAdapter implements IDBConnection { } /** - * @return self::PLATFORM_MYSQL|self::PLATFORM_ORACLE|self::PLATFORM_POSTGRES|self::PLATFORM_SQLITE + * @return self::PLATFORM_MYSQL|self::PLATFORM_ORACLE|self::PLATFORM_POSTGRES|self::PLATFORM_SQLITE|self::PLATFORM_MARIADB */ - public function getDatabaseProvider(): string { - return $this->inner->getDatabaseProvider(); + public function getDatabaseProvider(bool $strict = false): string { + return $this->inner->getDatabaseProvider($strict); } /** diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index 8d662b0508c..d9b80b81992 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -108,7 +108,7 @@ class ConnectionFactory { $normalizedType = $this->normalizeType($type); $eventManager = new EventManager(); $eventManager->addEventSubscriber(new SetTransactionIsolationLevel()); - $connectionParams = $this->createConnectionParams('', $additionalConnectionParams); + $connectionParams = $this->createConnectionParams('', $additionalConnectionParams, $type); switch ($normalizedType) { case 'pgsql': // pg_connect used by Doctrine DBAL does not support URI notation (enclosed in brackets) @@ -121,21 +121,9 @@ class ConnectionFactory { case 'oci': $eventManager->addEventSubscriber(new OracleSessionInit); - // the driverOptions are unused in dbal and need to be mapped to the parameters - if (isset($connectionParams['driverOptions'])) { - $connectionParams = array_merge($connectionParams, $connectionParams['driverOptions']); - } - $host = $connectionParams['host']; - $port = $connectionParams['port'] ?? null; - $dbName = $connectionParams['dbname']; - - // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string - if ($host === '') { - $connectionParams['dbname'] = $dbName; // use dbname as easy connect name - } else { - $connectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : '') . '/' . $dbName; - } - unset($connectionParams['host']); + $connectionParams = $this->forceConnectionStringOracle($connectionParams); + $connectionParams['primary'] = $this->forceConnectionStringOracle($connectionParams['primary']); + $connectionParams['replica'] = array_map([$this, 'forceConnectionStringOracle'], $connectionParams['replica']); break; case 'sqlite3': @@ -175,12 +163,10 @@ class ConnectionFactory { /** * Create the connection parameters for the config - * - * @param string $configPrefix - * @return array */ - public function createConnectionParams(string $configPrefix = '', array $additionalConnectionParams = []) { - $type = $this->config->getValue('dbtype', 'sqlite'); + public function createConnectionParams(string $configPrefix = '', array $additionalConnectionParams = [], ?string $type = null) { + // use provided type or if null use type from config + $type = $type ?? $this->config->getValue('dbtype', 'sqlite'); $connectionParams = array_merge($this->getDefaultConnectionParams($type), [ 'user' => $this->config->getValue($configPrefix . 'dbuser', $this->config->getValue('dbuser', '')), @@ -212,7 +198,7 @@ class ConnectionFactory { 'tablePrefix' => $connectionParams['tablePrefix'] ]; - if ($this->config->getValue('mysql.utf8mb4', false)) { + if ($type === 'mysql' && $this->config->getValue('mysql.utf8mb4', false)) { $connectionParams['defaultTableOptions'] = [ 'collate' => 'utf8mb4_bin', 'charset' => 'utf8mb4', @@ -267,4 +253,24 @@ class ConnectionFactory { return $params; } + + protected function forceConnectionStringOracle(array $connectionParams): array { + // the driverOptions are unused in dbal and need to be mapped to the parameters + if (isset($connectionParams['driverOptions'])) { + $connectionParams = array_merge($connectionParams, $connectionParams['driverOptions']); + } + $host = $connectionParams['host']; + $port = $connectionParams['port'] ?? null; + $dbName = $connectionParams['dbname']; + + // we set the connect string as dbname and unset the host to coerce doctrine into using it as connect string + if ($host === '') { + $connectionParams['dbname'] = $dbName; // use dbname as easy connect name + } else { + $connectionParams['dbname'] = '//' . $host . (!empty($port) ? ":{$port}" : '') . '/' . $dbName; + } + unset($connectionParams['host']); + + return $connectionParams; + } } diff --git a/lib/private/DB/Exceptions/DbalException.php b/lib/private/DB/Exceptions/DbalException.php index 05ea9e22a5d..2ce6ddf80a6 100644 --- a/lib/private/DB/Exceptions/DbalException.php +++ b/lib/private/DB/Exceptions/DbalException.php @@ -35,26 +35,29 @@ use OCP\DB\Exception; class DbalException extends Exception { /** @var \Doctrine\DBAL\Exception */ private $original; + public readonly ?string $query; /** * @param \Doctrine\DBAL\Exception $original * @param int $code * @param string $message */ - private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message) { + private function __construct(\Doctrine\DBAL\Exception $original, int $code, string $message, ?string $query = null) { parent::__construct( $message, $code, $original ); $this->original = $original; + $this->query = $query; } - public static function wrap(\Doctrine\DBAL\Exception $original, string $message = ''): self { + public static function wrap(\Doctrine\DBAL\Exception $original, string $message = '', ?string $query = null): self { return new self( $original, is_int($original->getCode()) ? $original->getCode() : 0, - empty($message) ? $original->getMessage() : $message + empty($message) ? $original->getMessage() : $message, + $query, ); } diff --git a/lib/private/DB/MigrationService.php b/lib/private/DB/MigrationService.php index 0b59509eaab..40579c7a898 100644 --- a/lib/private/DB/MigrationService.php +++ b/lib/private/DB/MigrationService.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-FileCopyrightText: 2017 ownCloud GmbH @@ -199,9 +200,9 @@ class MigrationService { if ($versionA !== $versionB) { return ($versionA < $versionB) ? -1 : 1; } - return ($matchA[2] < $matchB[2]) ? -1 : 1; + return strnatcmp($matchA[2], $matchB[2]); } - return (basename($a) < basename($b)) ? -1 : 1; + return strnatcmp(basename($a), basename($b)); } /** @@ -250,7 +251,7 @@ class MigrationService { $toBeExecuted = []; foreach ($availableMigrations as $v) { - if ($to !== 'latest' && $v > $to) { + if ($to !== 'latest' && ($this->sortMigrations($v, $to) > 0)) { continue; } if ($this->shallBeExecuted($v, $knownMigrations)) { diff --git a/lib/private/DB/MySqlTools.php b/lib/private/DB/MySqlTools.php index cd6b812be61..3413be43417 100644 --- a/lib/private/DB/MySqlTools.php +++ b/lib/private/DB/MySqlTools.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 ownCloud GmbH * SPDX-License-Identifier: AGPL-3.0-only @@ -45,7 +46,7 @@ class MySqlTools { return false; } - return str_contains($row, 'maria') && version_compare($row, '10.3', '>=') || - !str_contains($row, 'maria') && version_compare($row, '8.0', '>='); + return str_contains($row, 'maria') && version_compare($row, '10.3', '>=') + || !str_contains($row, 'maria') && version_compare($row, '8.0', '>='); } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php index 559c29df208..52f82db2232 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php index 2466493c1fa..48dc1da6330 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php index 6791430b1b0..47a8eaa6fd0 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -80,12 +81,12 @@ class OCIFunctionBuilder extends FunctionBuilder { public function octetLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTHB(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias); } public function charLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias); } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php index ee430a6bd71..354a2b126d7 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php index 956b2123f2c..53aa530054b 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php index 2942eeccdf7..d748c791321 100644 --- a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php @@ -126,8 +126,8 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { $selectPartition = null; } if ( - ($select === $checkColumn || $select === '*') && - $selectPartition === $partition + ($select === $checkColumn || $select === '*') + && $selectPartition === $partition ) { return; } @@ -151,8 +151,8 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { foreach ($this->selects as $select) { foreach ($this->partitions as $partition) { if (is_string($select['select']) && ( - $select['select'] === '*' || - $partition->isColumnInPartition($select['select'])) + $select['select'] === '*' + || $partition->isColumnInPartition($select['select'])) ) { if (isset($this->splitQueries[$partition->name])) { if ($select['alias']) { @@ -444,4 +444,19 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { public function getPartitionCount(): int { return count($this->splitQueries) + 1; } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if (str_contains($column, '.')) { + [$alias, $column] = explode('.', $column); + $partition = $this->getPartition($alias); + if ($partition) { + $this->splitQueries[$partition->name]->query->hintShardKey($column, $value, $overwrite); + } else { + parent::hintShardKey($column, $value, $overwrite); + } + } else { + parent::hintShardKey($column, $value, $overwrite); + } + return $this; + } } diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 56c860bc42c..1d44c049793 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -96,6 +96,7 @@ class QueryBuilder implements IQueryBuilder { return match($this->connection->getDatabaseProvider()) { IDBConnection::PLATFORM_ORACLE => new OCIExpressionBuilder($this->connection, $this, $this->logger), IDBConnection::PLATFORM_POSTGRES => new PgSqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_MARIADB, IDBConnection::PLATFORM_MYSQL => new MySqlExpressionBuilder($this->connection, $this, $this->logger), IDBConnection::PLATFORM_SQLITE => new SqliteExpressionBuilder($this->connection, $this, $this->logger), }; @@ -121,6 +122,7 @@ class QueryBuilder implements IQueryBuilder { return match($this->connection->getDatabaseProvider()) { IDBConnection::PLATFORM_ORACLE => new OCIFunctionBuilder($this->connection, $this, $this->helper), IDBConnection::PLATFORM_POSTGRES => new PgSqlFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_MARIADB, IDBConnection::PLATFORM_MYSQL => new FunctionBuilder($this->connection, $this, $this->helper), IDBConnection::PLATFORM_SQLITE => new SqliteFunctionBuilder($this->connection, $this, $this->helper), }; @@ -161,7 +163,7 @@ class QueryBuilder implements IQueryBuilder { try { $params = []; foreach ($this->getParameters() as $placeholder => $value) { - if ($value instanceof \DateTime) { + if ($value instanceof \DateTimeInterface) { $params[] = $placeholder . ' => DateTime:\'' . $value->format('c') . '\''; } elseif (is_array($value)) { $params[] = $placeholder . ' => (\'' . implode('\', \'', $value) . '\')'; @@ -1339,6 +1341,8 @@ class QueryBuilder implements IQueryBuilder { /** * Returns the table name with database prefix as needed by the implementation * + * Was protected until version 30. + * * @param string $table * @return string */ diff --git a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php index caedaa71c97..3a230ea544d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php +++ b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php @@ -108,7 +108,7 @@ class AutoIncrementHandler { } // discard the encoded initial shard - $current = $this->getMaxFromDb($shardDefinition) >> 8; + $current = $this->getMaxFromDb($shardDefinition); $next = max($current, self::MIN_VALID_KEY) + 1; if ($cache->cas($shardDefinition->table, 'empty-placeholder', $next)) { return $next; @@ -131,19 +131,22 @@ class AutoIncrementHandler { } /** - * Get the maximum primary key value from the shards + * Get the maximum primary key value from the shards, note that this has already stripped any embedded shard id */ private function getMaxFromDb(ShardDefinition $shardDefinition): int { - $max = 0; + $max = $shardDefinition->fromFileId; + $query = $this->shardConnectionManager->getConnection($shardDefinition, 0)->getQueryBuilder(); + $query->select($shardDefinition->primaryKey) + ->from($shardDefinition->table) + ->orderBy($shardDefinition->primaryKey, 'DESC') + ->setMaxResults(1); foreach ($shardDefinition->getAllShards() as $shard) { $connection = $this->shardConnectionManager->getConnection($shardDefinition, $shard); - $query = $connection->getQueryBuilder(); - $query->select($shardDefinition->primaryKey) - ->from($shardDefinition->table) - ->orderBy($shardDefinition->primaryKey, 'DESC') - ->setMaxResults(1); - $result = $query->executeQuery()->fetchOne(); + $result = $query->executeQuery($connection)->fetchOne(); if ($result) { + if ($result > $shardDefinition->fromFileId) { + $result = $result >> 8; + } $max = max($max, $result); } } diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php index 87cac58bc57..74358e3ca96 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php @@ -28,8 +28,17 @@ class ShardConnectionManager { public function getConnection(ShardDefinition $shardDefinition, int $shard): IDBConnection { $connectionKey = $shardDefinition->table . '_' . $shard; - if (!isset($this->connections[$connectionKey])) { + + if (isset($this->connections[$connectionKey])) { + return $this->connections[$connectionKey]; + } + + if ($shard === ShardDefinition::MIGRATION_SHARD) { + $this->connections[$connectionKey] = \OC::$server->get(IDBConnection::class); + } elseif (isset($shardDefinition->shards[$shard])) { $this->connections[$connectionKey] = $this->createConnection($shardDefinition->shards[$shard]); + } else { + throw new \InvalidArgumentException("invalid shard key $shard only " . count($shardDefinition->shards) . ' configured'); } return $this->connections[$connectionKey]; diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php index ebccbb639a6..4f98079d92d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php @@ -15,7 +15,9 @@ use OCP\DB\QueryBuilder\Sharded\IShardMapper; */ class ShardDefinition { // we reserve the bottom byte of the primary key for the initial shard, so the total shard count is limited to what we can fit there - public const MAX_SHARDS = 256; + // additionally, shard id 255 is reserved for migration purposes + public const MAX_SHARDS = 255; + public const MIGRATION_SHARD = 255; public const PRIMARY_KEY_MASK = 0x7F_FF_FF_FF_FF_FF_FF_00; public const PRIMARY_KEY_SHARD_MASK = 0x00_00_00_00_00_00_00_FF; @@ -37,8 +39,10 @@ class ShardDefinition { public array $companionKeys, public string $shardKey, public IShardMapper $shardMapper, - public array $companionTables = [], - public array $shards = [], + public array $companionTables, + public array $shards, + public int $fromFileId, + public int $fromStorageId, ) { if (count($this->shards) >= self::MAX_SHARDS) { throw new \Exception('Only allowed maximum of ' . self::MAX_SHARDS . ' shards allowed'); @@ -53,11 +57,21 @@ class ShardDefinition { } public function getShardForKey(int $key): int { + if ($key < $this->fromStorageId) { + return self::MIGRATION_SHARD; + } return $this->shardMapper->getShardForKey($key, count($this->shards)); } + /** + * @return list<int> + */ public function getAllShards(): array { - return array_keys($this->shards); + if ($this->fromStorageId !== 0) { + return array_merge(array_keys($this->shards), [self::MIGRATION_SHARD]); + } else { + return array_keys($this->shards); + } } public function isKey(string $column): bool { diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php index c020e72868e..25e2a3d5f2d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php @@ -55,6 +55,9 @@ class ShardQueryRunner { private function getLikelyShards(array $primaryKeys): array { $shards = []; foreach ($primaryKeys as $primaryKey) { + if ($primaryKey < $this->shardDefinition->fromFileId && !in_array(ShardDefinition::MIGRATION_SHARD, $shards)) { + $shards[] = ShardDefinition::MIGRATION_SHARD; + } $encodedShard = $primaryKey & ShardDefinition::PRIMARY_KEY_SHARD_MASK; if ($encodedShard < count($this->shardDefinition->shards) && !in_array($encodedShard, $shards)) { $shards[] = $encodedShard; diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index 473c0009237..0d5b2040513 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later |