diff options
Diffstat (limited to 'lib/private/DB/QueryBuilder')
11 files changed, 74 insertions, 21 deletions
diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php index 559c29df208..52f82db2232 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/SqliteExpressionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php index 2466493c1fa..48dc1da6330 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php index 6791430b1b0..47a8eaa6fd0 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later @@ -80,12 +81,12 @@ class OCIFunctionBuilder extends FunctionBuilder { public function octetLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTHB(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias); } public function charLength($field, $alias = ''): IQueryFunction { $alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : ''; $quotedName = $this->helper->quoteColumnName($field); - return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias); + return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias); } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php index ee430a6bd71..354a2b126d7 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php index 956b2123f2c..53aa530054b 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -1,4 +1,5 @@ <?php + /** * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php index 2942eeccdf7..d748c791321 100644 --- a/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php +++ b/lib/private/DB/QueryBuilder/Partitioned/PartitionedQueryBuilder.php @@ -126,8 +126,8 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { $selectPartition = null; } if ( - ($select === $checkColumn || $select === '*') && - $selectPartition === $partition + ($select === $checkColumn || $select === '*') + && $selectPartition === $partition ) { return; } @@ -151,8 +151,8 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { foreach ($this->selects as $select) { foreach ($this->partitions as $partition) { if (is_string($select['select']) && ( - $select['select'] === '*' || - $partition->isColumnInPartition($select['select'])) + $select['select'] === '*' + || $partition->isColumnInPartition($select['select'])) ) { if (isset($this->splitQueries[$partition->name])) { if ($select['alias']) { @@ -444,4 +444,19 @@ class PartitionedQueryBuilder extends ShardedQueryBuilder { public function getPartitionCount(): int { return count($this->splitQueries) + 1; } + + public function hintShardKey(string $column, mixed $value, bool $overwrite = false): self { + if (str_contains($column, '.')) { + [$alias, $column] = explode('.', $column); + $partition = $this->getPartition($alias); + if ($partition) { + $this->splitQueries[$partition->name]->query->hintShardKey($column, $value, $overwrite); + } else { + parent::hintShardKey($column, $value, $overwrite); + } + } else { + parent::hintShardKey($column, $value, $overwrite); + } + return $this; + } } diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 56c860bc42c..1d44c049793 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -96,6 +96,7 @@ class QueryBuilder implements IQueryBuilder { return match($this->connection->getDatabaseProvider()) { IDBConnection::PLATFORM_ORACLE => new OCIExpressionBuilder($this->connection, $this, $this->logger), IDBConnection::PLATFORM_POSTGRES => new PgSqlExpressionBuilder($this->connection, $this, $this->logger), + IDBConnection::PLATFORM_MARIADB, IDBConnection::PLATFORM_MYSQL => new MySqlExpressionBuilder($this->connection, $this, $this->logger), IDBConnection::PLATFORM_SQLITE => new SqliteExpressionBuilder($this->connection, $this, $this->logger), }; @@ -121,6 +122,7 @@ class QueryBuilder implements IQueryBuilder { return match($this->connection->getDatabaseProvider()) { IDBConnection::PLATFORM_ORACLE => new OCIFunctionBuilder($this->connection, $this, $this->helper), IDBConnection::PLATFORM_POSTGRES => new PgSqlFunctionBuilder($this->connection, $this, $this->helper), + IDBConnection::PLATFORM_MARIADB, IDBConnection::PLATFORM_MYSQL => new FunctionBuilder($this->connection, $this, $this->helper), IDBConnection::PLATFORM_SQLITE => new SqliteFunctionBuilder($this->connection, $this, $this->helper), }; @@ -161,7 +163,7 @@ class QueryBuilder implements IQueryBuilder { try { $params = []; foreach ($this->getParameters() as $placeholder => $value) { - if ($value instanceof \DateTime) { + if ($value instanceof \DateTimeInterface) { $params[] = $placeholder . ' => DateTime:\'' . $value->format('c') . '\''; } elseif (is_array($value)) { $params[] = $placeholder . ' => (\'' . implode('\', \'', $value) . '\')'; @@ -1339,6 +1341,8 @@ class QueryBuilder implements IQueryBuilder { /** * Returns the table name with database prefix as needed by the implementation * + * Was protected until version 30. + * * @param string $table * @return string */ diff --git a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php index caedaa71c97..3a230ea544d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php +++ b/lib/private/DB/QueryBuilder/Sharded/AutoIncrementHandler.php @@ -108,7 +108,7 @@ class AutoIncrementHandler { } // discard the encoded initial shard - $current = $this->getMaxFromDb($shardDefinition) >> 8; + $current = $this->getMaxFromDb($shardDefinition); $next = max($current, self::MIN_VALID_KEY) + 1; if ($cache->cas($shardDefinition->table, 'empty-placeholder', $next)) { return $next; @@ -131,19 +131,22 @@ class AutoIncrementHandler { } /** - * Get the maximum primary key value from the shards + * Get the maximum primary key value from the shards, note that this has already stripped any embedded shard id */ private function getMaxFromDb(ShardDefinition $shardDefinition): int { - $max = 0; + $max = $shardDefinition->fromFileId; + $query = $this->shardConnectionManager->getConnection($shardDefinition, 0)->getQueryBuilder(); + $query->select($shardDefinition->primaryKey) + ->from($shardDefinition->table) + ->orderBy($shardDefinition->primaryKey, 'DESC') + ->setMaxResults(1); foreach ($shardDefinition->getAllShards() as $shard) { $connection = $this->shardConnectionManager->getConnection($shardDefinition, $shard); - $query = $connection->getQueryBuilder(); - $query->select($shardDefinition->primaryKey) - ->from($shardDefinition->table) - ->orderBy($shardDefinition->primaryKey, 'DESC') - ->setMaxResults(1); - $result = $query->executeQuery()->fetchOne(); + $result = $query->executeQuery($connection)->fetchOne(); if ($result) { + if ($result > $shardDefinition->fromFileId) { + $result = $result >> 8; + } $max = max($max, $result); } } diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php index 87cac58bc57..74358e3ca96 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardConnectionManager.php @@ -28,8 +28,17 @@ class ShardConnectionManager { public function getConnection(ShardDefinition $shardDefinition, int $shard): IDBConnection { $connectionKey = $shardDefinition->table . '_' . $shard; - if (!isset($this->connections[$connectionKey])) { + + if (isset($this->connections[$connectionKey])) { + return $this->connections[$connectionKey]; + } + + if ($shard === ShardDefinition::MIGRATION_SHARD) { + $this->connections[$connectionKey] = \OC::$server->get(IDBConnection::class); + } elseif (isset($shardDefinition->shards[$shard])) { $this->connections[$connectionKey] = $this->createConnection($shardDefinition->shards[$shard]); + } else { + throw new \InvalidArgumentException("invalid shard key $shard only " . count($shardDefinition->shards) . ' configured'); } return $this->connections[$connectionKey]; diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php index ebccbb639a6..4f98079d92d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardDefinition.php @@ -15,7 +15,9 @@ use OCP\DB\QueryBuilder\Sharded\IShardMapper; */ class ShardDefinition { // we reserve the bottom byte of the primary key for the initial shard, so the total shard count is limited to what we can fit there - public const MAX_SHARDS = 256; + // additionally, shard id 255 is reserved for migration purposes + public const MAX_SHARDS = 255; + public const MIGRATION_SHARD = 255; public const PRIMARY_KEY_MASK = 0x7F_FF_FF_FF_FF_FF_FF_00; public const PRIMARY_KEY_SHARD_MASK = 0x00_00_00_00_00_00_00_FF; @@ -37,8 +39,10 @@ class ShardDefinition { public array $companionKeys, public string $shardKey, public IShardMapper $shardMapper, - public array $companionTables = [], - public array $shards = [], + public array $companionTables, + public array $shards, + public int $fromFileId, + public int $fromStorageId, ) { if (count($this->shards) >= self::MAX_SHARDS) { throw new \Exception('Only allowed maximum of ' . self::MAX_SHARDS . ' shards allowed'); @@ -53,11 +57,21 @@ class ShardDefinition { } public function getShardForKey(int $key): int { + if ($key < $this->fromStorageId) { + return self::MIGRATION_SHARD; + } return $this->shardMapper->getShardForKey($key, count($this->shards)); } + /** + * @return list<int> + */ public function getAllShards(): array { - return array_keys($this->shards); + if ($this->fromStorageId !== 0) { + return array_merge(array_keys($this->shards), [self::MIGRATION_SHARD]); + } else { + return array_keys($this->shards); + } } public function isKey(string $column): bool { diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php index c020e72868e..25e2a3d5f2d 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardQueryRunner.php @@ -55,6 +55,9 @@ class ShardQueryRunner { private function getLikelyShards(array $primaryKeys): array { $shards = []; foreach ($primaryKeys as $primaryKey) { + if ($primaryKey < $this->shardDefinition->fromFileId && !in_array(ShardDefinition::MIGRATION_SHARD, $shards)) { + $shards[] = ShardDefinition::MIGRATION_SHARD; + } $encodedShard = $primaryKey & ShardDefinition::PRIMARY_KEY_SHARD_MASK; if ($encodedShard < count($this->shardDefinition->shards) && !in_array($encodedShard, $shards)) { $shards[] = $encodedShard; |