From: Vsevolod Stakhov Date: Mon, 25 Dec 2023 09:28:14 +0000 (+0000) Subject: [Dep] Update unordered_dense to 4.4.0 X-Git-Tag: 3.8.0~20 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=35a828c7fe7b89b66a87dca91823dd9e74b59e11;p=rspamd.git [Dep] Update unordered_dense to 4.4.0 --- diff --git a/contrib/DEPENDENCY_INFO.md b/contrib/DEPENDENCY_INFO.md index 3fdf29a3d..300a38cee 100644 --- a/contrib/DEPENDENCY_INFO.md +++ b/contrib/DEPENDENCY_INFO.md @@ -36,6 +36,6 @@ | doctest | 2.4.6 | MIT | NO | | | function2 | 4.1.0 | Boost | NO | | | ankerl/svector | 1.0.2 | MIT | NO | | -| ankerl/unordered_dense | 2.0.1 | MIT | NO | | +| ankerl/unordered_dense | 4.4.0 | MIT | NO | | | backward-cpp | 1.6 | MIT | NO | | diff --git a/contrib/ankerl/unordered_dense.h b/contrib/ankerl/unordered_dense.h index 737d12bf3..2aaacd617 100644 --- a/contrib/ankerl/unordered_dense.h +++ b/contrib/ankerl/unordered_dense.h @@ -1,12 +1,12 @@ ///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// // A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. -// Version 2.0.1 +// Version 4.4.0 // https://github.com/martinus/unordered_dense // // Licensed under the MIT License . // SPDX-License-Identifier: MIT -// Copyright (c) 2022 Martin Leitner-Ankerl +// Copyright (c) 2022-2023 Martin Leitner-Ankerl // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,12 +30,15 @@ #define ANKERL_UNORDERED_DENSE_H // see https://semver.org/spec/v2.0.0.html -#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 2 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes -#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality -#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 1 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes +#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes +#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality +#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes // API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) #define ANKERL_UNORDERED_DENSE_NAMESPACE \ ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \ @@ -55,6 +58,23 @@ # define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop)) #endif +// exceptions +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage) +#else +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage) +#endif +#ifdef _MSC_VER +# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline) +#else +# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) +#endif + +// defined in unordered_dense.cpp +#if !defined(ANKERL_UNORDERED_DENSE_EXPORT) +# define ANKERL_UNORDERED_DENSE_EXPORT +#endif + #if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L # error ankerl::unordered_dense requires C++17 or higher #else @@ -66,6 +86,7 @@ # include // for pair, distance # include // for numeric_limits # include // for allocator, allocator_traits, shared_ptr +# include // for optional # include // for out_of_range # include // for basic_string # include // for basic_string_view, hash @@ -73,13 +94,17 @@ # include // for enable_if_t, declval, conditional_t, ena... # include // for forward, exchange, pair, as_const, piece... # include // for vector +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0 +# include // for abort +# endif -# define ANKERL_UNORDERED_DENSE_PMR 0 // NOLINT(cppcoreguidelines-macro-usage) # if defined(__has_include) # if __has_include() -# undef ANKERL_UNORDERED_DENSE_PMR -# define ANKERL_UNORDERED_DENSE_PMR 1 // NOLINT(cppcoreguidelines-macro-usage) -# include // for polymorphic_allocator +# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include // for polymorphic_allocator +# elif __has_include() +# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include // for polymorphic_allocator # endif # endif @@ -99,14 +124,46 @@ namespace ankerl::unordered_dense { inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE { +namespace detail { + +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() { + throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() { + throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() { + throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); +} + +# else + +[[noreturn]] inline void on_error_key_not_found() { + abort(); +} +[[noreturn]] inline void on_error_bucket_overflow() { + abort(); +} +[[noreturn]] inline void on_error_too_many_elements() { + abort(); +} + +# endif + +} // namespace detail + // hash /////////////////////////////////////////////////////////////////////// // This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash // No big-endian support (because different values on different machines don't matter), -// hardcodes seed and the secret, reformattes the code, and clang-tidy fixes. +// hardcodes seed and the secret, reformats the code, and clang-tidy fixes. namespace detail::wyhash { -static inline void mum(uint64_t* a, uint64_t* b) { +inline void mum(uint64_t* a, uint64_t* b) { # if defined(__SIZEOF_INT128__) __uint128_t r = *a; r *= *b; @@ -136,30 +193,30 @@ static inline void mum(uint64_t* a, uint64_t* b) { } // multiply and xor mix function, aka MUM -[[nodiscard]] static inline auto mix(uint64_t a, uint64_t b) -> uint64_t { +[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t { mum(&a, &b); return a ^ b; } // read functions. WARNING: we don't care about endianness, so results are different on big endian! -[[nodiscard]] static inline auto r8(const uint8_t* p) -> uint64_t { +[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t { uint64_t v{}; std::memcpy(&v, p, 8U); return v; } -[[nodiscard]] static inline auto r4(const uint8_t* p) -> uint64_t { +[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t { uint32_t v{}; std::memcpy(&v, p, 4); return v; } // reads 1, 2, or 3 bytes -[[nodiscard]] static inline auto r3(const uint8_t* p, size_t k) -> uint64_t { +[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t { return (static_cast(p[0]) << 16U) | (static_cast(p[k >> 1U]) << 8U) | p[k - 1]; } -[[maybe_unused]] [[nodiscard]] static inline auto hash(void const* key, size_t len) -> uint64_t { +[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t { static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f), UINT64_C(0xe7037ed1a0b428db), UINT64_C(0x8ebc6af09c88c6e3), @@ -206,13 +263,13 @@ static inline void mum(uint64_t* a, uint64_t* b) { return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); } -[[nodiscard]] static inline auto hash(uint64_t x) -> uint64_t { +[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t { return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); } } // namespace detail::wyhash -template +ANKERL_UNORDERED_DENSE_EXPORT template struct hash { auto operator()(T const& obj) const noexcept(noexcept(std::declval>().operator()(std::declval()))) -> uint64_t { @@ -272,6 +329,50 @@ struct hash::value>::type> { } }; +template +struct tuple_hash_helper { + // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest. + // If it isn't an integral we need to hash it. + template + [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t { + if constexpr (std::is_integral_v || std::is_enum_v) { + return static_cast(arg); + } else { + return hash{}(arg); + } + } + + [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t { + return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69}); + } + + // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If + // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized + // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer. + template + [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence) noexcept -> uint64_t { + auto h = uint64_t{}; + ((h = mix64(h, to64(std::get(t)))), ...); + return h; + } +}; + +template +struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::tuple const& t) const noexcept -> uint64_t { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + +template +struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::pair const& t) const noexcept -> uint64_t { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) # define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \ template <> \ @@ -291,7 +392,7 @@ ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool); ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char); ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char); ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char); -# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L +# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t) ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t); # endif ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t); @@ -371,8 +472,10 @@ using detect_reserve = decltype(std::declval().reserve(size_t{})); template constexpr bool is_map_v = !std::is_void_v; +// clang-format off template -constexpr bool is_transparent_v = is_detected_v&& is_detected_v; +constexpr bool is_transparent_v = is_detected_v && is_detected_v; +// clang-format on template constexpr bool is_neither_convertible_v = !std::is_convertible_v && !std::is_convertible_v; @@ -380,31 +483,340 @@ constexpr bool is_neither_convertible_v = !std::is_convertible_v && ! template constexpr bool has_reserve = is_detected_v; +// base type for map has mapped_type +template +struct base_table_type_map { + using mapped_type = T; +}; + +// base type for set doesn't have mapped_type +struct base_table_type_set {}; + +} // namespace detail + +// Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector +// API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}. +// It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new +// block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not +// linear and thus there is one more indirection necessary for indexing. +template , size_t MaxSegmentSizeBytes = 4096> +class segmented_vector { + template + class iter_t; + +public: + using allocator_type = Allocator; + using pointer = typename std::allocator_traits::pointer; + using const_pointer = typename std::allocator_traits::const_pointer; + using difference_type = typename std::allocator_traits::difference_type; + using value_type = T; + using size_type = std::size_t; + using reference = T&; + using const_reference = T const&; + using iterator = iter_t; + using const_iterator = iter_t; + +private: + using vec_alloc = typename std::allocator_traits::template rebind_alloc; + std::vector m_blocks{}; + size_t m_size{}; + + // Calculates the maximum number for x in (s << x) <= max_val + static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t { + auto f = size_t{0}; + while (s << (f + 1) <= max_val) { + ++f; + } + return f; + } + + using self_t = segmented_vector; + static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T)); + static constexpr auto num_elements_in_block = 1U << num_bits; + static constexpr auto mask = num_elements_in_block - 1U; + + /** + * Iterator class doubles as const_iterator and iterator + */ + template + class iter_t { + using ptr_t = typename std::conditional_t; + ptr_t m_data{}; + size_t m_idx{}; + + template + friend class iter_t; + + public: + using difference_type = segmented_vector::difference_type; + using value_type = T; + using reference = typename std::conditional_t; + using pointer = typename std::conditional_t; + using iterator_category = std::forward_iterator_tag; + + iter_t() noexcept = default; + + template ::type> + // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) + constexpr iter_t(iter_t const& other) noexcept + : m_data(other.m_data) + , m_idx(other.m_idx) {} + + constexpr iter_t(ptr_t data, size_t idx) noexcept + : m_data(data) + , m_idx(idx) {} + + template ::type> + constexpr auto operator=(iter_t const& other) noexcept -> iter_t& { + m_data = other.m_data; + m_idx = other.m_idx; + return *this; + } + + constexpr auto operator++() noexcept -> iter_t& { + ++m_idx; + return *this; + } + + constexpr auto operator+(difference_type diff) noexcept -> iter_t { + return {m_data, static_cast(static_cast(m_idx) + diff)}; + } + + template + constexpr auto operator-(iter_t const& other) noexcept -> difference_type { + return static_cast(m_idx) - static_cast(other.m_idx); + } + + constexpr auto operator*() const noexcept -> reference { + return m_data[m_idx >> num_bits][m_idx & mask]; + } + + constexpr auto operator->() const noexcept -> pointer { + return &m_data[m_idx >> num_bits][m_idx & mask]; + } + + template + constexpr auto operator==(iter_t const& o) const noexcept -> bool { + return m_idx == o.m_idx; + } + + template + constexpr auto operator!=(iter_t const& o) const noexcept -> bool { + return !(*this == o); + } + }; + + // slow path: need to allocate a new segment every once in a while + void increase_capacity() { + auto ba = Allocator(m_blocks.get_allocator()); + pointer block = std::allocator_traits::allocate(ba, num_elements_in_block); + m_blocks.push_back(block); + } + + // Moves everything from other + void append_everything_from(segmented_vector&& other) { + reserve(size() + other.size()); + for (auto&& o : other) { + emplace_back(std::move(o)); + } + } + + // Copies everything from other + void append_everything_from(segmented_vector const& other) { + reserve(size() + other.size()); + for (auto const& o : other) { + emplace_back(o); + } + } + + void dealloc() { + auto ba = Allocator(m_blocks.get_allocator()); + for (auto ptr : m_blocks) { + std::allocator_traits::deallocate(ba, ptr, num_elements_in_block); + } + } + + [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) { + return (capacity + num_elements_in_block - 1U) / num_elements_in_block; + } + +public: + segmented_vector() = default; + + // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) + segmented_vector(Allocator alloc) + : m_blocks(vec_alloc(alloc)) {} + + segmented_vector(segmented_vector&& other, Allocator alloc) + : segmented_vector(alloc) { + *this = std::move(other); + } + + segmented_vector(segmented_vector const& other, Allocator alloc) + : m_blocks(vec_alloc(alloc)) { + append_everything_from(other); + } + + segmented_vector(segmented_vector&& other) noexcept + : segmented_vector(std::move(other), get_allocator()) {} + + segmented_vector(segmented_vector const& other) { + append_everything_from(other); + } + + auto operator=(segmented_vector const& other) -> segmented_vector& { + if (this == &other) { + return *this; + } + clear(); + append_everything_from(other); + return *this; + } + + auto operator=(segmented_vector&& other) noexcept -> segmented_vector& { + clear(); + dealloc(); + if (other.get_allocator() == get_allocator()) { + m_blocks = std::move(other.m_blocks); + m_size = std::exchange(other.m_size, {}); + } else { + // make sure to construct with other's allocator! + m_blocks = std::vector(vec_alloc(other.get_allocator())); + append_everything_from(std::move(other)); + } + return *this; + } + + ~segmented_vector() { + clear(); + dealloc(); + } + + [[nodiscard]] constexpr auto size() const -> size_t { + return m_size; + } + + [[nodiscard]] constexpr auto capacity() const -> size_t { + return m_blocks.size() * num_elements_in_block; + } + + // Indexing is highly performance critical + [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& { + return m_blocks[i >> num_bits][i & mask]; + } + + [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& { + return m_blocks[i >> num_bits][i & mask]; + } + + [[nodiscard]] constexpr auto begin() -> iterator { + return {m_blocks.data(), 0U}; + } + [[nodiscard]] constexpr auto begin() const -> const_iterator { + return {m_blocks.data(), 0U}; + } + [[nodiscard]] constexpr auto cbegin() const -> const_iterator { + return {m_blocks.data(), 0U}; + } + + [[nodiscard]] constexpr auto end() -> iterator { + return {m_blocks.data(), m_size}; + } + [[nodiscard]] constexpr auto end() const -> const_iterator { + return {m_blocks.data(), m_size}; + } + [[nodiscard]] constexpr auto cend() const -> const_iterator { + return {m_blocks.data(), m_size}; + } + + [[nodiscard]] constexpr auto back() -> reference { + return operator[](m_size - 1); + } + [[nodiscard]] constexpr auto back() const -> const_reference { + return operator[](m_size - 1); + } + + void pop_back() { + back().~T(); + --m_size; + } + + [[nodiscard]] auto empty() const { + return 0 == m_size; + } + + void reserve(size_t new_capacity) { + m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity)); + while (new_capacity > capacity()) { + increase_capacity(); + } + } + + [[nodiscard]] auto get_allocator() const -> allocator_type { + return allocator_type{m_blocks.get_allocator()}; + } + + template + auto emplace_back(Args&&... args) -> reference { + if (m_size == capacity()) { + increase_capacity(); + } + auto* ptr = static_cast(&operator[](m_size)); + auto& ref = *new (ptr) T(std::forward(args)...); + ++m_size; + return ref; + } + + void clear() { + if constexpr (!std::is_trivially_destructible_v) { + for (size_t i = 0, s = size(); i < s; ++i) { + operator[](i).~T(); + } + } + m_size = 0; + } + + void shrink_to_fit() { + auto ba = Allocator(m_blocks.get_allocator()); + auto num_blocks_required = calc_num_blocks_for_capacity(m_size); + while (m_blocks.size() > num_blocks_required) { + std::allocator_traits::deallocate(ba, m_blocks.back(), num_elements_in_block); + m_blocks.pop_back(); + } + m_blocks.shrink_to_fit(); + } +}; + +namespace detail { + // This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set. template -class table { + class Bucket, + bool IsSegmented> +class table : public std::conditional_t, base_table_type_map, base_table_type_set> { + using underlying_value_type = typename std::conditional_t, std::pair, Key>; + using underlying_container_type = std::conditional_t, + std::vector>; + public: - using value_container_type = std::conditional_t< - is_detected_v, - AllocatorOrContainer, - typename std::vector, Key, std::pair>, AllocatorOrContainer>>; + using value_container_type = std:: + conditional_t, AllocatorOrContainer, underlying_container_type>; private: using bucket_alloc = typename std::allocator_traits::template rebind_alloc; using bucket_alloc_traits = std::allocator_traits; - static constexpr uint8_t initial_shifts = 64 - 3; // 2^(64-m_shift) number of buckets + static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets static constexpr float default_max_load_factor = 0.8F; public: using key_type = Key; - using mapped_type = T; using value_type = typename value_container_type::value_type; using size_type = typename value_container_type::size_type; using difference_type = typename value_container_type::difference_type; @@ -415,8 +827,8 @@ public: using const_reference = typename value_container_type::const_reference; using pointer = typename value_container_type::pointer; using const_pointer = typename value_container_type::const_pointer; - using iterator = typename value_container_type::iterator; using const_iterator = typename value_container_type::const_iterator; + using iterator = std::conditional_t, typename value_container_type::iterator, const_iterator>; using bucket_type = Bucket; private: @@ -427,7 +839,8 @@ private: static_assert(std::is_trivially_copyable_v, "assert we can just memset / memcpy"); value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes. - typename std::allocator_traits::pointer m_buckets{}; + using bucket_pointer = typename std::allocator_traits::pointer; + bucket_pointer m_buckets{}; size_t m_num_buckets = 0; size_t m_max_bucket_capacity = 0; float m_max_load_factor = default_max_load_factor; @@ -442,8 +855,7 @@ private: } // Helper to access bucket through pointer types - [[nodiscard]] static constexpr auto at(typename std::allocator_traits::pointer bucket_ptr, size_t offset) - -> Bucket& { + [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& { return *(bucket_ptr + static_cast::difference_type>(offset)); } @@ -483,10 +895,10 @@ private: } [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& { - if constexpr (std::is_void_v) { - return vt; - } else { + if constexpr (is_map_v) { return vt.first; + } else { + return vt; } } @@ -513,7 +925,7 @@ private: } [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t { - return std::min(max_bucket_count(), size_t{1} << (64U - shifts)); + return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts)); } [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t { @@ -526,7 +938,12 @@ private: // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS void copy_buckets(table const& other) { - if (!empty()) { + // assumes m_values has already the correct data copied over. + if (empty()) { + // when empty, at least allocate an initial buckets and clear them. + allocate_buckets_from_shift(); + clear_buckets(); + } else { m_shifts = other.m_shifts; allocate_buckets_from_shift(); std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count()); @@ -537,15 +954,15 @@ private: * True when no element can be added any more without increasing the size */ [[nodiscard]] auto is_full() const -> bool { - return size() >= m_max_bucket_capacity; + return size() > m_max_bucket_capacity; } void deallocate_buckets() { auto ba = bucket_alloc(m_values.get_allocator()); if (nullptr != m_buckets) { bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count()); + m_buckets = nullptr; } - m_buckets = nullptr; m_num_buckets = 0; m_max_bucket_capacity = 0; } @@ -581,8 +998,10 @@ private: } void increase_size() { - if (ANKERL_UNORDERED_DENSE_UNLIKELY(m_max_bucket_capacity == max_bucket_count())) { - throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size"); + if (m_max_bucket_capacity == max_bucket_count()) { + // remove the value again, we can't add it! + m_values.pop_back(); + on_error_bucket_overflow(); } --m_shifts; deallocate_buckets(); @@ -590,7 +1009,8 @@ private: clear_and_fill_buckets_from_values(); } - void do_erase(value_idx_type bucket_idx) { + template + void do_erase(value_idx_type bucket_idx, Op handle_erased_value) { auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx; // shift down until either empty or an element with correct spot is found @@ -601,6 +1021,7 @@ private: bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx)); } at(m_buckets, bucket_idx) = {}; + handle_erased_value(std::move(m_values[value_idx_to_remove])); // update m_values if (value_idx_to_remove != m_values.size() - 1) { @@ -621,8 +1042,8 @@ private: m_values.pop_back(); } - template - auto do_erase_key(K&& key) -> size_t { + template + auto do_erase_key(K&& key, Op handle_erased_value) -> size_t { if (empty()) { return 0; } @@ -638,7 +1059,7 @@ private: if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) { return 0; } - do_erase(bucket_idx); + do_erase(bucket_idx, handle_erased_value); return 1; } @@ -651,27 +1072,26 @@ private: return it_isinserted; } - template - auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, K&& key, Args&&... args) + template + auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args) -> std::pair { // emplace the new value. If that throws an exception, no harm done; index is still in a valid state - m_values.emplace_back(std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), - std::forward_as_tuple(std::forward(args)...)); + m_values.emplace_back(std::forward(args)...); - // place element and shift up until we find an empty spot auto value_idx = static_cast(m_values.size() - 1); - place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) { + increase_size(); + } else { + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + } + + // place element and shift up until we find an empty spot return {begin() + static_cast(value_idx), true}; } template auto do_try_emplace(K&& key, Args&&... args) -> std::pair { - if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) { - increase_size(); - } - auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); @@ -679,11 +1099,15 @@ private: while (true) { auto* bucket = &at(m_buckets, bucket_idx); if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) { - if (m_equal(key, m_values[bucket->m_value_idx].first)) { + if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) { return {begin() + static_cast(bucket->m_value_idx), false}; } } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) { - return do_place_element(dist_and_fingerprint, bucket_idx, std::forward(key), std::forward(args)...); + return do_place_element(dist_and_fingerprint, + bucket_idx, + std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); } dist_and_fingerprint = dist_inc(dist_and_fingerprint); bucket_idx = next(bucket_idx); @@ -737,10 +1161,10 @@ private: template , bool> = true> auto do_at(K const& key) -> Q& { - if (auto it = find(key); end() != it) { + if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) { return it->second; } - throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found"); + on_error_key_not_found(); } template , bool> = true> @@ -749,9 +1173,6 @@ private: } public: - table() - : table(0) {} - explicit table(size_t bucket_count, Hash const& hash = Hash(), KeyEqual const& equal = KeyEqual(), @@ -761,9 +1182,15 @@ public: , m_equal(equal) { if (0 != bucket_count) { reserve(bucket_count); + } else { + allocate_buckets_from_shift(); + clear_buckets(); } } + table() + : table(0) {} + table(size_t bucket_count, allocator_type const& alloc) : table(bucket_count, Hash(), KeyEqual(), alloc) {} @@ -807,15 +1234,8 @@ public: : table(std::move(other), other.m_values.get_allocator()) {} table(table&& other, allocator_type const& alloc) noexcept - : m_values(std::move(other.m_values), alloc) - , m_buckets(std::exchange(other.m_buckets, nullptr)) - , m_num_buckets(std::exchange(other.m_num_buckets, 0)) - , m_max_bucket_capacity(std::exchange(other.m_max_bucket_capacity, 0)) - , m_max_load_factor(std::exchange(other.m_max_load_factor, default_max_load_factor)) - , m_hash(std::exchange(other.m_hash, {})) - , m_equal(std::exchange(other.m_equal, {})) - , m_shifts(std::exchange(other.m_shifts, initial_shifts)) { - other.m_values.clear(); + : m_values(alloc) { + *this = std::move(other); } table(std::initializer_list ilist, @@ -834,8 +1254,10 @@ public: : table(init, bucket_count, hash, KeyEqual(), alloc) {} ~table() { - auto ba = bucket_alloc(m_values.get_allocator()); - bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count()); + if (nullptr != m_buckets) { + auto ba = bucket_alloc(m_values.get_allocator()); + bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count()); + } } auto operator=(table const& other) -> table& { @@ -851,20 +1273,38 @@ public: return *this; } - auto operator=(table&& other) noexcept( - noexcept(std::is_nothrow_move_assignable_v&& std::is_nothrow_move_assignable_v&& - std::is_nothrow_move_assignable_v)) -> table& { + auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v && + std::is_nothrow_move_assignable_v && + std::is_nothrow_move_assignable_v)) -> table& { if (&other != this) { deallocate_buckets(); // deallocate before m_values is set (might have another allocator) m_values = std::move(other.m_values); - m_buckets = std::exchange(other.m_buckets, nullptr); - m_num_buckets = std::exchange(other.m_num_buckets, 0); - m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0); - m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor); - m_hash = std::exchange(other.m_hash, {}); - m_equal = std::exchange(other.m_equal, {}); - m_shifts = std::exchange(other.m_shifts, initial_shifts); other.m_values.clear(); + + // we can only reuse m_buckets when both maps have the same allocator! + if (get_allocator() == other.get_allocator()) { + m_buckets = std::exchange(other.m_buckets, nullptr); + m_num_buckets = std::exchange(other.m_num_buckets, 0); + m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0); + m_shifts = std::exchange(other.m_shifts, initial_shifts); + m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor); + m_hash = std::exchange(other.m_hash, {}); + m_equal = std::exchange(other.m_equal, {}); + other.allocate_buckets_from_shift(); + other.clear_buckets(); + } else { + // set max_load_factor *before* copying the other's buckets, so we have the same + // behavior + m_max_load_factor = other.m_max_load_factor; + + // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts + copy_buckets(other); + // clear's the other's buckets so other is now already usable. + other.clear_buckets(); + m_hash = other.m_hash; + m_equal = other.m_equal; + } + // map "other" is now already usable, it's empty. } return *this; } @@ -916,7 +1356,7 @@ public: } [[nodiscard]] static constexpr auto max_size() noexcept -> size_t { - if constexpr (std::numeric_limits::max() == std::numeric_limits::max()) { + if constexpr ((std::numeric_limits::max)() == (std::numeric_limits::max)()) { return size_t{1} << (sizeof(value_idx_type) * 8 - 1); } else { return size_t{1} << (sizeof(value_idx_type) * 8); @@ -977,10 +1417,9 @@ public: // nonstandard API: // Discards the internally held container and replaces it with the one passed. Erases non-unique elements. auto replace(value_container_type&& container) { - if (container.size() > max_size()) { - throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); + if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) { + on_error_too_many_elements(); } - auto shifts = calc_shifts_for_size(container.size()); if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) { m_shifts = shifts; @@ -1009,7 +1448,7 @@ public: break; } if (dist_and_fingerprint == bucket.m_dist_and_fingerprint && - m_equal(key, m_values[bucket.m_value_idx].first)) { + m_equal(key, get_key(m_values[bucket.m_value_idx]))) { key_found = true; break; } @@ -1076,10 +1515,6 @@ public: typename KE = KeyEqual, std::enable_if_t && is_transparent_v, bool> = true> auto emplace(K&& key) -> std::pair { - if (is_full()) { - increase_size(); - } - auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); @@ -1095,19 +1530,11 @@ public: } // value is new, insert element first, so when exception happens we are in a valid state - m_values.emplace_back(std::forward(key)); - // now place the bucket and shift up until we find an empty spot - auto value_idx = static_cast(m_values.size() - 1); - place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); - return {begin() + static_cast(value_idx), true}; + return do_place_element(dist_and_fingerprint, bucket_idx, std::forward(key)); } template auto emplace(Args&&... args) -> std::pair { - if (is_full()) { - increase_size(); - } - // we have to instantiate the value_type to be able to access the key. // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop. auto& key = get_key(m_values.emplace_back(std::forward(args)...)); @@ -1127,8 +1554,13 @@ public: // value is new, place the bucket and shift up until we find an empty spot auto value_idx = static_cast(m_values.size() - 1); - place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); - + if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) { + // increase_size just rehashes all the data we have in m_values + increase_size(); + } else { + // place element and shift up until we find an empty spot + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + } return {begin() + static_cast(value_idx), true}; } @@ -1190,14 +1622,37 @@ public: bucket_idx = next(bucket_idx); } - do_erase(bucket_idx); + do_erase(bucket_idx, [](value_type&& /*unused*/) { + }); return begin() + static_cast(value_idx_to_remove); } + auto extract(iterator it) -> value_type { + auto hash = mixed_hash(get_key(*it)); + auto bucket_idx = bucket_idx_from_hash(hash); + + auto const value_idx_to_remove = static_cast(it - cbegin()); + while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) { + bucket_idx = next(bucket_idx); + } + + auto tmp = std::optional{}; + do_erase(bucket_idx, [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return std::move(tmp).value(); + } + + template , bool> = true> auto erase(const_iterator it) -> iterator { return erase(begin() + (it - cbegin())); } + template , bool> = true> + auto extract(const_iterator it) -> value_type { + return extract(begin() + (it - cbegin())); + } + auto erase(const_iterator first, const_iterator last) -> iterator { auto const idx_first = first - cbegin(); auto const idx_last = last - cbegin(); @@ -1205,7 +1660,7 @@ public: auto const last_to_end = std::distance(last, cend()); // remove elements from left to right which moves elements from the end back - auto const mid = idx_first + std::min(first_to_last, last_to_end); + auto const mid = idx_first + (std::min)(first_to_last, last_to_end); auto idx = idx_first; while (idx != mid) { erase(begin() + idx); @@ -1223,16 +1678,35 @@ public: } auto erase(Key const& key) -> size_t { - return do_erase_key(key); + return do_erase_key(key, [](value_type&& /*unused*/) { + }); + } + + auto extract(Key const& key) -> std::optional { + auto tmp = std::optional{}; + do_erase_key(key, [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return tmp; } template , bool> = true> auto erase(K&& key) -> size_t { - return do_erase_key(std::forward(key)); + return do_erase_key(std::forward(key), [](value_type&& /*unused*/) { + }); + } + + template , bool> = true> + auto extract(K&& key) -> std::optional { + auto tmp = std::optional{}; + do_erase_key(std::forward(key), [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return tmp; } - void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v&& - std::is_nothrow_swappable_v&& std::is_nothrow_swappable_v)) { + void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v && + std::is_nothrow_swappable_v && std::is_nothrow_swappable_v)) { using std::swap; swap(other, *this); } @@ -1372,8 +1846,8 @@ public: } void rehash(size_t count) { - count = std::min(count, max_size()); - auto shifts = calc_shifts_for_size(std::max(count, size())); + count = (std::min)(count, max_size()); + auto shifts = calc_shifts_for_size((std::max)(count, size())); if (shifts != m_shifts) { m_shifts = shifts; deallocate_buckets(); @@ -1384,12 +1858,12 @@ public: } void reserve(size_t capa) { - capa = std::min(capa, max_size()); + capa = (std::min)(capa, max_size()); if constexpr (has_reserve) { // std::deque doesn't have reserve(). Make sure we only call when available m_values.reserve(capa); } - auto shifts = calc_shifts_for_size(std::max(capa, size())); + auto shifts = calc_shifts_for_size((std::max)(capa, size())); if (0 == m_num_buckets || shifts < m_shifts) { m_shifts = shifts; deallocate_buckets(); @@ -1424,14 +1898,14 @@ public: } for (auto const& b_entry : b) { auto it = a.find(get_key(b_entry)); - if constexpr (std::is_void_v) { - // set: only check that the key is here - if (a.end() == it) { + if constexpr (is_map_v) { + // map: check that key is here, then also check that value is the same + if (a.end() == it || !(b_entry.second == it->second)) { return false; } } else { - // map: check that key is here, then also check that value is the same - if (a.end() == it || !(b_entry.second == it->second)) { + // set: only check that the key is here + if (a.end() == it) { return false; } } @@ -1446,34 +1920,68 @@ public: } // namespace detail -template , - class KeyEqual = std::equal_to, - class AllocatorOrContainer = std::allocator>, - class Bucket = bucket_type::standard> -using map = detail::table; - -template , - class KeyEqual = std::equal_to, - class AllocatorOrContainer = std::allocator, - class Bucket = bucket_type::standard> -using set = detail::table; - -# if ANKERL_UNORDERED_DENSE_PMR +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class AllocatorOrContainer = std::allocator>, + class Bucket = bucket_type::standard> +using map = detail::table; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class AllocatorOrContainer = std::allocator>, + class Bucket = bucket_type::standard> +using segmented_map = detail::table; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class AllocatorOrContainer = std::allocator, + class Bucket = bucket_type::standard> +using set = detail::table; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class AllocatorOrContainer = std::allocator, + class Bucket = bucket_type::standard> +using segmented_set = detail::table; + +# if defined(ANKERL_UNORDERED_DENSE_PMR) namespace pmr { -template , - class KeyEqual = std::equal_to, - class Bucket = bucket_type::standard> -using map = detail::table>, Bucket>; - -template , class KeyEqual = std::equal_to, class Bucket = bucket_type::standard> -using set = detail::table, Bucket>; +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::standard> +using map = + detail::table>, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::standard> +using segmented_map = + detail::table>, Bucket, true>; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::standard> +using set = detail::table, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template , + class KeyEqual = std::equal_to, + class Bucket = bucket_type::standard> +using segmented_set = + detail::table, Bucket, true>; } // namespace pmr @@ -1491,10 +1999,18 @@ using set = detail::table -auto erase_if(ankerl::unordered_dense::detail::table& map, Pred pred) - -> size_t { - using map_t = ankerl::unordered_dense::detail::table; +ANKERL_UNORDERED_DENSE_EXPORT template +// NOLINTNEXTLINE(cert-dcl58-cpp) +auto erase_if(ankerl::unordered_dense::detail::table& map, + Pred pred) -> size_t { + using map_t = ankerl::unordered_dense::detail::table; // going back to front because erase() invalidates the end iterator auto const old_size = map.size(); @@ -1507,7 +2023,7 @@ auto erase_if(ankerl::unordered_dense::detail::table