// ______ _____ ______ _________ // ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ / // __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ / // _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ / // /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/ // _/_____/ // // Fast & memory efficient hashtable based on robin hood hashing for // C++11/14/17/20 // https://github.com/martinus/robin-hood-hashing // // Licensed under the MIT License . // SPDX-License-Identifier: MIT // Copyright (c) 2018-2021 Martin Ankerl // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all // copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. #ifndef ROBIN_HOOD_H_INCLUDED #define ROBIN_HOOD_H_INCLUDED // see https://semver.org/ #define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes #define ROBIN_HOOD_VERSION_MINOR \ 11 // for adding functionality in a backwards-compatible manner #define ROBIN_HOOD_VERSION_PATCH 1 // for backwards-compatible bug fixes #include #include #include #include #include // only to support hash of smart pointers #include #include #include #include #if __cplusplus >= 201703L #include #endif // #define ROBIN_HOOD_LOG_ENABLED #ifdef ROBIN_HOOD_LOG_ENABLED #include #define ROBIN_HOOD_LOG(...) \ std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ \ << std::endl; #else #define ROBIN_HOOD_LOG(x) #endif // #define ROBIN_HOOD_TRACE_ENABLED #ifdef ROBIN_HOOD_TRACE_ENABLED #include #define ROBIN_HOOD_TRACE(...) \ std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ \ << std::endl; #else #define ROBIN_HOOD_TRACE(x) #endif // #define ROBIN_HOOD_COUNT_ENABLED #ifdef ROBIN_HOOD_COUNT_ENABLED #include #define ROBIN_HOOD_COUNT(x) ++counts().x; namespace robin_hood { struct Counts { uint64_t shiftUp{}; uint64_t shiftDown{}; }; inline std::ostream &operator<<(std::ostream &os, Counts const &c) { return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; } static Counts &counts() { static Counts counts{}; return counts; } } // namespace robin_hood #else #define ROBIN_HOOD_COUNT(x) #endif // all non-argument macros should use this facility. See // https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/ #define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x() // mark unused members with this macro #define ROBIN_HOOD_UNUSED(identifier) // bitness #if SIZE_MAX == UINT32_MAX #define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 #elif SIZE_MAX == UINT64_MAX #define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 #else #error Unsupported bitness #endif // endianess #ifdef _MSC_VER #define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 #define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 #else #define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \ (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() \ (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #endif // inline #ifdef _MSC_VER #define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) #else #define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) #endif // exceptions #if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) #define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 #else #define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 #endif // count leading/trailing bits #if !defined(ROBIN_HOOD_DISABLE_INTRINSICS) #ifdef _MSC_VER #if ROBIN_HOOD(BITNESS) == 32 #define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward #else #define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 #endif #include #pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) #define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ [](size_t mask) noexcept->int { \ unsigned long index; \ return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ : ROBIN_HOOD(BITNESS); \ } \ (x) #else #if ROBIN_HOOD(BITNESS) == 32 #define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl #define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl #else #define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll #define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll #endif #define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) \ ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) #define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) #endif #endif // fallthrough #ifndef __has_cpp_attribute // For backwards compatibility #define __has_cpp_attribute(x) 0 #endif #if __has_cpp_attribute(clang::fallthrough) #define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] #elif __has_cpp_attribute(gnu::fallthrough) #define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] #else #define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() #endif // likely/unlikely #ifdef _MSC_VER #define ROBIN_HOOD_LIKELY(condition) condition #define ROBIN_HOOD_UNLIKELY(condition) condition #else #define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) #define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) #endif // detect if native wchar_t type is availiable in MSVC #ifdef _MSC_VER #ifdef _NATIVE_WCHAR_T_DEFINED #define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 #else #define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 #endif #else #define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 #endif // detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor // being constexpr #ifdef _MSC_VER #if _MSC_VER <= 1900 #define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1 #else #define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 #endif #else #define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 #endif // workaround missing "is_trivially_copyable" in g++ < 5.0 // See https://stackoverflow.com/a/31798726/48181 #if defined(__GNUC__) && __GNUC__ < 5 #define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) #else #define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) \ std::is_trivially_copyable<__VA_ARGS__>::value #endif // helpers for C++ versions, see // https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L #define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) #define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] #else #define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() #endif namespace robin_hood { #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) #define ROBIN_HOOD_STD std #else // c++11 compatibility layer namespace ROBIN_HOOD_STD { template struct alignment_of : std::integral_constant< std::size_t, alignof(typename std::remove_all_extents::type)> {}; template class integer_sequence { public: using value_type = T; static_assert(std::is_integral::value, "not integral type"); static constexpr std::size_t size() noexcept { return sizeof...(Ints); } }; template using index_sequence = integer_sequence; namespace detail_ { template struct IntSeqImpl { using TValue = T; static_assert(std::is_integral::value, "not integral type"); static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); template struct IntSeqCombiner; template struct IntSeqCombiner, integer_sequence> { using TResult = integer_sequence; }; using TResult = typename IntSeqCombiner< typename IntSeqImpl::TResult, typename IntSeqImpl::TResult>::TResult; }; template struct IntSeqImpl { using TValue = T; static_assert(std::is_integral::value, "not integral type"); static_assert(Begin >= 0, "unexpected argument (Begin<0)"); using TResult = integer_sequence; }; template struct IntSeqImpl { using TValue = T; static_assert(std::is_integral::value, "not integral type"); static_assert(Begin >= 0, "unexpected argument (Begin<0)"); using TResult = integer_sequence; }; } // namespace detail_ template using make_integer_sequence = typename detail_::IntSeqImpl::TResult; template using make_index_sequence = make_integer_sequence; template using index_sequence_for = make_index_sequence; } // namespace ROBIN_HOOD_STD #endif namespace detail { // make sure we static_cast to the correct type for hash_int #if ROBIN_HOOD(BITNESS) == 64 using SizeT = uint64_t; #else using SizeT = uint32_t; #endif template T rotr(T x, unsigned k) { return (x >> k) | (x << (8U * sizeof(T) - k)); } // This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned // char*'} to // 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target // type". Use with // care! template inline T reinterpret_cast_no_cast_align_warning(void *ptr) noexcept { return reinterpret_cast(ptr); } template inline T reinterpret_cast_no_cast_align_warning(void const *ptr) noexcept { return reinterpret_cast(ptr); } // make sure this is not inlined as it is slow and dramatically enlarges code, // thus making other // inlinings more difficult. Throws are also generally the slow path. template [[noreturn]] ROBIN_HOOD(NOINLINE) #if ROBIN_HOOD(HAS_EXCEPTIONS) void doThrow(Args &&... args) { // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) throw E(std::forward(args)...); } #else void doThrow(Args &&... ROBIN_HOOD_UNUSED(args) /*unused*/) { abort(); } #endif template T *assertNotNull(T *t, Args &&... args) { if (ROBIN_HOOD_UNLIKELY(nullptr == t)) { doThrow(std::forward(args)...); } return t; } template inline T unaligned_load(void const *ptr) noexcept { // using memcpy so we don't get into unaligned load problems. // compiler should optimize this very well anyways. T t; std::memcpy(&t, ptr, sizeof(T)); return t; } // Allocates bulks of memory for objects of type T. This deallocates the memory // in the destructor, // and keeps a linked list of the allocated memory around. Overhead per // allocation is the size of a // pointer. template class BulkPoolAllocator { public: BulkPoolAllocator() noexcept = default; // does not copy anything, just creates a new allocator. BulkPoolAllocator(const BulkPoolAllocator &ROBIN_HOOD_UNUSED( o) /*unused*/) noexcept : mHead(nullptr), mListForFree(nullptr) {} BulkPoolAllocator(BulkPoolAllocator &&o) noexcept : mHead(o.mHead), mListForFree(o.mListForFree) { o.mListForFree = nullptr; o.mHead = nullptr; } BulkPoolAllocator &operator=(BulkPoolAllocator &&o) noexcept { reset(); mHead = o.mHead; mListForFree = o.mListForFree; o.mListForFree = nullptr; o.mHead = nullptr; return *this; } BulkPoolAllocator & // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) operator=(const BulkPoolAllocator &ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { // does not do anything return *this; } ~BulkPoolAllocator() noexcept { reset(); } // Deallocates all allocated memory. void reset() noexcept { while (mListForFree) { T *tmp = *mListForFree; ROBIN_HOOD_LOG("std::free") std::free(mListForFree); mListForFree = reinterpret_cast_no_cast_align_warning(tmp); } mHead = nullptr; } // allocates, but does NOT initialize. Use in-place new constructor, e.g. // T* obj = pool.allocate(); // ::new (static_cast(obj)) T(); T *allocate() { T *tmp = mHead; if (!tmp) { tmp = performAllocation(); } mHead = *reinterpret_cast_no_cast_align_warning(tmp); return tmp; } // does not actually deallocate but puts it in store. // make sure you have already called the destructor! e.g. with // obj->~T(); // pool.deallocate(obj); void deallocate(T *obj) noexcept { *reinterpret_cast_no_cast_align_warning(obj) = mHead; mHead = obj; } // Adds an already allocated block of memory to the allocator. This allocator // is from now on // responsible for freeing the data (with free()). If the provided data is not // large enough to // make use of, it is immediately freed. Otherwise it is reused and freed in // the destructor. void addOrFree(void *ptr, const size_t numBytes) noexcept { // calculate number of available elements in ptr if (numBytes < ALIGNMENT + ALIGNED_SIZE) { // not enough data for at least one element. Free and return. ROBIN_HOOD_LOG("std::free") std::free(ptr); } else { ROBIN_HOOD_LOG("add to buffer") add(ptr, numBytes); } } void swap(BulkPoolAllocator &other) noexcept { using std::swap; swap(mHead, other.mHead); swap(mListForFree, other.mListForFree); } private: // iterates the list of allocated memory to calculate how many to alloc next. // Recalculating this each time saves us a size_t member. // This ignores the fact that memory blocks might have been added manually // with addOrFree. In // practice, this should not matter much. ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept { auto tmp = mListForFree; size_t numAllocs = MinNumAllocs; while (numAllocs * 2 <= MaxNumAllocs && tmp) { auto x = reinterpret_cast(tmp); tmp = *x; numAllocs *= 2; } return numAllocs; } // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). void add(void *ptr, const size_t numBytes) noexcept { const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; auto data = reinterpret_cast(ptr); // link free list auto x = reinterpret_cast(data); *x = mListForFree; mListForFree = data; // create linked list for newly allocated data auto *const headT = reinterpret_cast_no_cast_align_warning( reinterpret_cast(ptr) + ALIGNMENT); auto *const head = reinterpret_cast(headT); // Visual Studio compiler automatically unrolls this loop, which is pretty // cool for (size_t i = 0; i < numElements; ++i) { *reinterpret_cast_no_cast_align_warning( head + i * ALIGNED_SIZE) = head + (i + 1) * ALIGNED_SIZE; } // last one points to 0 *reinterpret_cast_no_cast_align_warning( head + (numElements - 1) * ALIGNED_SIZE) = mHead; mHead = headT; } // Called when no memory is available (mHead == 0). // Don't inline this slow path. ROBIN_HOOD(NOINLINE) T *performAllocation() { size_t const numElementsToAlloc = calcNumElementsToAlloc(); // alloc new memory: [prev |T, T, ... T] size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE << " * " << numElementsToAlloc) add(assertNotNull(std::malloc(bytes)), bytes); return mHead; } // enforce byte alignment of the T's #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) static constexpr size_t ALIGNMENT = (std::max)(std::alignment_of::value, std::alignment_of::value); #else static const size_t ALIGNMENT = (ROBIN_HOOD_STD::alignment_of::value > ROBIN_HOOD_STD::alignment_of::value) ? ROBIN_HOOD_STD::alignment_of::value : +ROBIN_HOOD_STD::alignment_of::value; // the + is for // walkarround #endif static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; static_assert(MinNumAllocs >= 1, "MinNumAllocs"); static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); static_assert(ALIGNED_SIZE >= sizeof(T *), "ALIGNED_SIZE"); static_assert(0 == (ALIGNED_SIZE % sizeof(T *)), "ALIGNED_SIZE mod"); static_assert(ALIGNMENT >= sizeof(T *), "ALIGNMENT"); T *mHead{nullptr}; T **mListForFree{nullptr}; }; template struct NodeAllocator; // dummy allocator that does nothing template struct NodeAllocator { // we are not using the data, so just free it. void addOrFree(void *ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { ROBIN_HOOD_LOG("std::free") std::free(ptr); } }; template struct NodeAllocator : public BulkPoolAllocator {}; // c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it // either, so I'm making // my own here. namespace swappable { #if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) using std::swap; template struct nothrow { static const bool value = noexcept(swap(std::declval(), std::declval())); }; #else template struct nothrow { static const bool value = std::is_nothrow_swappable::value; }; #endif } // namespace swappable } // namespace detail struct is_transparent_tag {}; // A custom pair implementation is used in the map because std::pair is not // is_trivially_copyable, // which means it would not be allowed to be used in std::memcpy. This struct // is copyable, which is // also tested. template struct pair { using first_type = T1; using second_type = T2; template ::value && std::is_default_constructible::value>::type> constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) : first(), second() {} // pair constructors are explicit so we don't accidentally call this ctor when // we don't have to. explicit constexpr pair(std::pair const &o) noexcept( noexcept(T1(std::declval())) && noexcept(T2(std::declval()))) : first(o.first), second(o.second) {} // pair constructors are explicit so we don't accidentally call this ctor when // we don't have to. explicit constexpr pair(std::pair &&o) noexcept( noexcept(T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(o.first)), second(std::move(o.second)) {} constexpr pair(T1 &&a, T2 &&b) noexcept( noexcept(T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(a)), second(std::move(b)) {} template constexpr pair(U1 &&a, U2 &&b) noexcept( noexcept(T1(std::forward(std::declval()))) && noexcept(T2(std::forward(std::declval())))) : first(std::forward(a)), second(std::forward(b)) {} template // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize // all members" // if this constructor is constexpr #if !ROBIN_HOOD(BROKEN_CONSTEXPR) constexpr #endif pair(std::piecewise_construct_t /*unused*/, std::tuple a, std::tuple b) noexcept(noexcept(pair(std::declval &>(), std::declval &>(), ROBIN_HOOD_STD::index_sequence_for< U1...>(), ROBIN_HOOD_STD::index_sequence_for< U2...>()))) : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), ROBIN_HOOD_STD::index_sequence_for()) { } // constructor called from the std::piecewise_construct_t ctor template pair( std::tuple &a, std::tuple &b, ROBIN_HOOD_STD::index_sequence /*unused*/, ROBIN_HOOD_STD::index_sequence< I2...> /*unused*/) noexcept(noexcept(T1(std:: forward(std::get( std::declval< std::tuple &>()))...)) && noexcept(T2(std::forward(std::get( std::declval< std::tuple &>()))...))) : first(std::forward(std::get(a))...), second(std::forward(std::get(b))...) { // make visual studio compiler happy about warning about unused a & b. // Visual studio's pair implementation disables warning 4100. (void)a; (void)b; } void swap(pair &o) noexcept((detail::swappable::nothrow::value) && (detail::swappable::nothrow::value)) { using std::swap; swap(first, o.first); swap(second, o.second); } T1 first; // NOLINT(misc-non-private-member-variables-in-classes) T2 second; // NOLINT(misc-non-private-member-variables-in-classes) }; template inline void swap(pair &a, pair &b) noexcept( noexcept(std::declval &>().swap(std::declval &>()))) { a.swap(b); } template inline constexpr bool operator==(pair const &x, pair const &y) { return (x.first == y.first) && (x.second == y.second); } template inline constexpr bool operator!=(pair const &x, pair const &y) { return !(x == y); } template inline constexpr bool operator<(pair const &x, pair const &y) noexcept( noexcept(std::declval() < std::declval()) && noexcept(std::declval() < std::declval())) { return x.first < y.first || (!(y.first < x.first) && x.second < y.second); } template inline constexpr bool operator>(pair const &x, pair const &y) { return y < x; } template inline constexpr bool operator<=(pair const &x, pair const &y) { return !(x > y); } template inline constexpr bool operator>=(pair const &x, pair const &y) { return !(x < y); } inline size_t hash_bytes(void const *ptr, size_t len) noexcept { static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); static constexpr uint64_t seed = UINT64_C(0xe17a1465); static constexpr unsigned int r = 47; auto const *const data64 = static_cast(ptr); uint64_t h = seed ^ (len * m); size_t const n_blocks = len / 8; for (size_t i = 0; i < n_blocks; ++i) { auto k = detail::unaligned_load(data64 + i); k *= m; k ^= k >> r; k *= m; h ^= k; h *= m; } auto const *const data8 = reinterpret_cast(data64 + n_blocks); switch (len & 7U) { case 7: h ^= static_cast(data8[6]) << 48U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 6: h ^= static_cast(data8[5]) << 40U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 5: h ^= static_cast(data8[4]) << 32U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 4: h ^= static_cast(data8[3]) << 24U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 3: h ^= static_cast(data8[2]) << 16U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 2: h ^= static_cast(data8[1]) << 8U; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH case 1: h ^= static_cast(data8[0]); h *= m; ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH default: break; } h ^= h >> r; // not doing the final step here, because this will be done by keyToIdx // anyways // h *= m; // h ^= h >> r; return static_cast(h); } inline size_t hash_int(uint64_t x) noexcept { // tried lots of different hashes, let's stick with murmurhash3. It's simple, // fast, well tested, // and doesn't need any special 128bit operations. x ^= x >> 33U; x *= UINT64_C(0xff51afd7ed558ccd); x ^= x >> 33U; // not doing the final step here, because this will be done by keyToIdx // anyways // x *= UINT64_C(0xc4ceb9fe1a85ec53); // x ^= x >> 33U; return static_cast(x); } // A thin wrapper around std::hash, performing an additional simple mixing step // of the result. template struct hash : public std::hash { size_t operator()(T const &obj) const noexcept(noexcept( std::declval>().operator()(std::declval()))) { // call base hash auto result = std::hash::operator()(obj); // return mixed of that, to be save against identity has return hash_int(static_cast(result)); } }; template struct hash> { size_t operator()(std::basic_string const &str) const noexcept { return hash_bytes(str.data(), sizeof(CharT) * str.size()); } }; #if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) template struct hash> { size_t operator()(std::basic_string_view const &sv) const noexcept { return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); } }; #endif template struct hash { size_t operator()(T *ptr) const noexcept { return hash_int(reinterpret_cast(ptr)); } }; template struct hash> { size_t operator()(std::unique_ptr const &ptr) const noexcept { return hash_int(reinterpret_cast(ptr.get())); } }; template struct hash> { size_t operator()(std::shared_ptr const &ptr) const noexcept { return hash_int(reinterpret_cast(ptr.get())); } }; template struct hash::value>::type> { size_t operator()(Enum e) const noexcept { using Underlying = typename std::underlying_type::type; return hash{}(static_cast(e)); } }; #define ROBIN_HOOD_HASH_INT(T) \ template <> \ struct hash { \ size_t operator()(T const &obj) const noexcept { \ return hash_int(static_cast(obj)); \ } \ } #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wuseless-cast" #endif // see https://en.cppreference.com/w/cpp/utility/hash ROBIN_HOOD_HASH_INT(bool); ROBIN_HOOD_HASH_INT(char); ROBIN_HOOD_HASH_INT(signed char); ROBIN_HOOD_HASH_INT(unsigned char); ROBIN_HOOD_HASH_INT(char16_t); ROBIN_HOOD_HASH_INT(char32_t); #if ROBIN_HOOD(HAS_NATIVE_WCHART) ROBIN_HOOD_HASH_INT(wchar_t); #endif ROBIN_HOOD_HASH_INT(short); ROBIN_HOOD_HASH_INT(unsigned short); ROBIN_HOOD_HASH_INT(int); ROBIN_HOOD_HASH_INT(unsigned int); ROBIN_HOOD_HASH_INT(long); ROBIN_HOOD_HASH_INT(long long); ROBIN_HOOD_HASH_INT(unsigned long); ROBIN_HOOD_HASH_INT(unsigned long long); #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop #endif namespace detail { template struct void_type { using type = void; }; template struct has_is_transparent : public std::false_type {}; template struct has_is_transparent::type> : public std::true_type {}; // using wrapper classes for hash and key_equal prevents the diamond problem // when the same type // is used. see https://stackoverflow.com/a/28771920/48181 template struct WrapHash : public T { WrapHash() = default; explicit WrapHash(T const &o) noexcept(noexcept(T(std::declval()))) : T(o) {} }; template struct WrapKeyEqual : public T { WrapKeyEqual() = default; explicit WrapKeyEqual(T const &o) noexcept( noexcept(T(std::declval()))) : T(o) {} }; // A highly optimized hashmap implementation, using the Robin Hood algorithm. // // In most cases, this map should be usable as a drop-in replacement for // std::unordered_map, but // be about 2x faster in most cases and require much less allocations. // // This implementation uses the following memory layout: // // [Node, Node, ... Node | info, info, ... infoSentinel ] // // * Node: either a DataNode that directly has the std::pair as // member, // or a DataNode with a pointer to std::pair. Which DataNode // representation to use // depends on how fast the swap() operation is. Heuristically, this is // automatically choosen // based on sizeof(). there are always 2^n Nodes. // // * info: Each Node in the map has a corresponding info byte, so there are 2^n // info bytes. // Each byte is initialized to 0, meaning the corresponding Node is empty. Set // to 1 means the // corresponding node contains data. Set to 2 means the corresponding Node is // filled, but it // actually belongs to the previous position and was pushed out because that // place is already // taken. // // * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at // end() without the // need for a idx variable. // // According to STL, order of templates has effect on throughput. That's why // I've moved the // boolean to the front. // https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ template class Table : public WrapHash, public WrapKeyEqual, detail::NodeAllocator< typename std::conditional< std::is_void::value, Key, robin_hood::pair< typename std::conditional::type, T>>::type, 4, 16384, IsFlat> { public: static constexpr bool is_flat = IsFlat; static constexpr bool is_map = !std::is_void::value; static constexpr bool is_set = !is_map; static constexpr bool is_transparent = has_is_transparent::value && has_is_transparent::value; using key_type = Key; using mapped_type = T; using value_type = typename std::conditional< is_set, Key, robin_hood::pair::type, T>>::type; using size_type = size_t; using hasher = Hash; using key_equal = KeyEqual; using Self = Table; private: static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, "MaxLoadFactor100 needs to be >10 && < 100"); using WHash = WrapHash; using WKeyEqual = WrapKeyEqual; // configuration defaults // make sure we have 8 elements, needed to quickly rehash mInfo static constexpr size_t InitialNumElements = sizeof(uint64_t); static constexpr uint32_t InitialInfoNumBits = 5; static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; static constexpr size_t InfoMask = InitialInfoInc - 1U; static constexpr uint8_t InitialInfoHashShift = 0; using DataPool = detail::NodeAllocator; // type needs to be wider than uint8_t. using InfoType = uint32_t; // DataNode //////////////////////////////////////////////////////// // Primary template for the data node. We have special implementations for // small and big // objects. For large objects it is assumed that swap() is fairly slow, so we // allocate these // on the heap so swap merely swaps a pointer. template class DataNode {}; // Small: just allocate on the stack. template class DataNode final { public: template explicit DataNode( M &ROBIN_HOOD_UNUSED(map) /*unused*/, Args &&... args) noexcept(noexcept(value_type(std:: forward( args)...))) : mData(std::forward(args)...) {} DataNode( M &ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode &&n) noexcept(std::is_nothrow_move_constructible::value) : mData(std::move(n.mData)) {} // doesn't do anything void destroy(M &ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {} void destroyDoNotDeallocate() noexcept {} value_type const *operator->() const noexcept { return &mData; } value_type *operator->() noexcept { return &mData; } const value_type &operator*() const noexcept { return mData; } value_type &operator*() noexcept { return mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() noexcept { return mData.first; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() noexcept { return mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() const noexcept { return mData.first; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() const noexcept { return mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getSecond() noexcept { return mData.second; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getSecond() const noexcept { return mData.second; } void swap(DataNode &o) noexcept( noexcept(std::declval().swap(std::declval()))) { mData.swap(o.mData); } private: value_type mData; }; // big object: allocate on heap. template class DataNode { public: template explicit DataNode(M &map, Args &&... args) : mData(map.allocate()) { ::new (static_cast(mData)) value_type(std::forward(args)...); } DataNode(M &ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode &&n) noexcept : mData(std::move(n.mData)) {} void destroy(M &map) noexcept { // don't deallocate, just put it into list of datapool. mData->~value_type(); map.deallocate(mData); } void destroyDoNotDeallocate() noexcept { mData->~value_type(); } value_type const *operator->() const noexcept { return mData; } value_type *operator->() noexcept { return mData; } const value_type &operator*() const { return *mData; } value_type &operator*() { return *mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() noexcept { return mData->first; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() noexcept { return *mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() const noexcept { return mData->first; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getFirst() const noexcept { return *mData; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getSecond() noexcept { return mData->second; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::type getSecond() const noexcept { return mData->second; } void swap(DataNode &o) noexcept { using std::swap; swap(mData, o.mData); } private: value_type *mData; }; using Node = DataNode; // helpers for insertKeyPrepareEmptySpot: extract first entry (only const // required) ROBIN_HOOD(NODISCARD) key_type const &getFirstConst(Node const &n) const noexcept { return n.getFirst(); } // in case we have void mapped_type, we are not using a pair, thus we just // route k through. // No need to disable this because it's just not used if not applicable. ROBIN_HOOD(NODISCARD) key_type const &getFirstConst(key_type const &k) const noexcept { return k; } // in case we have non-void mapped_type, we have a standard robin_hood::pair template ROBIN_HOOD(NODISCARD) typename std::enable_if::value, key_type const &>::type getFirstConst(value_type const &vt) const noexcept { return vt.first; } // Cloner ////////////////////////////////////////////////////////// template struct Cloner; // fast path: Just copy data, without allocating anything. template struct Cloner { void operator()(M const &source, M &target) const { auto const *const src = reinterpret_cast(source.mKeyVals); auto *tgt = reinterpret_cast(target.mKeyVals); auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); } }; template struct Cloner { void operator()(M const &s, M &t) const { auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); for (size_t i = 0; i < numElementsWithBuffer; ++i) { if (t.mInfo[i]) { ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); } } } }; // Destroyer /////////////////////////////////////////////////////// template struct Destroyer {}; template struct Destroyer { void nodes(M &m) const noexcept { m.mNumElements = 0; } void nodesDoNotDeallocate(M &m) const noexcept { m.mNumElements = 0; } }; template struct Destroyer { void nodes(M &m) const noexcept { m.mNumElements = 0; // clear also resets mInfo to 0, that's sometimes not necessary. auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { if (0 != m.mInfo[idx]) { Node &n = m.mKeyVals[idx]; n.destroy(m); n.~Node(); } } } void nodesDoNotDeallocate(M &m) const noexcept { m.mNumElements = 0; // clear also resets mInfo to 0, that's sometimes not necessary. auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { if (0 != m.mInfo[idx]) { Node &n = m.mKeyVals[idx]; n.destroyDoNotDeallocate(); n.~Node(); } } } }; // Iter //////////////////////////////////////////////////////////// struct fast_forward_tag {}; // generic iterator for both const_iterator and iterator. template // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) class Iter { private: using NodePtr = typename std::conditional::type; public: using difference_type = std::ptrdiff_t; using value_type = typename Self::value_type; using reference = typename std::conditional::type; using pointer = typename std::conditional::type; using iterator_category = std::forward_iterator_tag; // default constructed iterator can be compared to itself, but WON'T return // true when // compared to end(). Iter() = default; // Rule of zero: nothing specified. The conversion constructor is only // enabled for // iterator to const_iterator, so it doesn't accidentally work as a copy // ctor. // Conversion constructor from iterator to const_iterator. template ::type> // NOLINTNEXTLINE(hicpp-explicit-conversions) Iter(Iter const &other) noexcept : mKeyVals(other.mKeyVals), mInfo(other.mInfo) {} Iter(NodePtr valPtr, uint8_t const *infoPtr) noexcept : mKeyVals(valPtr), mInfo(infoPtr) {} Iter(NodePtr valPtr, uint8_t const *infoPtr, fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept : mKeyVals(valPtr), mInfo(infoPtr) { fastForward(); } template ::type> Iter &operator=(Iter const &other) noexcept { mKeyVals = other.mKeyVals; mInfo = other.mInfo; return *this; } // prefix increment. Undefined behavior if we are at end()! Iter &operator++() noexcept { mInfo++; mKeyVals++; fastForward(); return *this; } Iter operator++(int)noexcept { Iter tmp = *this; ++(*this); return tmp; } reference operator*() const { return **mKeyVals; } pointer operator->() const { return &**mKeyVals; } template bool operator==(Iter const &o) const noexcept { return mKeyVals == o.mKeyVals; } template bool operator!=(Iter const &o) const noexcept { return mKeyVals != o.mKeyVals; } private: // fast forward to the next non-free info byte // I've tried a few variants that don't depend on intrinsics, but // unfortunately they are // quite a bit slower than this one. So I've reverted that change again. See // map_benchmark. void fastForward() noexcept { size_t n = 0; while (0U == (n = detail::unaligned_load(mInfo))) { mInfo += sizeof(size_t); mKeyVals += sizeof(size_t); } #if defined(ROBIN_HOOD_DISABLE_INTRINSICS) // we know for certain that within the next 8 bytes we'll find a non-zero // one. if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { mInfo += 4; mKeyVals += 4; } if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { mInfo += 2; mKeyVals += 2; } if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) { mInfo += 1; mKeyVals += 1; } #else #if ROBIN_HOOD(LITTLE_ENDIAN) auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; #else auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; #endif mInfo += inc; mKeyVals += inc; #endif } friend class Table; NodePtr mKeyVals{nullptr}; uint8_t const *mInfo{nullptr}; }; //////////////////////////////////////////////////////////////////// // highly performance relevant code. // Lower bits are used for indexing into the array (2^n size) // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. template void keyToIdx(HashKey &&key, size_t *idx, InfoType *info) const { // In addition to whatever hash is used, add another mul & shift so we get // better hashing. // This serves as a bad hash prevention, if the given data is // badly mixed. auto h = static_cast(WHash::operator()(key)); h *= mHashMultiplier; h ^= h >> 33U; // the lower InitialInfoNumBits are reserved for info. *info = mInfoInc + static_cast((h & InfoMask) >> mInfoHashShift); *idx = (static_cast(h) >> InitialInfoNumBits) & mMask; } // forwards the index by one, wrapping around at the end void next(InfoType *info, size_t *idx) const noexcept { *idx = *idx + 1; *info += mInfoInc; } void nextWhileLess(InfoType *info, size_t *idx) const noexcept { // unrolling this by hand did not bring any speedups. while (*info < mInfo[*idx]) { next(info, idx); } } // Shift everything up by one element. Tries to move stuff around. void shiftUp(size_t startIdx, size_t const insertion_idx) noexcept( std::is_nothrow_move_assignable::value) { auto idx = startIdx; ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); while (--idx != insertion_idx) { mKeyVals[idx] = std::move(mKeyVals[idx - 1]); } idx = startIdx; while (idx != insertion_idx) { ROBIN_HOOD_COUNT(shiftUp) mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { mMaxNumElementsAllowed = 0; } --idx; } } void shiftDown(size_t idx) noexcept( std::is_nothrow_move_assignable::value) { // until we find one that is either empty or has zero offset. // TODO(martinus) we don't need to move everything, just the last one for // the same // bucket. mKeyVals[idx].destroy(*this); // until we find one that is either empty or has zero offset. while (mInfo[idx + 1] >= 2 * mInfoInc) { ROBIN_HOOD_COUNT(shiftDown) mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); mKeyVals[idx] = std::move(mKeyVals[idx + 1]); ++idx; } mInfo[idx] = 0; // don't destroy, we've moved it // mKeyVals[idx].destroy(*this); mKeyVals[idx].~Node(); } // copy of find(), except that it returns iterator instead of const_iterator. template ROBIN_HOOD(NODISCARD) size_t findIdx(Other const &key) const { size_t idx{}; InfoType info{}; keyToIdx(key, &idx, &info); do { // unrolling this twice gives a bit of a speedup. More unrolling did not // help. if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()( key, mKeyVals[idx].getFirst()))) { return idx; } next(&info, &idx); if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()( key, mKeyVals[idx].getFirst()))) { return idx; } next(&info, &idx); } while (info <= mInfo[idx]); // nothing found! return mMask == 0 ? 0 : static_cast(std::distance( mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); } void cloneData(const Table &o) { Cloner()(o, *this); } // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is // resized. // @return True on success, false if something went wrong void insert_move(Node &&keyval) { // we don't retry, fail if overflowing // don't need to check max num elements if (0 == mMaxNumElementsAllowed && !try_increase_info()) { throwOverflowError(); } size_t idx{}; InfoType info{}; keyToIdx(keyval.getFirst(), &idx, &info); // skip forward. Use <= because we are certain that the element is not // there. while (info <= mInfo[idx]) { idx = idx + 1; info += mInfoInc; } // key not found, so we are now exactly where we want to insert it. auto const insertion_idx = idx; auto const insertion_info = static_cast(info); if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { mMaxNumElementsAllowed = 0; } // find an empty spot while (0 != mInfo[idx]) { next(&info, &idx); } auto &l = mKeyVals[insertion_idx]; if (idx == insertion_idx) { ::new (static_cast(&l)) Node(std::move(keyval)); } else { shiftUp(idx, insertion_idx); l = std::move(keyval); } // put at empty spot mInfo[insertion_idx] = insertion_info; ++mNumElements; } public: using iterator = Iter; using const_iterator = Iter; Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) : WHash(), WKeyEqual() { ROBIN_HOOD_TRACE(this) } // Creates an empty hash map. Nothing is allocated yet, this happens at the // first insert. // This tremendously speeds up ctor & dtor of a map that never receives an // element. The // penalty is payed at the first insert, and not before. Lookup of this empty // map works // because everybody points to DummyInfoByte::b. parameter bucket_count is // dictated by the // standard, but we can ignore it. explicit Table( size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash &h = Hash{}, const KeyEqual &equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) : WHash(h), WKeyEqual(equal) { ROBIN_HOOD_TRACE(this) } template Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash &h = Hash{}, const KeyEqual &equal = KeyEqual{}) : WHash(h), WKeyEqual(equal) { ROBIN_HOOD_TRACE(this) insert(first, last); } Table(std::initializer_list initlist, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash &h = Hash{}, const KeyEqual &equal = KeyEqual{}) : WHash(h), WKeyEqual(equal) { ROBIN_HOOD_TRACE(this) insert(initlist.begin(), initlist.end()); } Table(Table &&o) noexcept : WHash(std::move(static_cast(o))), WKeyEqual(std::move(static_cast(o))), DataPool(std::move(static_cast(o))) { ROBIN_HOOD_TRACE(this) if (o.mMask) { mHashMultiplier = std::move(o.mHashMultiplier); mKeyVals = std::move(o.mKeyVals); mInfo = std::move(o.mInfo); mNumElements = std::move(o.mNumElements); mMask = std::move(o.mMask); mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); mInfoInc = std::move(o.mInfoInc); mInfoHashShift = std::move(o.mInfoHashShift); // set other's mask to 0 so its destructor won't do anything o.init(); } } Table &operator=(Table &&o) noexcept { ROBIN_HOOD_TRACE(this) if (&o != this) { if (o.mMask) { // only move stuff if the other map actually has some data destroy(); mHashMultiplier = std::move(o.mHashMultiplier); mKeyVals = std::move(o.mKeyVals); mInfo = std::move(o.mInfo); mNumElements = std::move(o.mNumElements); mMask = std::move(o.mMask); mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); mInfoInc = std::move(o.mInfoInc); mInfoHashShift = std::move(o.mInfoHashShift); WHash::operator=(std::move(static_cast(o))); WKeyEqual::operator=(std::move(static_cast(o))); DataPool::operator=(std::move(static_cast(o))); o.init(); } else { // nothing in the other map => just clear us. clear(); } } return *this; } Table(const Table &o) : WHash(static_cast(o)), WKeyEqual(static_cast(o)), DataPool(static_cast(o)) { ROBIN_HOOD_TRACE(this) if (!o.empty()) { // not empty: create an exact copy. it is also possible to just iterate // through all // elements and insert them, but copying is probably faster. auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer << ")") mHashMultiplier = o.mHashMultiplier; mKeyVals = static_cast( detail::assertNotNull(std::malloc(numBytesTotal))); // no need for calloc because clonData does memcpy mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); mNumElements = o.mNumElements; mMask = o.mMask; mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; mInfoInc = o.mInfoInc; mInfoHashShift = o.mInfoHashShift; cloneData(o); } } // Creates a copy of the given map. Copy constructor of each entry is used. // Not sure why clang-tidy thinks this doesn't handle self assignment, it does // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) Table &operator=(Table const &o) { ROBIN_HOOD_TRACE(this) if (&o == this) { // prevent assigning of itself return *this; } // we keep using the old allocator and not assign the new one, because we // want to keep // the memory available. when it is the same size. if (o.empty()) { if (0 == mMask) { // nothing to do, we are empty too return *this; } // not empty: destroy what we have there // clear also resets mInfo to 0, that's sometimes not necessary. destroy(); init(); WHash::operator=(static_cast(o)); WKeyEqual::operator=(static_cast(o)); DataPool::operator=(static_cast(o)); return *this; } // clean up old stuff Destroyer::value>{} .nodes(*this); if (mMask != o.mMask) { // no luck: we don't have the same array size allocated, so we need to // realloc. if (0 != mMask) { // only deallocate if we actually have data! ROBIN_HOOD_LOG("std::free") std::free(mKeyVals); } auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer << ")") mKeyVals = static_cast( detail::assertNotNull(std::malloc(numBytesTotal))); // no need for calloc here because cloneData performs a memcpy. mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); // sentinel is set in cloneData } WHash::operator=(static_cast(o)); WKeyEqual::operator=(static_cast(o)); DataPool::operator=(static_cast(o)); mHashMultiplier = o.mHashMultiplier; mNumElements = o.mNumElements; mMask = o.mMask; mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; mInfoInc = o.mInfoInc; mInfoHashShift = o.mInfoHashShift; cloneData(o); return *this; } // Swaps everything between the two maps. void swap(Table &o) { ROBIN_HOOD_TRACE(this) using std::swap; swap(o, *this); } // Clears all data, without resizing. void clear() { ROBIN_HOOD_TRACE(this) if (empty()) { // don't do anything! also important because we don't want to write to // DummyInfoByte::b, even though we would just write 0 to it. return; } Destroyer::value>{} .nodes(*this); auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); // clear everything, then set the sentinel again uint8_t const z = 0; std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); mInfo[numElementsWithBuffer] = 1; mInfoInc = InitialInfoInc; mInfoHashShift = InitialInfoHashShift; } // Destroys the map and all it's contents. ~Table() { ROBIN_HOOD_TRACE(this) destroy(); } // Checks if both tables contain the same entries. Order is irrelevant. bool operator==(const Table &other) const { ROBIN_HOOD_TRACE(this) if (other.size() != size()) { return false; } for (auto const &otherEntry : other) { if (!has(otherEntry)) { return false; } } return true; } bool operator!=(const Table &other) const { ROBIN_HOOD_TRACE(this) return !operator==(other); } template typename std::enable_if::value, Q &>::type operator[]( const key_type &key) { ROBIN_HOOD_TRACE(this) auto idxAndState = insertKeyPrepareEmptySpot(key); switch (idxAndState.second) { case InsertionState::key_found: break; case InsertionState::new_node: ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple()); break; case InsertionState::overwrite_node: mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple()); break; case InsertionState::overflow_error: throwOverflowError(); } return mKeyVals[idxAndState.first].getSecond(); } template typename std::enable_if::value, Q &>::type operator[]( key_type &&key) { ROBIN_HOOD_TRACE(this) auto idxAndState = insertKeyPrepareEmptySpot(key); switch (idxAndState.second) { case InsertionState::key_found: break; case InsertionState::new_node: ::new (static_cast(&mKeyVals[idxAndState.first])) Node( *this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::forward_as_tuple()); break; case InsertionState::overwrite_node: mKeyVals[idxAndState.first] = Node( *this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::forward_as_tuple()); break; case InsertionState::overflow_error: throwOverflowError(); } return mKeyVals[idxAndState.first].getSecond(); } template void insert(Iter first, Iter last) { for (; first != last; ++first) { // value_type ctor needed because this might be called with std::pair's insert(value_type(*first)); } } void insert(std::initializer_list ilist) { for (auto &&vt : ilist) { insert(std::move(vt)); } } template std::pair emplace(Args &&... args) { ROBIN_HOOD_TRACE(this) Node n{*this, std::forward(args)...}; auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n)); switch (idxAndState.second) { case InsertionState::key_found: n.destroy(*this); break; case InsertionState::new_node: ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::move(n)); break; case InsertionState::overwrite_node: mKeyVals[idxAndState.first] = std::move(n); break; case InsertionState::overflow_error: n.destroy(*this); throwOverflowError(); break; } return std::make_pair( iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), InsertionState::key_found != idxAndState.second); } template std::pair try_emplace(const key_type &key, Args &&... args) { return try_emplace_impl(key, std::forward(args)...); } template std::pair try_emplace(key_type &&key, Args &&... args) { return try_emplace_impl(std::move(key), std::forward(args)...); } template std::pair try_emplace(const_iterator hint, const key_type &key, Args &&... args) { (void)hint; return try_emplace_impl(key, std::forward(args)...); } template std::pair try_emplace(const_iterator hint, key_type &&key, Args &&... args) { (void)hint; return try_emplace_impl(std::move(key), std::forward(args)...); } template std::pair insert_or_assign(const key_type &key, Mapped &&obj) { return insertOrAssignImpl(key, std::forward(obj)); } template std::pair insert_or_assign(key_type &&key, Mapped &&obj) { return insertOrAssignImpl(std::move(key), std::forward(obj)); } template std::pair insert_or_assign(const_iterator hint, const key_type &key, Mapped &&obj) { (void)hint; return insertOrAssignImpl(key, std::forward(obj)); } template std::pair insert_or_assign(const_iterator hint, key_type &&key, Mapped &&obj) { (void)hint; return insertOrAssignImpl(std::move(key), std::forward(obj)); } std::pair insert(const value_type &keyval) { ROBIN_HOOD_TRACE(this) return emplace(keyval); } std::pair insert(value_type &&keyval) { return emplace(std::move(keyval)); } // Returns 1 if key is found, 0 otherwise. size_t count(const key_type &key) const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { return 1; } return 0; } template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::type count( const OtherKey &key) const { ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { return 1; } return 0; } bool contains(const key_type &key) const { // NOLINT(modernize-use-nodiscard) return 1U == count(key); } template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::type contains( const OtherKey &key) const { return 1U == count(key); } // Returns a reference to the value found for key. // Throws std::out_of_range if element cannot be found template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::value, Q &>::type at( key_type const &key) { ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); } return kv->getSecond(); } // Returns a reference to the value found for key. // Throws std::out_of_range if element cannot be found template // NOLINTNEXTLINE(modernize-use-nodiscard) typename std::enable_if::value, Q const &>::type at( key_type const &key) const { ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); } return kv->getSecond(); } const_iterator find( const key_type &key) const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } template const_iterator find(const OtherKey &key, is_transparent_tag /*unused*/) const { ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } template typename std::enable_if< Self_::is_transparent, // NOLINT(modernize-use-nodiscard) const_iterator>::type // NOLINT(modernize-use-nodiscard) find(const OtherKey &key) const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } iterator find(const key_type &key) { ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } template iterator find(const OtherKey &key, is_transparent_tag /*unused*/) { ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } template typename std::enable_if::type find( const OtherKey &key) { ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } iterator begin() { ROBIN_HOOD_TRACE(this) if (empty()) { return end(); } return iterator(mKeyVals, mInfo, fast_forward_tag{}); } const_iterator begin() const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return cbegin(); } const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) if (empty()) { return cend(); } return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); } iterator end() { ROBIN_HOOD_TRACE(this) // no need to supply valid info pointer: end() must not be dereferenced, and // only node // pointer is compared. return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } const_iterator end() const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return cend(); } const_iterator cend() const { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } iterator erase(const_iterator pos) { ROBIN_HOOD_TRACE(this) // its safe to perform const cast here // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); } // Erases element at pos, returns iterator to the next element. iterator erase(iterator pos) { ROBIN_HOOD_TRACE(this) // we assume that pos always points to a valid entry, and not end(). auto const idx = static_cast(pos.mKeyVals - mKeyVals); shiftDown(idx); --mNumElements; if (*pos.mInfo) { // we've backward shifted, return this again return pos; } // no backward shift, return next element return ++pos; } size_t erase(const key_type &key) { ROBIN_HOOD_TRACE(this) size_t idx{}; InfoType info{}; keyToIdx(key, &idx, &info); // check while info matches with the source idx do { if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { shiftDown(idx); --mNumElements; return 1; } next(&info, &idx); } while (info <= mInfo[idx]); // nothing found to delete return 0; } // reserves space for the specified number of elements. Makes sure the old // data fits. // exactly the same as reserve(c). void rehash(size_t c) { // forces a reserve reserve(c, true); } // reserves space for the specified number of elements. Makes sure the old // data fits. // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. void reserve(size_t c) { // reserve, but don't force rehash reserve(c, false); } // If possible reallocates the map to a smaller one. This frees the underlying // table. // Does not do anything if load_factor is too large for decreasing the table's // size. void compact() { ROBIN_HOOD_TRACE(this) auto newSize = InitialNumElements; while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) { newSize *= 2; } if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { throwOverflowError(); } ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") // only actually do anything when the new size is bigger than the old one. // This prevents to // continuously allocate for each reserve() call. if (newSize < mMask + 1) { rehashPowerOfTwo(newSize, true); } } size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return mNumElements; } size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return static_cast(-1); } ROBIN_HOOD(NODISCARD) bool empty() const noexcept { ROBIN_HOOD_TRACE(this) return 0 == mNumElements; } float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return MaxLoadFactor100 / 100.0F; } // Average number of elements per bucket. Since we allow only 1 per bucket float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) ROBIN_HOOD_TRACE(this) return static_cast(size()) / static_cast(mMask + 1); } ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { ROBIN_HOOD_TRACE(this) return mMask; } ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept { if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits::max)() / 100)) { return maxElements * MaxLoadFactor100 / 100; } // we might be a bit inprecise, but since maxElements is quite large that // doesn't matter return (maxElements / 100) * MaxLoadFactor100; } ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { // we add a uint64_t, which houses the sentinel (first byte) and padding so // we can load // 64bit types. return numElements + sizeof(uint64_t); } ROBIN_HOOD(NODISCARD) size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); } // calculation only allowed for 2^n values ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { #if ROBIN_HOOD(BITNESS) == 64 return numElements * sizeof(Node) + calcNumBytesInfo(numElements); #else // make sure we're doing 64bit operations, so we are at least safe against // 32bit overflows. auto const ne = static_cast(numElements); auto const s = static_cast(sizeof(Node)); auto const infos = static_cast(calcNumBytesInfo(numElements)); auto const total64 = ne * s + infos; auto const total = static_cast(total64); if (ROBIN_HOOD_UNLIKELY(static_cast(total) != total64)) { throwOverflowError(); } return total; #endif } private: template ROBIN_HOOD(NODISCARD) typename std::enable_if::value, bool>::type has(const value_type &e) const { ROBIN_HOOD_TRACE(this) auto it = find(e.first); return it != end() && it->second == e.second; } template ROBIN_HOOD(NODISCARD) typename std::enable_if::value, bool>::type has(const value_type &e) const { ROBIN_HOOD_TRACE(this) return find(e) != end(); } void reserve(size_t c, bool forceRehash) { ROBIN_HOOD_TRACE(this) auto const minElementsAllowed = (std::max)(c, mNumElements); auto newSize = InitialNumElements; while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { newSize *= 2; } if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { throwOverflowError(); } ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") // only actually do anything when the new size is bigger than the old one. // This prevents to // continuously allocate for each reserve() call. if (forceRehash || newSize > mMask + 1) { rehashPowerOfTwo(newSize, false); } } // reserves space for at least the specified number of elements. // only works if numBuckets if power of two // True on success, false otherwise void rehashPowerOfTwo(size_t numBuckets, bool forceFree) { ROBIN_HOOD_TRACE(this) Node *const oldKeyVals = mKeyVals; uint8_t const *const oldInfo = mInfo; const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); // resize operation: move stuff initData(numBuckets); if (oldMaxElementsWithBuffer > 1) { for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { if (oldInfo[i] != 0) { // might throw an exception, which is really bad since we are in the // middle of // moving stuff. insert_move(std::move(oldKeyVals[i])); // destroy the node but DON'T destroy the data. oldKeyVals[i].~Node(); } } // this check is not necessary as it's guarded by the previous if, but it // helps // silence g++'s overeager "attempt to free a non-heap object 'map' // [-Werror=free-nonheap-object]" warning. if (oldKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { // don't destroy old data: put it into the pool instead if (forceFree) { std::free(oldKeyVals); } else { DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); } } } } ROBIN_HOOD(NOINLINE) void throwOverflowError() const { #if ROBIN_HOOD(HAS_EXCEPTIONS) throw std::overflow_error("robin_hood::map overflow"); #else abort(); #endif } template std::pair try_emplace_impl(OtherKey &&key, Args &&... args) { ROBIN_HOOD_TRACE(this) auto idxAndState = insertKeyPrepareEmptySpot(key); switch (idxAndState.second) { case InsertionState::key_found: break; case InsertionState::new_node: ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(args)...)); break; case InsertionState::overwrite_node: mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(args)...)); break; case InsertionState::overflow_error: throwOverflowError(); break; } return std::make_pair( iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), InsertionState::key_found != idxAndState.second); } template std::pair insertOrAssignImpl(OtherKey &&key, Mapped &&obj) { ROBIN_HOOD_TRACE(this) auto idxAndState = insertKeyPrepareEmptySpot(key); switch (idxAndState.second) { case InsertionState::key_found: mKeyVals[idxAndState.first].getSecond() = std::forward(obj); break; case InsertionState::new_node: ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(obj))); break; case InsertionState::overwrite_node: mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(obj))); break; case InsertionState::overflow_error: throwOverflowError(); break; } return std::make_pair( iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), InsertionState::key_found != idxAndState.second); } void initData(size_t max_elements) { mNumElements = 0; mMask = max_elements - 1; mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); // calloc also zeroes everything auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer << ")") mKeyVals = reinterpret_cast( detail::assertNotNull(std::calloc(1, numBytesTotal))); mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); // set sentinel mInfo[numElementsWithBuffer] = 1; mInfoInc = InitialInfoInc; mInfoHashShift = InitialInfoHashShift; } enum class InsertionState { overflow_error, key_found, new_node, overwrite_node }; // Finds key, and if not already present prepares a spot where to pot the key // & value. // This potentially shifts nodes out of the way, updates mInfo and number of // inserted // elements, so the only operation left to do is create/assign a new node at // that spot. template std::pair insertKeyPrepareEmptySpot(OtherKey &&key) { for (int i = 0; i < 256; ++i) { size_t idx{}; InfoType info{}; keyToIdx(key, &idx, &info); nextWhileLess(&info, &idx); // while we potentially have a match while (info == mInfo[idx]) { if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { // key already exists, do NOT insert. // see http://en.cppreference.com/w/cpp/container/unordered_map/insert return std::make_pair(idx, InsertionState::key_found); } next(&info, &idx); } // unlikely that this evaluates to true if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { if (!increase_size()) { return std::make_pair(size_t(0), InsertionState::overflow_error); } continue; } // key not found, so we are now exactly where we want to insert it. auto const insertion_idx = idx; auto const insertion_info = info; if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { mMaxNumElementsAllowed = 0; } // find an empty spot while (0 != mInfo[idx]) { next(&info, &idx); } if (idx != insertion_idx) { shiftUp(idx, insertion_idx); } // put at empty spot mInfo[insertion_idx] = static_cast(insertion_info); ++mNumElements; return std::make_pair( insertion_idx, idx == insertion_idx ? InsertionState::new_node : InsertionState::overwrite_node); } // enough attempts failed, so finally give up. return std::make_pair(size_t(0), InsertionState::overflow_error); } bool try_increase_info() { ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements << ", maxNumElementsAllowed=" << calcMaxNumElementsAllowed(mMask + 1)) if (mInfoInc <= 2) { // need to be > 2 so that shift works (otherwise undefined behavior!) return false; } // we got space left, try to make info smaller mInfoInc = static_cast(mInfoInc >> 1U); // remove one bit of the hash, leaving more space for the distance info. // This is extremely fast because we can operate on 8 bytes at once. ++mInfoHashShift; auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); for (size_t i = 0; i < numElementsWithBuffer; i += 8) { auto val = unaligned_load(mInfo + i); val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); std::memcpy(mInfo + i, &val, sizeof(val)); } // update sentinel, which might have been cleared out! mInfo[numElementsWithBuffer] = 1; mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); return true; } // True if resize was possible, false otherwise bool increase_size() { // nothing allocated yet? just allocate InitialNumElements if (0 == mMask) { initData(InitialNumElements); return true; } auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); if (mNumElements < maxNumElementsAllowed && try_increase_info()) { return true; } ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" << maxNumElementsAllowed << ", load=" << (static_cast(mNumElements) * 100.0 / (static_cast(mMask) + 1))) nextHashMultiplier(); if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { // we have to resize, even though there would still be plenty of space // left! // Try to rehash instead. Delete freed memory so we don't steadyily // increase mem in case // we have to rehash a few times rehashPowerOfTwo(mMask + 1, true); } else { // Each resize use a different hash so we don't so easily overflow. // Make sure we only have odd numbers, so that the multiplication is // reversible! rehashPowerOfTwo((mMask + 1) * 2, false); } return true; } void nextHashMultiplier() { // adding an *even* number, so that the multiplier will always stay odd. // This is necessary // so that the hash stays a mixing function (and thus doesn't have any // information loss). mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54); } void destroy() { if (0 == mMask) { // don't deallocate! return; } Destroyer::value>{} .nodesDoNotDeallocate(*this); // This protection against not deleting mMask shouldn't be needed as it's // sufficiently // protected with the 0==mMask check, but I have this anyways because g++ 7 // otherwise // reports a compile error: attempt to free a non-heap object 'fm' // [-Werror=free-nonheap-object] if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { ROBIN_HOOD_LOG("std::free") std::free(mKeyVals); } } void init() noexcept { mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); mInfo = reinterpret_cast(&mMask); mNumElements = 0; mMask = 0; mMaxNumElementsAllowed = 0; mInfoInc = InitialInfoInc; mInfoHashShift = InitialInfoHashShift; } // members are sorted so no padding occurs uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8 Node *mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 16 uint8_t *mInfo = reinterpret_cast(&mMask); // 8 byte 24 size_t mNumElements = 0; // 8 byte 32 size_t mMask = 0; // 8 byte 40 size_t mMaxNumElementsAllowed = 0; // 8 byte 48 InfoType mInfoInc = InitialInfoInc; // 4 byte 52 InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56 // 16 byte 56 if NodeAllocator }; } // namespace detail // map template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_flat_map = detail::Table; template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_node_map = detail::Table; template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_map = detail::Table< sizeof(robin_hood::pair) <= sizeof(size_t) * 6 && std::is_nothrow_move_constructible>::value && std::is_nothrow_move_assignable>::value, MaxLoadFactor100, Key, T, Hash, KeyEqual>; // set template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_flat_set = detail::Table; template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_node_set = detail::Table; template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_set = detail::Table::value && std::is_nothrow_move_assignable::value, MaxLoadFactor100, Key, void, Hash, KeyEqual>; } // namespace robin_hood #endif